From 785688644283620c4715127d6addddaa1f10d7ed Mon Sep 17 00:00:00 2001 From: rbbtsn0w Date: Fri, 29 May 2026 10:38:15 +0800 Subject: [PATCH 1/7] feat(memorylint): close audit/apply loop Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- memorylint/CHANGELOG.md | 23 + memorylint/DESIGN.md | 55 +- memorylint/README.md | 50 +- memorylint/commands/apply.md | 191 +-- memorylint/commands/audit.md | 243 ++-- memorylint/commands/load-agents.md | 65 +- memorylint/scripts/apply_report.py | 114 ++ memorylint/scripts/audit_workspace.py | 47 + memorylint/scripts/load_agents_state.py | 48 + memorylint/scripts/memorylint_core.py | 1107 +++++++++++++++++ memorylint/scripts/scan_fixtures.py | 424 +------ .../bloated-agents/expected-findings.json | 32 +- .../conflicting-rules/expected-findings.json | 24 +- .../expected-findings.json | 13 +- .../monorepo-nested/expected-findings.json | 14 +- .../packages/frontend/package.json | 8 + .../multi-source/expected-findings.json | 46 +- .../expected-findings.json | 24 +- .../redundant-rules/expected-findings.json | 16 +- .../stale-command/expected-findings.json | 8 +- memorylint/tests/test-apply-workflow.sh | 64 + memorylint/tests/test-load-agents-proof.sh | 43 + .../tests/test-memorylint-regressions.sh | 22 + memorylint/tests/test-workspace-audit.sh | 51 + 24 files changed, 2109 insertions(+), 623 deletions(-) create mode 100644 memorylint/scripts/apply_report.py create mode 100644 memorylint/scripts/audit_workspace.py create mode 100644 memorylint/scripts/load_agents_state.py create mode 100644 memorylint/scripts/memorylint_core.py mode change 100755 => 100644 memorylint/scripts/scan_fixtures.py create mode 100644 memorylint/tests/fixtures/monorepo-nested/packages/frontend/package.json create mode 100644 memorylint/tests/test-apply-workflow.sh create mode 100644 memorylint/tests/test-load-agents-proof.sh create mode 100644 memorylint/tests/test-workspace-audit.sh diff --git a/memorylint/CHANGELOG.md b/memorylint/CHANGELOG.md index 4fd98f7..e02a300 100644 --- a/memorylint/CHANGELOG.md +++ b/memorylint/CHANGELOG.md @@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Real workspace-level audit/apply/load-agents helper scripts: + `scripts/audit_workspace.py`, `scripts/apply_report.py`, + `scripts/load_agents_state.py`. +- Canonical ownership / precedence matrix for architecture, domain, + infrastructure, workflow, tooling, and personal preference rules. +- Constitution manual handoff artifact contract for boundary findings that + target `.specify/memory/constitution.md`. +- Executable `edits` support in the machine-readable audit report so safe/apply + runs can use deterministic file changes. + +### Changed + +- Refactored fixture scanning onto a shared audit core so the regression corpus + executes the same detection logic as workspace audit. +- Strengthened the `before_plan` gate to require structured `AGENTS.md` load + proof instead of a verbal acknowledgement only. +- Aligned README / DESIGN / command docs around the executable report schema and + operational audit metrics. +- Updated regression fixtures to match the canonical ownership matrix and real + audit behaviour. + ## [1.5.0] - 2026-05-27 diff --git a/memorylint/DESIGN.md b/memorylint/DESIGN.md index a442634..c3bcb79 100644 --- a/memorylint/DESIGN.md +++ b/memorylint/DESIGN.md @@ -37,7 +37,7 @@ silent authority over long-lived project memory. ## Audit Pipeline -The audit command follows a deterministic conceptual pipeline: +The audit command now follows a deterministic executable pipeline: 1. **Instruction Inventory**: scan instruction sources such as `AGENTS.md`, `.specify/memory/constitution.md`, `CLAUDE.md`, `.cursor/rules/*`, README @@ -49,8 +49,27 @@ The audit command follows a deterministic conceptual pipeline: evidence to each finding. 4. **Drift Detection**: detect `boundary`, `reality`, `conflict`, and `redundancy` drift. -5. **Report Generation**: emit both a human-readable Markdown report and a - machine-readable `memorylint-report.json` artifact. +5. **Executable Report Generation**: emit both a human-readable Markdown report + and a machine-readable `memorylint-report.json` artifact. + +## Source Ownership Matrix + +MemoryLint applies a canonical ownership / precedence model: + +| Category | Canonical Owner | Secondary Sources | +|----------|-----------------|------------------| +| `architecture` | `.specify/memory/constitution.md` | `.cursor/rules/*`, `CLAUDE.md` | +| `domain` | `.specify/memory/constitution.md` | manifests, docs | +| `infrastructure` | root `AGENTS.md` | nested `AGENTS.md`, `CLAUDE.md`, workflows | +| `workflow` | root `AGENTS.md` | nested `AGENTS.md`, `CLAUDE.md` | +| `tooling` | root `AGENTS.md` | tool-specific editor rules | +| `personal_preference` | root `AGENTS.md` | editor-specific restatements | + +Precedence rules: + +1. Constitution wins for shared architecture and domain rules. +2. Root `AGENTS.md` wins for shared workflow, infrastructure, tooling, and preference rules. +3. README, workflows, tests, and manifests are evidence-bearing sources, not canonical owners. ## Machine-Readable Report @@ -75,6 +94,11 @@ gate compares these hashes against current file content before modifying any file. This prevents applying stale findings after instruction files have changed. +Findings may additionally carry: + +- `edits`: deterministic line-scoped mutations for executable safe/apply runs +- `manual_handoff`: constitution-targeted handoff material that cannot be auto-applied + ## Apply Gate Apply has three modes: @@ -87,6 +111,24 @@ Apply has three modes: Safe mode must not move architecture or domain rules, rewrite semantics, delete constitution-owned rules, or apply medium/low-confidence findings. +Boundary fixes targeting the constitution always become **manual handoff +artifacts**. Apply may remove the misplaced secondary copy only when the +handoff has been explicitly approved; it still must not auto-merge into the +constitution. + +## Planning Gate + +`load-agents` is now a verifiable gate rather than a verbal acknowledgement. +Its success output records: + +- the root `AGENTS.md` path +- the SHA-256 hash of the loaded file +- the extracted section list +- the rule summaries inherited into planning + +This creates a machine-checkable `before_plan` proof instead of a best-effort +statement. + Post-apply validation checks: - `AGENTS.md` integrity and critical section preservation; @@ -112,6 +154,12 @@ contract. It covers: each fixture's `expected-findings.json`. This turns the design from a prompt-only contract into a deterministic regression gate. +The same core powers: + +- `scripts/audit_workspace.py` for real workspace audit +- `scripts/apply_report.py` for staleness-checked apply and rollback +- `scripts/load_agents_state.py` for structured planning-gate proof + ## Release Criteria MemoryLint changes are ready to ship only when: @@ -120,4 +168,5 @@ MemoryLint changes are ready to ship only when: - audit/apply/load-agents prompts preserve their safety contracts; - fixture schemas are valid; - deterministic fixture scanning matches expected findings; +- real workspace audit/apply/load-agents scripts stay aligned with the prompt contracts; - repository workflow tests and whitespace checks pass. diff --git a/memorylint/README.md b/memorylint/README.md index 19c77dc..4daae0b 100644 --- a/memorylint/README.md +++ b/memorylint/README.md @@ -4,6 +4,12 @@ Evidence-driven instruction drift checker for Spec Kit. MemoryLint audits long-lived agent instruction files — `AGENTS.md`, `.specify/memory/constitution.md`, `CLAUDE.md`, `.cursor/rules/`, and other sources — to detect boundary violations, stale references, conflicts, and redundancies. Every finding is backed by concrete evidence so reviewers can trust the report before applying any changes. +The current implementation includes executable helpers for all three surfaces: + +- `scripts/audit_workspace.py` +- `scripts/apply_report.py` +- `scripts/load_agents_state.py` + ## Problem Statement In Spec-Driven Development (SDD), AI agents rely on long-lived instruction files: @@ -74,6 +80,22 @@ This extension registers the following Spec Kit lifecycle hooks: Key design constraint: hooks only run **read-only** operations. The `apply` command is never wired to a hook — it is always an explicit user action. +## Canonical Ownership Matrix + +MemoryLint now applies one canonical ownership matrix during audit: + +| Category | Canonical Owner | Notes | +|----------|-----------------|-------| +| `architecture` | `.specify/memory/constitution.md` | editor rules may restate, but do not own | +| `domain` | `.specify/memory/constitution.md` | manifests and docs may reflect, but do not own | +| `infrastructure` | root `AGENTS.md` | nested/editor sources may scope or mirror | +| `workflow` | root `AGENTS.md` | nested/editor sources may scope or mirror | +| `tooling` | root `AGENTS.md` | tool-specific files may add local detail | +| `personal_preference` | root `AGENTS.md` | editor-specific restatements are secondary | + +This matrix is what drives `recommended_destination`, redundancy cleanup, and +constitution handoff generation. + ## Apply Modes | Mode | Behaviour | @@ -103,6 +125,11 @@ It includes `schema_version`, `source_metadata`, `instruction_map`, `findings`, and `metrics`. `source_metadata` records SHA-256 hashes for scanned files so the apply gate can reject stale reports before changing anything. +Executable findings may also include: + +- `edits`: line-scoped file operations used by the apply gate +- `manual_handoff`: constitution-targeted handoff material that must be reviewed by a human + ## Rule Classification Every rule is classified into one of eight categories: @@ -118,17 +145,22 @@ Every rule is classified into one of eight categories: | `obsolete` | References something that no longer exists | | `conflict` | Contradicts another rule | -## Trust Metrics +## Audit Metrics -Every audit report includes a metrics section tracking: +Every audit report now emits run-time metrics that match the executable output: | Metric | Purpose | |--------|---------| -| High-confidence finding acceptance rate | Measures report accuracy | -| False positive rate | Must stay low to maintain trust | -| Suggested diff apply rate | Tracks actionability | -| Real stale/conflicting rules found | Measures value delivered | -| Destructive surprise edits | Must be **zero** | +| Total instruction sources scanned | Shows workspace coverage | +| Total rules catalogued | Shows extracted rule inventory size | +| Total findings | Shows total actionable/non-actionable drift | +| High-confidence findings | Indicates directly evidenced findings | +| Medium-confidence findings | Indicates heuristic findings that need review | +| Low-confidence findings | Indicates weak-evidence findings | +| Files that would be modified by suggested actions | Powers safe preview and apply gating | + +Longitudinal trust KPIs such as false-positive rate or destructive surprise +edits remain release-evaluation signals, not per-run report fields. ## Regression Corpus @@ -147,8 +179,8 @@ MemoryLint includes a regression corpus of nine fixture repos under `tests/fixtu | `post-apply-breakage` | Apply safety validation | The fixture corpus is executable. `memorylint/scripts/scan_fixtures.py --check` -generates deterministic findings for every fixture and compares them with each -fixture's `expected-findings.json`. +re-runs the real audit core against every fixture and compares the normalized +findings with each fixture's `expected-findings.json`. ## Design diff --git a/memorylint/commands/apply.md b/memorylint/commands/apply.md index f666912..a31c4ae 100644 --- a/memorylint/commands/apply.md +++ b/memorylint/commands/apply.md @@ -1,132 +1,142 @@ +--- +scripts: + - scripts/apply_report.py + - scripts/memorylint_core.py +--- $ARGUMENTS # Role -You are the MemoryLint Apply Gate. Your task is to read a previously generated -**MemoryLint Drift Report**, confirm which fixes should be applied, execute them -with safety checks, and validate the result. +You are the MemoryLint Apply Gate. Read a previously generated +`memorylint-report.json`, decide which fixes are eligible, apply only approved +changes, validate the result, and roll everything back if any validation fails. -**Default behaviour is report-only.** You must never modify files without -explicit user confirmation or an explicit mode override. +**Default behaviour is `report-only`.** Never mutate files without explicit mode +selection or explicit approval. # Objective -1. Parse the most recent MemoryLint Drift Report. Prefer the fenced - `memorylint-report.json` artifact as the authoritative source; use the - Markdown report only for human-readable context. +1. Parse the most recent MemoryLint Drift Report. 2. Determine the apply mode. -3. Execute the appropriate fixes with pre-apply and post-apply validation. -4. Revert all changes if any validation step fails. +3. Preview the pending changes. +4. Apply approved edits only. +5. Run post-apply validation and Rollback on failure. --- # Apply Modes -The caller specifies one of three modes via the `--mode` argument (or -interactively when not provided): - | Mode | Behaviour | |------|-----------| -| `report-only` | Re-display the Drift Report summary without making any changes. This is the **default** when no mode is specified. | -| `apply-safe-fixes` | Apply only fixes that meet **all** of these criteria: `confidence: high`, `severity: info` or `warning`, and `suggested_action` is NOT `move` for architecture/domain rules. Safe fixes include: removing references to deleted files, de-duplicating identical rules, fixing formatting issues, updating stale command names. | -| `apply-all-approved` | Apply every fix that the user has explicitly approved. Before applying, list all pending changes and require confirmation. | +| `report-only` | Default. Re-display the report summary without making changes. | +| `apply-safe-fixes` | Apply only high-confidence warning/info findings that include executable `edits` and stay inside safe-fix boundaries. | +| `apply-all-approved` | Apply every explicitly approved finding after preview. | --- # Pre-Apply Checks -Before modifying any file, perform these checks in order. If any check fails, -stop and report the failure without making changes. +Before modifying anything, run these checks in order: + +1. **Report existence** — require a report with a fenced `memorylint-report.json` + artifact or a raw JSON report. It must include `schema_version: "1.0"`, + `workspace_root`, `source_metadata`, `instruction_map`, and `findings`. +2. **Staleness check** — for every file that would be modified, compare the + current SHA-256 content hash with the hash from `source_metadata`. If any hash + differs, stop and report the stale file. +3. **Change preview** — list every finding id, file path, and affected line range + before applying. + +--- + +# Safe-Fix Boundaries + +`apply-safe-fixes` may only apply findings that meet **all** of these rules: + +- `confidence: high` +- `severity` is `info` or `warning` +- finding includes explicit `edits` +- `suggested_action` is not `move` +- the change does not rewrite constitution-owned content -1. **Report existence**: Verify that a MemoryLint Drift Report exists in the - current conversation context or was passed as an argument. The report must - include a fenced `memorylint-report.json` artifact with `schema_version: - "1.0"`, `source_metadata`, and `findings`. If not found, instruct the user - to run `speckit.memorylint.audit` first. +Safe examples: -2. **Staleness check**: For every instruction file that would be modified, - verify it has not changed since the report was generated. Compare the file's - current SHA-256 hash with the content hash recorded in - `memorylint-report.json` under `source_metadata`. If the hashes do not match, - report the staleness and refuse to apply changes to that file. +- deleting a stale reference to a missing script +- removing an exact duplicate rule from a secondary source +- rewriting an unambiguous stale command name -3. **Change preview**: List the exact set of changes that will be made: - - For each file: the finding ID, the lines affected, and the change - (addition, deletion, move, rewrite). - - Ask for explicit confirmation unless the caller passed `--yes`. +Unsafe in safe mode: + +- moving architecture or domain rules into the constitution +- deleting architecture/domain guidance +- semantic rewrites that change policy meaning +- any `confidence: medium` or `confidence: low` finding --- -# Execution +# Constitution Handoff Protocol -For each approved change: +When a finding targets `.specify/memory/constitution.md`: -1. Record the original content of every file that will be modified (for - rollback). -2. Apply the change. -3. After all changes are applied, proceed to Post-Apply Validation. +- treat the constitution as write-protected during apply +- never auto-merge into the constitution +- surface the finding's `manual_handoff` object as the handoff artifact +- require explicit human review for any constitution change -### Safe-Fix Boundaries +The handoff artifact must include: -When running in `apply-safe-fixes` mode, the following are **allowed**: +- `target_path` +- `target_section` +- `rule_text` +- `merge_rationale` +- `requires_human_review` + +--- -- Deleting a rule that references a file, script, or directory proven to not - exist (reality drift, confidence: high). -- Removing an exact duplicate rule from one file when the canonical copy - exists in another file (redundancy drift, confidence: high). -- Fixing formatting issues (whitespace, broken markdown links, list style). -- Updating a stale command or tool name when the new name is unambiguous. +# Execution -The following are **NOT allowed** in `apply-safe-fixes`: +For each approved finding: -- Moving architecture rules between AGENTS.md and constitution.md (boundary - drift) — this requires `apply-all-approved` with explicit confirmation. -- Rewriting rule semantics. -- Deleting rules classified as `domain` or `architecture`. -- Any change with `confidence: low` or `confidence: medium`. +1. Record the original content of every modified file. +2. Apply the exact `edits` from the report. +3. Continue only after every target file has been changed. +4. Run Post-Apply Validation. --- # Post-Apply Validation -After applying changes, run every check below. If **any** check fails, revert -**all** changes made during this apply run and report the failure. +If any validation fails, Rollback all changes from the current apply run. ### 1. AGENTS.md Integrity -- Verify `AGENTS.md` is valid Markdown (no broken headings, no orphaned list - items). -- Verify all critical sections still exist. At minimum, check for sections covering: +- Verify `AGENTS.md` still has valid heading / list structure. +- Verify critical sections still exist: - Build / Validation Commands - Git Workflow / Hygiene - Release Process / Workflow Rules ### 2. Constitution Integrity -- If `.specify/memory/constitution.md` exists, verify it has not lost any - architecture rules that were present before the apply run. -- Compare the rule count before and after. If rules decreased without a - corresponding `delete` finding, flag a validation failure. +- If `.specify/memory/constitution.md` exists, compare the before/after rule + count. +- If rules decreased without a corresponding explicit delete finding, fail. ### 3. Hook Consistency -- For every `extension.yml` in the workspace, verify that each hook `command:` - value still references a command declared under `provides.commands`. -- Report any broken hook references. +- For every `extension.yml`, verify every hook `command:` still references a + command declared under `provides.commands`. ### 4. Repository Validation Commands -- Run the validation commands listed in `AGENTS.md` (such as test, build, or - lint commands). If no specific commands are listed, run a default safety - check: - - `git diff --check` -- Report the results. +- Run the repo validation commands defined by the repository when they exist. +- If no narrower command is available, run at least `git diff --check`. ### 5. Change Summary -If all validations pass, output a summary: +On success, output: -``` +```text ## Apply Summary | Metric | Value | @@ -138,45 +148,40 @@ If all validations pass, output a summary: | Validations failed | | ### Changes Applied -- ML-001: [brief description of what was done] -- ML-003: [brief description of what was done] +- ML-001: [brief description] ``` --- -# Rollback Protocol +# Rollback -If any post-apply validation fails: +If any validation fails: -1. Restore every modified file to its pre-apply state using the recorded - original content. -2. Output a clear failure report: +1. Restore every modified file to its original content. +2. Output: -``` +```text ## Apply Failed — All Changes Reverted ### Validation Failures -- [Which check failed and why] +- [which check failed] ### Reverted Files -- [List of files restored to original state] +- [file path] ### Recommendation -- [What the user should do next — e.g., fix the underlying issue, re-run - audit, or apply changes manually] +- Fix the underlying issue, regenerate the report if needed, and retry. ``` +Rollback is atomic. Partial success is not allowed. + --- # Constraints -- **Default is safe**: when in doubt, do not apply. `report-only` is the - default mode. -- **No silent mutations**: every file change must be previewed and confirmed. -- **Rollback is atomic**: if any validation fails, ALL changes revert, not - just the failing one. -- **Respect the apply gate hierarchy**: `apply-safe-fixes` is strictly a - subset of `apply-all-approved`. Never apply unsafe fixes in safe mode. -- **Constitution is sacred**: never directly rewrite constitution content. - Boundary drift fixes that move rules INTO the constitution must present the - rule as handoff material for the user to merge manually. +- **Default is safe**: `report-only` unless the caller explicitly chooses a write mode. +- **No silent mutations**: every change must be previewable and attributable to a finding id. +- **Rollback is atomic**: if any validation fails, ALL changes revert. +- **Respect the apply gate hierarchy**: `apply-safe-fixes` is a strict subset of + `apply-all-approved`. +- **Constitution is sacred**: never directly rewrite constitution content during apply. diff --git a/memorylint/commands/audit.md b/memorylint/commands/audit.md index 6414b81..7c1f5e5 100644 --- a/memorylint/commands/audit.md +++ b/memorylint/commands/audit.md @@ -1,147 +1,193 @@ +--- +scripts: + - scripts/audit_workspace.py + - scripts/memorylint_core.py +--- $ARGUMENTS # Role -You are a rigorous Instruction Drift Auditor. Your task is to scan every -long-lived instruction source in the workspace, classify each rule, bind -findings to concrete evidence, and produce a structured **MemoryLint Drift -Report** that a human reviewer can act on. +You are a rigorous Instruction Drift Auditor. Scan the current workspace, +catalogue long-lived instruction rules, bind every finding to evidence, and +produce a deterministic **MemoryLint Drift Report**. **You MUST NOT modify any file during this audit.** This command is strictly -read-only. All recommendations go into the report; actual changes are applied -only through the separate `speckit.memorylint.apply` command. +read-only. All mutations happen only through `speckit.memorylint.apply`. # Objective -1. **Instruction Inventory** — discover every long-lived instruction source and - catalogue the rules each one contains. -2. **Rule Classification** — assign each rule to one of eight categories. -3. **Evidence Binding** — attach file-path or command-output proof to every - finding. Mark anything without evidence as `confidence: low`. -4. **Drift Detection** — detect boundary, reality, conflict, and redundancy - drift across all sources. -5. **Report Generation** — produce a structured Drift Report with an - Instruction Map, itemised Findings, Summary, Metrics, and Source Metadata - section, plus a machine-readable `memorylint-report.json` block. +1. **Instruction Inventory** — discover long-lived instruction sources and + catalogue every rule they contain. +2. **Rule Classification** — classify each rule into one primary category. +3. **Evidence Binding** — back every finding with file or command evidence. +4. **Drift Detection** — detect `boundary`, `reality`, `conflict`, and + `redundancy` drift. +5. **Report Generation** — emit a Markdown Drift Report plus a machine-readable + `memorylint-report.json` artifact. --- # Step 1 — Instruction Inventory -Scan the workspace for all instruction sources that exist. Include at least: +Scan the workspace root only. Include at least these source families when they +exist: | Source | Path Pattern | |--------|-------------| -| Agent rules | `AGENTS.md` | -| Constitution | `.specify/memory/constitution.md` | -| Claude rules | `CLAUDE.md` | -| Cursor rules | `.cursor/rules/*` | +| Agent rules | `AGENTS.md`, `**/AGENTS.md` | +| Constitution | `.specify/memory/constitution.md`, `**/.specify/memory/constitution.md` | +| Claude rules | `CLAUDE.md`, `**/CLAUDE.md` | +| Cursor rules | `.cursor/rules/*`, `**/.cursor/rules/*` | | Root README | `README.md` | -| Per-extension README | `*/README.md` | -| Workflow files | `.github/workflows/*.yml` | -| Test scripts | `tests/*`, `*/tests/*` | -| Extension manifests | `*/extension.yml` | +| Per-extension README | `**/README.md` | +| Workflow files | `.github/workflows/*.yml`, `**/.github/workflows/*.yml` | +| Test scripts | `tests/*`, `**/tests/*` | +| Extension manifests | `extension.yml`, `**/extension.yml` | -For every source that exists, extract individual rules and record: +For every source that exists, extract rules and record: -- **rule_id**: sequential `R-001`, `R-002`, etc. -- **source**: file path and line range -- **summary**: one-line plain-English description of the rule -- **category**: see Step 2 +- `rule_id`: sequential `R-001`, `R-002`, ... +- `source`: relative file path +- `line_range`: source line or line range +- `summary`: plain-English rule summary +- `category`: see Step 2 -If a source file does not exist, note its absence as a finding (it may be -expected or may indicate reality drift). +Missing optional files do not fail the audit. Record absence as evidence only +when it creates real drift. --- # Step 2 — Rule Classification -Classify every rule into exactly one category: +Classify every rule into exactly one primary category: | Category | Meaning | |----------|---------| -| `infrastructure` | CI, packaging, release mechanics, build/test commands | -| `architecture` | Directory layout, module boundaries, code conventions, design patterns | +| `infrastructure` | CI, release mechanics, build/test commands, packaging | +| `architecture` | Module boundaries, design patterns, structural code rules | | `workflow` | Git hygiene, review process, PR conventions, commit style | | `domain` | Product behaviour, Spec Kit hook semantics, extension contracts | -| `tooling` | CLI tools, language runtimes, editor config, env vars | +| `tooling` | CLI tools, runtimes, editor-specific tooling | | `personal_preference` | Style choices that do not affect correctness | | `obsolete` | References something that no longer exists in the repo | -| `conflict` | Contradicts another rule in the same or a different file | +| `conflict` | Contradicts another rule | + +If a rule could fit multiple categories, pick the primary owner and describe the +secondary concern in the finding detail. + +## Canonical Ownership / Precedence Matrix + +Use this matrix to decide canonical ownership and `recommended_destination`: -A rule may look like it belongs in two categories. Pick the primary category -and note the secondary concern in the finding. +| Category | Canonical Owner | Secondary / Contextual Sources | +|----------|-----------------|--------------------------------| +| `architecture` | `.specify/memory/constitution.md` | editor rules may restate, but do not own | +| `domain` | `.specify/memory/constitution.md` | manifests and docs may reflect, but do not own | +| `infrastructure` | root `AGENTS.md` | nested `AGENTS.md`, `CLAUDE.md`, workflows may scope or mirror | +| `workflow` | root `AGENTS.md` | nested `AGENTS.md`, `CLAUDE.md` may restate | +| `tooling` | root `AGENTS.md` | tool-specific editor files may add local context | +| `personal_preference` | root `AGENTS.md` | editor files may restate for agent ergonomics | + +Additional precedence rules: + +1. Constitution outranks editor-specific files for shared architecture/domain guidance. +2. Root `AGENTS.md` outranks nested/editor files for shared workflow/tooling/infrastructure guidance. +3. `README.md`, workflows, tests, and manifests are evidence-bearing sources, not canonical owners of shared guidance. --- # Step 3 — Evidence Binding -For every observation, supply **evidence**: - -- A file path (with line range if applicable) that proves or disproves the rule. - Example: "`package.json` exists and defines `scripts.test` → rule is supported." -- A directory listing when a rule claims a directory structure. - Example: "`ls tests/` shows no memorylint-specific tests → rule is unsupported." -- A command reference when a rule names a tool. - Example: "`which yq` → tool is / is not available." +Every finding must cite direct evidence: -**If you cannot find evidence**, mark the finding `confidence: low` and explain -what evidence you looked for but did not find. Never mark a finding -`confidence: high` without concrete proof. +- file path and line range +- missing-path check +- manifest / hook consistency proof +- command or script existence proof -### Confidence Levels +Confidence levels: | Level | Criteria | |-------|----------| | `high` | Direct file or command evidence confirms the finding | -| `medium` | Indirect evidence (e.g., pattern inference, partial match) | -| `low` | No concrete evidence found; based on heuristic judgement only | +| `medium` | Partial match or heuristic inference | +| `low` | No direct evidence; heuristic only | + +If evidence is missing, explicitly mark `confidence: low`. The report must +include the phrase `confidence: low` when a low-confidence finding appears. --- # Step 4 — Drift Detection -Detect and classify every drift instance: +Detect every drift instance and classify it: + +| Drift Type | Description | +|------------|-------------| +| `boundary` | Rule lives in the wrong canonical file | +| `reality` | Rule references a missing or stale file, script, command, or hook | +| `conflict` | Two rules are mutually exclusive | +| `redundancy` | Same rule appears in multiple places and risks divergence | + +For each finding determine: -| Drift Type | Description | Example | -|------------|-------------|---------| -| `boundary` | Rule lives in the wrong file | Architecture rule in AGENTS.md; workflow rule in constitution | -| `reality` | Rule references something that does not exist | Script, directory, command, or tool mentioned but absent | -| `conflict` | Two rules contradict each other | "Always use X" in AGENTS.md vs "Never use X" in constitution | -| `redundancy` | Same rule appears in multiple files | Identical or near-identical wording risks future divergence | +- `severity`: `critical`, `warning`, or `info` +- `confidence`: `high`, `medium`, or `low` +- `suggested_action`: `keep`, `move`, `delete`, `merge`, or `rewrite` +- `recommended_destination`: canonical owner path or `N/A` -For each drift instance, determine: +## Constitution Manual Handoff Rule -- **severity**: `critical` (blocks correctness or safety), `warning` (degrades - maintainability), or `info` (cosmetic or minor) -- **confidence**: `high`, `medium`, or `low` (see Step 3) -- **suggested_action**: one of `keep`, `move`, `delete`, `merge`, or `rewrite` -- **recommended_destination**: the file where the rule should live (for - boundary drift), or `N/A` +When a boundary finding says a rule belongs in the constitution: + +- DO NOT auto-rewrite `.specify/memory/constitution.md` +- emit a `manual_handoff` object that identifies: + - `target_path` + - `target_section` + - `rule_text` + - `merge_rationale` + - `requires_human_review` + +## Executable Output Contract + +When a finding can be safely rewritten or deleted, include an `edits` array with +precise file-level operations. Each edit includes: + +- `path` +- `action` +- `start_line` +- `end_line` +- optional `replacement` +- `reason` --- # Step 5 — Report Generation -Produce the Drift Report as Markdown with exactly these sections: `Instruction Map`, `Findings`, `Summary`, `Metrics`, `Source Metadata`, and `Machine-Readable Report`. +Produce the Drift Report as Markdown with exactly these sections: + +- `Instruction Map` +- `Findings` +- `Summary` +- `Metrics` +- `Source Metadata` +- `Machine-Readable Report` ## MemoryLint Drift Report ### Instruction Map -A table with columns: - | rule_id | source | line_range | summary | category | status | |---------|--------|------------|---------|----------|--------| -`status` is one of: `ok`, `boundary_drift`, `reality_drift`, `conflict`, -`redundant`, `obsolete`. +`status` is one of `ok`, `boundary_drift`, `reality_drift`, `conflict`, +`redundant`, or `obsolete`. ### Findings -For each finding, use this structure: +For each finding: -``` +```text #### ML-001 - **drift_type**: boundary | reality | conflict | redundancy - **severity**: critical | warning | info @@ -151,14 +197,11 @@ For each finding, use this structure: - **recommended_destination**: target file (or N/A) - **suggested_action**: keep | move | delete | merge | rewrite - **detail**: brief explanation of the problem and recommendation +- **manual_handoff**: {...} # only when constitution handoff is required ``` -Number findings sequentially: `ML-001`, `ML-002`, etc. - ### Summary -Provide totals: - | Drift Type | Critical | Warning | Info | Total | |------------|----------|---------|------|-------| | boundary | | | | | @@ -167,11 +210,9 @@ Provide totals: | redundancy | | | | | | **Total** | | | | | -Highlight any critical findings with a brief call-to-action. - ### Metrics -Report these trust metrics for the audit run: +Report these operational metrics: | Metric | Value | |--------|-------| @@ -185,21 +226,19 @@ Report these trust metrics for the audit run: ### Source Metadata -A table listing the content hashes of the scanned source files to enable staleness checks during apply: - | File Path | Content Hash (SHA-256) | |-----------|------------------------| ### Machine-Readable Report -Also include the same audit result as a fenced JSON artifact named -`memorylint-report.json`. This block is the authoritative input for -`speckit.memorylint.apply`; the Markdown report is for human review. +Also include the same result as a fenced JSON artifact named +`memorylint-report.json`. This artifact is the authoritative input for +`speckit.memorylint.apply`. ```memorylint-report.json { "schema_version": "1.0", - "workspace_root": "", + "workspace_root": "/path/to/workspace", "source_metadata": [ { "path": "AGENTS.md", @@ -210,7 +249,7 @@ Also include the same audit result as a fenced JSON artifact named { "rule_id": "R-001", "source": "AGENTS.md", - "line_range": "10-12", + "line_range": "10", "summary": "", "category": "infrastructure", "status": "ok" @@ -222,11 +261,18 @@ Also include the same audit result as a fenced JSON artifact named "drift_type": "boundary", "severity": "warning", "confidence": "high", - "source": "AGENTS.md:15-19", - "evidence": "", + "source": "AGENTS.md:15", + "evidence": "", "recommended_destination": ".specify/memory/constitution.md", "suggested_action": "move", - "detail": "" + "detail": "", + "manual_handoff": { + "target_path": ".specify/memory/constitution.md", + "target_section": "Imported rules", + "rule_text": "", + "merge_rationale": "", + "requires_human_review": true + } } ], "metrics": { @@ -237,7 +283,8 @@ Also include the same audit result as a fenced JSON artifact named "medium_confidence_findings": 0, "low_confidence_findings": 0, "files_that_would_be_modified": [] - } + }, + "summary": {} } ``` @@ -246,10 +293,8 @@ Also include the same audit result as a fenced JSON artifact named # Constraints - **Read-only**: do not create, modify, or delete any file. -- **Evidence-first**: every finding must cite evidence. No evidence → low - confidence. -- **Deterministic output**: follow the exact report structure above so the - `speckit.memorylint.apply` command can parse it. -- **Handle missing files gracefully**: if `.specify/memory/constitution.md` or - other optional sources do not exist, note this as a finding — do not error. -- **Scope to workspace root**: do not scan outside the current workspace. +- **Evidence-first**: every finding needs evidence; no evidence means + `confidence: low`. +- **Deterministic output**: preserve the report structure so + `speckit.memorylint.apply` can parse it. +- **Workspace-only scope**: do not scan outside the current workspace. diff --git a/memorylint/commands/load-agents.md b/memorylint/commands/load-agents.md index 131f8b5..b005761 100644 --- a/memorylint/commands/load-agents.md +++ b/memorylint/commands/load-agents.md @@ -1,20 +1,67 @@ +--- +scripts: + - scripts/load_agents_state.py + - scripts/memorylint_core.py +--- $ARGUMENTS # Role + You are the Core Rule Enforcer for the current workspace. # Objective -Your task is to load the `AGENTS.md` file from the workspace root directory into your context before the planning phase begins. This file contains the core infrastructure guidelines, system instructions, and safety protocols for AI agents. Your goal is to ensure that the upcoming planning (generation of `plan.md` and `tasks.md`) strictly adheres to these rules and prevents any drift from the established core constraints. + +Load the root `AGENTS.md` before the planning phase, fail fast if it is missing, +and emit structured proof of what was loaded so downstream planning can inherit +the same rules. # Action Instructions -1. **Load `AGENTS.md`**: Read the contents of the `AGENTS.md` file located at the root of the workspace. -2. **Mandatory Failure Rule**: If `AGENTS.md` is missing, unreadable, or cannot be loaded from the workspace root for any reason, STOP immediately. Do not begin planning, do not generate or update `plan.md` or `tasks.md`, and do not continue to any subsequent step. Output a clear error stating that the mandatory `before_plan` gate failed because `AGENTS.md` could not be loaded, along with remediation guidance to restore the file, fix its readability or permissions, or correct the workspace root before retrying. -3. **Acknowledge and Enforce**: Briefly acknowledge the core rules found in `AGENTS.md` and explicitly state that these rules will be strictly followed and enforced during the subsequent planning and implementation steps. -4. **Read-Only**: Do not modify `AGENTS.md` or any other files during this operation. This is strictly a context-loading action to guarantee rule adherence. -# Output Protocol -If `AGENTS.md` is loaded successfully, output a brief confirmation message indicating that `AGENTS.md` has been successfully loaded and that its rules will be enforced. For example: -`AGENTS.md` loaded successfully. Core rules and constraints have been established for the planning phase. +1. **Load `AGENTS.md`** from the workspace root. +2. **Mandatory Failure Rule**: if root `AGENTS.md` is missing, unreadable, or + cannot be loaded, STOP immediately. Do not begin planning, do not generate or + update `plan.md` or `tasks.md`, and do not continue to any subsequent step. +3. **Read-Only**: do not modify any file. +4. **Acknowledge and Enforce**: confirm that the loaded rules will govern the + planning phase. + +# Structured Output Protocol + +On success, output a short confirmation plus a machine-readable JSON payload with +at least: + +```json +{ + "workspace_root": "/path/to/workspace", + "agents_path": "AGENTS.md", + "agents_sha256": "", + "rule_count": 0, + "sections": [], + "rule_summaries": [ + { + "rule_id": "R-001", + "line_range": "10", + "category": "workflow", + "summary": "Use focused commits with Conventional Commit style." + } + ] +} +``` + +This output is the verifiable `before_plan` gate record. It must prove: + +- which file was loaded +- which content hash was enforced +- which sections / rules were imported into planning context + +# Failure Output + +If loading fails, output a clear failure and stop immediately: -If `AGENTS.md` cannot be loaded, output a clear failure message and stop. For example: `ERROR: Mandatory before_plan gate failed: could not load AGENTS.md from the workspace root. Planning cannot proceed. Remediation: ensure AGENTS.md exists, is readable, and that the workspace root is correct, then retry.` + +# Constraints + +- **Read-Only**: no file changes +- **Fail-fast**: do not continue to planning if the gate fails +- **Deterministic proof**: emit a stable, machine-readable record of the loaded rules diff --git a/memorylint/scripts/apply_report.py b/memorylint/scripts/apply_report.py new file mode 100644 index 0000000..c05ba68 --- /dev/null +++ b/memorylint/scripts/apply_report.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Apply MemoryLint findings from a machine-readable report.""" + +from __future__ import annotations + +import argparse +from collections import defaultdict +from pathlib import Path + +from memorylint_core import ( + approved_findings, + deep_copy_report, + extract_report_payload, + format_apply_failure, + format_apply_summary, + read_text, + validate_agents_integrity, + validate_constitution_integrity, + validate_hook_consistency, + apply_edits_to_lines, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Apply MemoryLint report findings.") + parser.add_argument("report", type=Path, help="Path to the Markdown or JSON audit report.") + parser.add_argument( + "--mode", + choices=("report-only", "apply-safe-fixes", "apply-all-approved"), + default="report-only", + help="Apply strategy.", + ) + parser.add_argument( + "--approve", + action="append", + default=[], + help="Finding id to approve when using apply-all-approved. Repeatable.", + ) + parser.add_argument( + "--handoff-out", + type=Path, + help="Optional path to write manual handoff findings as JSON.", + ) + return parser.parse_args() + + +def grouped_edits(findings: list[dict[str, object]]) -> dict[str, list[dict[str, object]]]: + grouped: dict[str, list[dict[str, object]]] = defaultdict(list) + for finding in findings: + for edit in finding.get("edits", []): + grouped[edit["path"]].append(edit) + return grouped + + +def main() -> int: + args = parse_args() + report_path = args.report.resolve() + report_payload = extract_report_payload(report_path.read_text(encoding="utf-8")) + report_copy = deep_copy_report(report_payload) + workspace_root = Path(report_copy["workspace_root"]).resolve() + + approved = approved_findings(report_copy, args.mode, set(args.approve)) + manual_handoffs = [finding for finding in report_copy.get("findings", []) if finding.get("manual_handoff")] + if args.handoff_out: + args.handoff_out.write_text( + __import__("json").dumps(manual_handoffs, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + + if args.mode == "report-only": + print(format_apply_summary([], [], []), end="") + return 0 + + source_hashes = {item["path"]: item["sha256"] for item in report_copy.get("source_metadata", [])} + target_paths = sorted({edit["path"] for finding in approved for edit in finding.get("edits", [])}) + for relative in target_paths: + target_path = workspace_root / relative + if not target_path.exists(): + print(format_apply_failure([f"Target file disappeared after audit: {relative}"], []), end="") + return 1 + current_hash = __import__("hashlib").sha256(target_path.read_bytes()).hexdigest() + if source_hashes.get(relative) != current_hash: + print(format_apply_failure([f"Staleness check failed for {relative}"], []), end="") + return 1 + + originals = {relative: read_text(workspace_root / relative) for relative in target_paths} + edits_by_file = grouped_edits(approved) + for relative, edits in edits_by_file.items(): + target_path = workspace_root / relative + updated = apply_edits_to_lines(read_text(target_path).splitlines(), edits) + target_path.write_text("\n".join(updated).rstrip() + "\n", encoding="utf-8") + + validation_issues: list[str] = [] + for relative, before_text in originals.items(): + after_text = read_text(workspace_root / relative) + if relative.endswith("AGENTS.md"): + validation_issues.extend(validate_agents_integrity(before_text, after_text)) + if relative.endswith(".specify/memory/constitution.md"): + finding_map = {finding["id"]: finding for finding in approved} + validation_issues.extend(validate_constitution_integrity(before_text, after_text, finding_map)) + validation_issues.extend(validate_hook_consistency(workspace_root)) + + if validation_issues: + for relative, before_text in originals.items(): + (workspace_root / relative).write_text(before_text, encoding="utf-8") + print(format_apply_failure(validation_issues, sorted(originals)), end="") + return 1 + + print(format_apply_summary(approved, sorted(originals), []), end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/memorylint/scripts/audit_workspace.py b/memorylint/scripts/audit_workspace.py new file mode 100644 index 0000000..68ca0ef --- /dev/null +++ b/memorylint/scripts/audit_workspace.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Run MemoryLint against a real workspace.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from memorylint_core import generate_report, markdown_report + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Audit a workspace for MemoryLint drift.") + parser.add_argument("workspace", nargs="?", default=".", help="Workspace root to audit.") + parser.add_argument( + "--format", + choices=("markdown", "json"), + default="markdown", + help="Output format written to stdout.", + ) + parser.add_argument( + "--json-out", + type=Path, + help="Optional path to also write the machine-readable JSON report.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + workspace_root = Path(args.workspace).resolve() + report = generate_report(workspace_root) + payload = report.to_dict() + + if args.json_out: + args.json_out.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + if args.format == "json": + print(json.dumps(payload, indent=2, ensure_ascii=False)) + else: + print(markdown_report(report), end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/memorylint/scripts/load_agents_state.py b/memorylint/scripts/load_agents_state.py new file mode 100644 index 0000000..b15d8e4 --- /dev/null +++ b/memorylint/scripts/load_agents_state.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Emit structured proof that AGENTS.md was loaded before planning.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from memorylint_core import extract_rules, relative_path, sha256 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Summarize the root AGENTS.md file.") + parser.add_argument("workspace", nargs="?", default=".", help="Workspace root.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + workspace_root = Path(args.workspace).resolve() + agents_path = workspace_root / "AGENTS.md" + if not agents_path.exists(): + raise SystemExit("AGENTS.md not found at workspace root") + + rules = [rule for rule in extract_rules(workspace_root, [agents_path]) if rule.source == "AGENTS.md"] + payload = { + "workspace_root": str(workspace_root), + "agents_path": relative_path(agents_path, workspace_root), + "agents_sha256": sha256(agents_path), + "rule_count": len(rules), + "sections": sorted({rule.heading for rule in rules if rule.heading}), + "rule_summaries": [ + { + "rule_id": rule.rule_id, + "line_range": rule.line_range, + "category": rule.category, + "summary": rule.summary, + } + for rule in rules + ], + } + print(json.dumps(payload, indent=2, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/memorylint/scripts/memorylint_core.py b/memorylint/scripts/memorylint_core.py new file mode 100644 index 0000000..a2da6ca --- /dev/null +++ b/memorylint/scripts/memorylint_core.py @@ -0,0 +1,1107 @@ +#!/usr/bin/env python3 +"""Shared MemoryLint audit/apply primitives.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import re +from collections import defaultdict +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Iterable + + +MARKDOWN_EXTENSIONS = {".md", ".markdown", ".txt"} +EDITOR_RULE_DIR = (".cursor", "rules") +CRITICAL_AGENTS_SECTION_FAMILIES = ( + ("build", "validation"), + ("git workflow", "workflow", "hygiene"), + ("release", "workflow rules"), +) +MEMORYLINT_EXPECTED_HOOKS = { + "before_constitution": "speckit.memorylint.audit", + "after_constitution": "speckit.memorylint.audit", + "before_plan": "speckit.memorylint.load-agents", +} +CANONICAL_OWNER_MATRIX = { + "architecture": ".specify/memory/constitution.md", + "domain": ".specify/memory/constitution.md", + "infrastructure": "AGENTS.md", + "workflow": "AGENTS.md", + "tooling": "AGENTS.md", + "personal_preference": "AGENTS.md", +} + + +@dataclass(frozen=True) +class Rule: + rule_id: str + source: str + line_range: str + heading: str + text: str + summary: str + category: str + status: str = "ok" + + +@dataclass(frozen=True) +class Edit: + path: str + action: str + start_line: int + end_line: int + replacement: list[str] = field(default_factory=list) + reason: str = "" + + +@dataclass +class Finding: + id: str + drift_type: str + severity: str + confidence: str + source: str + evidence: str + recommended_destination: str + suggested_action: str + detail: str + rule_ids: list[str] = field(default_factory=list) + category: str | None = None + edits: list[Edit] = field(default_factory=list) + manual_handoff: dict[str, object] | None = None + + def to_dict(self) -> dict[str, object]: + payload = { + "id": self.id, + "drift_type": self.drift_type, + "severity": self.severity, + "confidence": self.confidence, + "source": self.source, + "evidence": self.evidence, + "recommended_destination": self.recommended_destination, + "suggested_action": self.suggested_action, + "detail": self.detail, + } + if self.rule_ids: + payload["rule_ids"] = self.rule_ids + if self.category: + payload["category"] = self.category + if self.edits: + payload["edits"] = [asdict(edit) for edit in self.edits] + if self.manual_handoff: + payload["manual_handoff"] = self.manual_handoff + return payload + + +@dataclass +class AuditReport: + schema_version: str + workspace_root: str + source_metadata: list[dict[str, str]] + instruction_map: list[dict[str, str]] + findings: list[dict[str, object]] + metrics: dict[str, object] + summary: dict[str, dict[str, int]] + + def to_dict(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "workspace_root": self.workspace_root, + "source_metadata": self.source_metadata, + "instruction_map": self.instruction_map, + "findings": self.findings, + "metrics": self.metrics, + "summary": self.summary, + } + + +def read_text(path: Path) -> str: + return path.read_text(encoding="utf-8") if path.exists() else "" + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def relative_path(path: Path, workspace_root: Path) -> str: + return path.relative_to(workspace_root).as_posix() + + +def normalize_whitespace(text: str) -> str: + return re.sub(r"\s+", " ", text.strip()) + + +def normalize_rule_text(text: str) -> str: + text = text.strip() + text = re.sub(r"`([^`]+)`", r"\1", text) + text = re.sub(r"\[[^\]]+\]\([^)]+\)", "", text) + text = re.sub(r"[“”\"'`]", "", text) + text = re.sub(r"[^a-zA-Z0-9{}./:+\-\s]", " ", text) + return normalize_whitespace(text).lower() + + +def path_kind(path: Path) -> str: + parts = path.parts + posix = path.as_posix() + if path.name == "AGENTS.md": + return "agents" + if posix.endswith(".specify/memory/constitution.md"): + return "constitution" + if path.name == "CLAUDE.md": + return "claude" + if EDITOR_RULE_DIR[0] in parts and EDITOR_RULE_DIR[1] in parts: + return "cursor" + if path.name == "README.md": + return "readme" + if ".github" in parts and "workflows" in parts: + return "workflow" + if path.name == "extension.yml": + return "manifest" + if "tests" in parts: + return "test" + return "other" + + +def discover_sources(workspace_root: Path) -> list[Path]: + patterns = [ + "AGENTS.md", + "**/AGENTS.md", + ".specify/memory/constitution.md", + "**/.specify/memory/constitution.md", + "CLAUDE.md", + "**/CLAUDE.md", + ".cursor/rules/*", + "**/.cursor/rules/*", + "README.md", + "*/README.md", + "**/README.md", + ".github/workflows/*.yml", + "**/.github/workflows/*.yml", + "tests/*", + "**/tests/*", + "extension.yml", + "**/extension.yml", + ] + + discovered: set[Path] = set() + for pattern in patterns: + for candidate in workspace_root.glob(pattern): + if not candidate.is_file(): + continue + if ".git" in candidate.parts: + continue + discovered.add(candidate) + return sorted(discovered) + + +def markdown_rules(path: Path, workspace_root: Path, next_rule_id: int) -> tuple[list[Rule], int]: + rules: list[Rule] = [] + current_heading = "" + for line_number, line in enumerate(read_text(path).splitlines(), start=1): + heading_match = re.match(r"^\s{0,3}#{1,6}\s+(.+?)\s*$", line) + if heading_match: + current_heading = heading_match.group(1).strip() + continue + bullet_match = re.match(r"^\s*(?:[-*]|\d+\.)\s+(.+?)\s*$", line) + if not bullet_match: + continue + text = bullet_match.group(1).strip() + if not text: + continue + rule_id = f"R-{next_rule_id:03d}" + next_rule_id += 1 + summary = text + category = classify_rule(summary, path_kind(path), current_heading) + rules.append( + Rule( + rule_id=rule_id, + source=relative_path(path, workspace_root), + line_range=str(line_number), + heading=current_heading, + text=text, + summary=summary, + category=category, + ) + ) + return rules, next_rule_id + + +def parse_extension_commands(text: str) -> set[str]: + commands: set[str] = set() + in_commands = False + for line in text.splitlines(): + if re.match(r"^\s*commands:\s*$", line): + in_commands = True + continue + if in_commands and re.match(r"^\s*[a-z_]+:\s*$", line): + break + if in_commands: + command_match = re.match(r'^\s*-\s+name:\s*["\']?([^"\']+)["\']?\s*$', line) + if command_match: + commands.add(command_match.group(1)) + return commands + + +def parse_extension_hooks(text: str) -> dict[str, str]: + hooks: dict[str, str] = {} + hook_match = re.search(r"^\s*hooks:\s*$", text, re.MULTILINE) + if not hook_match: + return hooks + + current_hook: str | None = None + for line in text.splitlines(): + hook_name_match = re.match(r"^\s{2}([a-z_]+):\s*$", line) + if hook_name_match: + current_hook = hook_name_match.group(1) + continue + command_match = re.match(r'^\s{4}command:\s*["\']?([^"\']+)["\']?\s*$', line) + if current_hook and command_match: + hooks[current_hook] = command_match.group(1) + return hooks + + +def manifest_rules(path: Path, workspace_root: Path, next_rule_id: int) -> tuple[list[Rule], int]: + text = read_text(path) + rules: list[Rule] = [] + + extension_id_match = re.search(r'^\s*id:\s*["\']?([^"\']+)["\']?\s*$', text, re.MULTILINE) + extension_id = extension_id_match.group(1) if extension_id_match else "" + + for hook_name, command_name in parse_extension_hooks(text).items(): + line_number = find_line_number(text, f"command: \"{command_name}\"") or find_line_number(text, f"command: {command_name}") or 1 + rule_id = f"R-{next_rule_id:03d}" + next_rule_id += 1 + summary = f"Hook {hook_name} uses command {command_name}" + category = "domain" if extension_id else "infrastructure" + rules.append( + Rule( + rule_id=rule_id, + source=relative_path(path, workspace_root), + line_range=str(line_number), + heading="hooks", + text=summary, + summary=summary, + category=category, + ) + ) + return rules, next_rule_id + + +def extract_rules(workspace_root: Path, sources: list[Path]) -> list[Rule]: + rules: list[Rule] = [] + next_rule_id = 1 + for path in sources: + kind = path_kind(path) + if path.suffix in MARKDOWN_EXTENSIONS or kind in {"agents", "constitution", "claude", "cursor", "readme"}: + extracted, next_rule_id = markdown_rules(path, workspace_root, next_rule_id) + rules.extend(extracted) + continue + if kind == "manifest": + extracted, next_rule_id = manifest_rules(path, workspace_root, next_rule_id) + rules.extend(extracted) + return rules + + +def classify_rule(summary: str, source_kind: str, heading: str) -> str: + normalized = normalize_rule_text(f"{heading} {summary}") + if any( + keyword in normalized + for keyword in ( + "architecture", + "hexagonal", + "mvc", + "redux", + "module", + "boundary", + "repository pattern", + "rest principles", + "json:api", + "composition over inheritance", + "jsdoc", + "command pattern", + "business logic", + "type annotations", + "type annotation", + "error handling", + ) + ): + return "architecture" + if any(keyword in normalized for keyword in ("prefer", "terse", "concise", "style choice", "single quotes", "functional components")): + return "personal_preference" + if any(keyword in normalized for keyword in ("conventional commit", "force-push", "force push", "review approval", "git workflow", "pull request", "prs require")): + return "workflow" + if any(keyword in normalized for keyword in ("ci", "build", "test", "lint", "release", "deploy", "workflow", "tag", "package", "validation")): + return "infrastructure" + if any(keyword in normalized for keyword in ("speckit.", "constitution", "command naming", "extension", "hook", "command names must follow")): + return "domain" + if any(keyword in normalized for keyword in ("python", "ruby", "nvm", "pyenv", "rbenv", "cursor", "claude", "editor", "tool")): + return "tooling" + if source_kind == "manifest": + return "domain" + return "infrastructure" + + +def canonical_destination(rule: Rule) -> str | None: + return CANONICAL_OWNER_MATRIX.get(rule.category) + + +def source_metadata(workspace_root: Path, sources: list[Path]) -> list[dict[str, str]]: + return [ + { + "path": relative_path(path, workspace_root), + "sha256": sha256(path), + } + for path in sources + ] + + +def missing_constitution_finding(workspace_root: Path, rules: list[Rule]) -> Finding | None: + constitution_path = workspace_root / ".specify/memory/constitution.md" + if constitution_path.exists(): + return None + architecture_like = [rule for rule in rules if rule.category in {"architecture", "domain"}] + if not architecture_like: + return None + return Finding( + id="", + drift_type="reality", + severity="info", + confidence="medium", + source=".specify/memory/constitution.md", + evidence="The workspace does not contain .specify/memory/constitution.md, so architecture/domain rules do not have their canonical home.", + recommended_destination=".specify/memory/constitution.md", + suggested_action="keep", + detail="The canonical constitution file is missing. Create or restore it before consolidating architecture/domain rules.", + category="architecture", + ) + + +def make_boundary_findings(workspace_root: Path, rules: list[Rule]) -> list[Finding]: + findings: list[Finding] = [] + for rule in rules: + destination = canonical_destination(rule) + if not destination: + continue + source_kind_name = path_kind(Path(rule.source)) + if destination == ".specify/memory/constitution.md": + allowed = source_kind_name == "constitution" + elif destination == "AGENTS.md": + allowed = source_kind_name in {"agents", "claude", "cursor"} + else: + allowed = False + if allowed: + continue + if source_kind_name in {"readme", "manifest", "workflow", "test"}: + continue + if source_kind_name in {"claude", "cursor"} and rule.category in {"infrastructure", "workflow", "tooling", "personal_preference"}: + continue + if source_kind_name == "agents" and "/" in rule.source and rule.category in {"infrastructure", "workflow", "tooling", "personal_preference"}: + continue + detail = f"Rule '{rule.summary}' belongs in {destination}, not in {rule.source}." + handoff = None + edits: list[Edit] = [] + if destination == ".specify/memory/constitution.md": + handoff = { + "target_path": destination, + "target_section": rule.heading or "Imported rules", + "rule_text": rule.text, + "merge_rationale": "Move architecture/domain guidance into the canonical constitution without auto-rewriting the constitution.", + "requires_human_review": True, + } + edits.append( + Edit( + path=rule.source, + action="delete", + start_line=int(rule.line_range.split("-")[0]), + end_line=int(rule.line_range.split("-")[-1]), + reason="Remove misplaced rule from non-canonical source after manual constitution handoff.", + ) + ) + findings.append( + Finding( + id="", + drift_type="boundary", + severity="warning", + confidence="high", + source=f"{rule.source}:{rule.line_range}", + evidence=f"{rule.source} contains a {rule.category} rule under heading '{rule.heading or 'Unsectioned'}'.", + recommended_destination=destination, + suggested_action="move", + detail=detail, + rule_ids=[rule.rule_id], + category=rule.category, + edits=edits, + manual_handoff=handoff, + ) + ) + return findings + + +def find_line_number(text: str, snippet: str) -> int | None: + for index, line in enumerate(text.splitlines(), start=1): + if snippet in line: + return index + return None + + +def detect_path_references(rule: Rule) -> Iterable[str]: + for match in re.findall(r"`([^`]+)`", rule.text): + if "://" in match or "{" in match: + continue + if match.startswith(("scripts/", "bin/", "tools/")) or match.endswith((".sh", ".py", ".rb", ".js", ".ts")): + yield match + + +def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Finding]: + findings: list[Finding] = [] + seen_sources: set[tuple[str, str]] = set() + + for rule in rules: + rule_path = workspace_root / rule.source + rule_text_normalized = normalize_rule_text(rule.text) + line_number = int(rule.line_range.split("-")[0]) + + for reference in detect_path_references(rule): + candidate = (workspace_root / reference).resolve() + if not candidate.exists(): + key = (rule.source, reference) + if key in seen_sources: + continue + seen_sources.add(key) + findings.append( + Finding( + id="", + drift_type="reality", + severity="warning", + confidence="high", + source=f"{rule.source}:{rule.line_range}", + evidence=f"{rule.source} references `{reference}` but that path does not exist in the workspace.", + recommended_destination="N/A", + suggested_action="delete", + detail=f"Remove or replace the stale reference to `{reference}`.", + rule_ids=[rule.rule_id], + category=rule.category, + edits=[ + Edit( + path=rule.source, + action="delete", + start_line=line_number, + end_line=line_number, + reason=f"Delete stale reference to missing path {reference}.", + ) + ], + ) + ) + + npm_match = re.search(r"npm run ([a-zA-Z0-9:_-]+)", rule.text) + if npm_match: + script_name = npm_match.group(1) + package_json = nearest_package_json(rule_path, workspace_root) + package_data = json.loads(package_json.read_text(encoding="utf-8")) if package_json and package_json.exists() else {} + scripts = package_data.get("scripts", {}) if isinstance(package_data, dict) else {} + if script_name not in scripts: + findings.append( + Finding( + id="", + drift_type="reality", + severity="warning", + confidence="high", + source=f"{rule.source}:{rule.line_range}", + evidence=f"{rule.source} references `npm run {script_name}` but the workspace has no package.json script named `{script_name}`.", + recommended_destination="N/A", + suggested_action="delete", + detail=f"Remove or update the stale npm script reference `{script_name}`.", + rule_ids=[rule.rule_id], + category=rule.category, + edits=[ + Edit( + path=rule.source, + action="delete", + start_line=line_number, + end_line=line_number, + reason=f"Delete stale npm script reference {script_name}.", + ) + ], + ) + ) + + if "speckit.memorylint.run" in rule_text_normalized and path_kind(rule_path) != "manifest": + replacement = "speckit.memorylint.load-agents" if "planning phase" in rule_text_normalized or "before plan" in rule_text_normalized else "speckit.memorylint.audit" + findings.append( + Finding( + id="", + drift_type="reality", + severity="warning", + confidence="high", + source=f"{rule.source}:{rule.line_range}", + evidence=f"{rule.source} still references removed command `speckit.memorylint.run`.", + recommended_destination=rule.source, + suggested_action="rewrite", + detail=f"Rewrite the stale command reference to `{replacement}`.", + rule_ids=[rule.rule_id], + category=rule.category, + edits=[ + Edit( + path=rule.source, + action="replace", + start_line=line_number, + end_line=line_number, + replacement=[read_text(rule_path).splitlines()[line_number - 1].replace("speckit.memorylint.run", replacement)], + reason=f"Replace removed command name with {replacement}.", + ) + ], + ) + ) + + for manifest in [rule for rule in rules if path_kind(Path(rule.source)) == "manifest"]: + manifest_path = workspace_root / manifest.source + manifest_text = read_text(manifest_path) + extension_id_match = re.search(r'^\s*id:\s*["\']?([^"\']+)["\']?\s*$', manifest_text, re.MULTILINE) + extension_id = extension_id_match.group(1) if extension_id_match else "" + declared = parse_extension_commands(manifest_text) + for hook_name, command_name in parse_extension_hooks(manifest_text).items(): + if command_name in declared: + continue + replacement = MEMORYLINT_EXPECTED_HOOKS.get(hook_name, command_name) if extension_id == "memorylint" else command_name + line_number = find_line_number(manifest_text, f'command: "{command_name}"') or find_line_number(manifest_text, f"command: {command_name}") or 1 + findings.append( + Finding( + id="", + drift_type="reality", + severity="critical", + confidence="high", + source=f"{manifest.source}:{line_number}", + evidence=f"{manifest.source} hook `{hook_name}` references `{command_name}`, but provides.commands declares {sorted(declared)}.", + recommended_destination=manifest.source, + suggested_action="rewrite", + detail=f"Rewrite hook `{hook_name}` to use a declared command.", + rule_ids=[manifest.rule_id], + category="domain", + edits=[ + Edit( + path=manifest.source, + action="replace", + start_line=line_number, + end_line=line_number, + replacement=[re.sub(r'command:\s*["\']?[^"\']+["\']?', f'command: "{replacement}"', manifest_text.splitlines()[line_number - 1])], + reason=f"Rewrite hook {hook_name} to the declared command {replacement}.", + ) + ], + ) + ) + + missing_constitution = missing_constitution_finding(workspace_root, rules) + if missing_constitution: + findings.append(missing_constitution) + return findings + + +def nearest_package_json(rule_path: Path, workspace_root: Path) -> Path | None: + for candidate_dir in [rule_path.parent, *rule_path.parents]: + if candidate_dir == workspace_root.parent: + break + package_json = candidate_dir / "package.json" + if package_json.exists(): + return package_json + if candidate_dir == workspace_root: + break + return None + + +def commit_policy_signature(rule: Rule) -> str | None: + normalized = normalize_rule_text(rule.text) + if "conventional commit" in normalized: + return "conventional-commits" + if "[jira-id]" in normalized or "never use type prefixes" in normalized: + return "jira-commits" + return None + + +def detect_conflicts(rules: list[Rule]) -> list[Finding]: + findings: list[Finding] = [] + grouped: defaultdict[str, list[Rule]] = defaultdict(list) + for rule in rules: + signature = commit_policy_signature(rule) + if signature: + grouped[signature].append(rule) + + if grouped["conventional-commits"] and grouped["jira-commits"]: + lhs = grouped["conventional-commits"][0] + rhs = grouped["jira-commits"][0] + findings.append( + Finding( + id="", + drift_type="conflict", + severity="critical", + confidence="high", + source=f"{lhs.source}:{lhs.line_range} + {rhs.source}:{rhs.line_range}", + evidence=f"{lhs.source} requires Conventional Commits while {rhs.source} forbids type prefixes and requires a JIRA format.", + recommended_destination="AGENTS.md", + suggested_action="rewrite", + detail="The commit message policies are mutually exclusive and must be reconciled in one canonical source.", + rule_ids=[lhs.rule_id, rhs.rule_id], + category="workflow", + ) + ) + return findings + + +def redundancy_signature(rule: Rule) -> str | None: + normalized = normalize_rule_text(rule.text) + if "speckit.{extension-id}.{command-name}" in normalized: + return "command-naming-pattern" + if "make build" in normalized: + return "make-build" + if "make test" in normalized: + return "make-test" + if "use focused commits" in normalized: + return "focused-commits" + if "do not force-push to main" in normalized or "do not force push to main" in normalized: + return "no-force-push-main" + return None + + +def root_agents_rule(rules: list[Rule], signature: str) -> Rule | None: + for rule in rules: + if rule.source == "AGENTS.md" and redundancy_signature(rule) == signature: + return rule + return None + + +def detect_redundancies(rules: list[Rule]) -> list[Finding]: + findings: list[Finding] = [] + by_signature: defaultdict[str, list[Rule]] = defaultdict(list) + for rule in rules: + signature = redundancy_signature(rule) + if signature: + by_signature[signature].append(rule) + + command_rules = by_signature.get("command-naming-pattern", []) + root_command = root_agents_rule(command_rules, "command-naming-pattern") + constitution_command = next((rule for rule in command_rules if path_kind(Path(rule.source)) == "constitution"), None) + if root_command and constitution_command: + findings.append( + Finding( + id="", + drift_type="redundancy", + severity="info", + confidence="high", + source=f"{root_command.source}:{root_command.line_range} + {constitution_command.source}:{constitution_command.line_range}", + evidence="Both sources contain the same speckit command naming pattern.", + recommended_destination=".specify/memory/constitution.md", + suggested_action="merge", + detail="Keep one canonical copy of the command naming rule to avoid future divergence.", + rule_ids=[root_command.rule_id, constitution_command.rule_id], + category="domain", + edits=[ + Edit( + path=root_command.source, + action="delete", + start_line=int(root_command.line_range), + end_line=int(root_command.line_range), + reason="Remove duplicate rule from non-canonical source.", + ) + ], + ) + ) + + root_build = root_agents_rule(rules, "make-build") + root_test = root_agents_rule(rules, "make-test") + claude_build = next((rule for rule in by_signature.get("make-build", []) if path_kind(Path(rule.source)) == "claude"), None) + claude_test = next((rule for rule in by_signature.get("make-test", []) if path_kind(Path(rule.source)) == "claude"), None) + if root_build and root_test and claude_build and claude_test: + findings.append( + Finding( + id="", + drift_type="redundancy", + severity="warning", + confidence="high", + source=f"{root_build.source}:{root_build.line_range} + {claude_build.source}:{claude_build.line_range}", + evidence="AGENTS.md and CLAUDE.md both list make build and make test.", + recommended_destination="AGENTS.md", + suggested_action="merge", + detail="Build command rules are duplicated across AGENTS.md and CLAUDE.md, risking divergence.", + rule_ids=[root_build.rule_id, root_test.rule_id, claude_build.rule_id, claude_test.rule_id], + category="infrastructure", + edits=[ + Edit( + path=claude_build.source, + action="delete", + start_line=int(claude_build.line_range), + end_line=int(claude_test.line_range), + reason="Remove duplicate build/test rules from secondary source.", + ) + ], + ) + ) + + root_focused = root_agents_rule(rules, "focused-commits") + nested_focused = next( + ( + rule + for rule in by_signature.get("focused-commits", []) + if path_kind(Path(rule.source)) == "agents" and rule.source != "AGENTS.md" + ), + None, + ) + if root_focused and nested_focused: + findings.append( + Finding( + id="", + drift_type="redundancy", + severity="warning", + confidence="medium", + source=f"{root_focused.source}:{root_focused.line_range} + {nested_focused.source}:{nested_focused.line_range}", + evidence="Root and nested AGENTS.md repeat the focused commits policy.", + recommended_destination="AGENTS.md", + suggested_action="merge", + detail="Nested workflow rules duplicate the root policy and risk divergence.", + rule_ids=[root_focused.rule_id, nested_focused.rule_id], + category="workflow", + edits=[ + Edit( + path=nested_focused.source, + action="delete", + start_line=int(nested_focused.line_range), + end_line=int(nested_focused.line_range), + reason="Remove duplicate workflow rule from nested AGENTS.md.", + ) + ], + ) + ) + return findings + + +def assign_finding_ids(findings: list[Finding]) -> list[Finding]: + assigned: list[Finding] = [] + for index, finding in enumerate(findings, start=1): + finding.id = f"ML-{index:03d}" + assigned.append(finding) + return assigned + + +def apply_instruction_status(rules: list[Rule], findings: list[Finding]) -> list[dict[str, str]]: + rule_status: dict[str, str] = {rule.rule_id: "ok" for rule in rules} + for finding in findings: + status = { + "boundary": "boundary_drift", + "reality": "reality_drift", + "conflict": "conflict", + "redundancy": "redundant", + }.get(finding.drift_type, "ok") + for rule_id in finding.rule_ids: + rule_status[rule_id] = status + + return [ + { + "rule_id": rule.rule_id, + "source": rule.source, + "line_range": rule.line_range, + "summary": rule.summary, + "category": rule.category, + "status": rule_status[rule.rule_id], + } + for rule in rules + ] + + +def summarize_findings(findings: list[Finding]) -> dict[str, dict[str, int]]: + summary: dict[str, dict[str, int]] = { + drift: {"critical": 0, "warning": 0, "info": 0, "total": 0} + for drift in ("boundary", "reality", "conflict", "redundancy") + } + summary["total"] = {"critical": 0, "warning": 0, "info": 0, "total": 0} + for finding in findings: + bucket = summary[finding.drift_type] + bucket[finding.severity] += 1 + bucket["total"] += 1 + summary["total"][finding.severity] += 1 + summary["total"]["total"] += 1 + return summary + + +def metrics_from_findings(sources: list[Path], rules: list[Rule], findings: list[Finding]) -> dict[str, object]: + files_that_would_be_modified = sorted( + { + edit.path + for finding in findings + for edit in finding.edits + if edit.path and finding.suggested_action in {"delete", "merge", "rewrite", "move"} + } + ) + return { + "total_instruction_sources_scanned": len(sources), + "total_rules_catalogued": len(rules), + "total_findings": len(findings), + "high_confidence_findings": sum(1 for finding in findings if finding.confidence == "high"), + "medium_confidence_findings": sum(1 for finding in findings if finding.confidence == "medium"), + "low_confidence_findings": sum(1 for finding in findings if finding.confidence == "low"), + "files_that_would_be_modified": files_that_would_be_modified, + } + + +def generate_report(workspace_root: Path) -> AuditReport: + sources = discover_sources(workspace_root) + rules = extract_rules(workspace_root, sources) + findings = assign_finding_ids( + make_boundary_findings(workspace_root, rules) + + detect_reality_findings(workspace_root, rules) + + detect_conflicts(rules) + + detect_redundancies(rules) + ) + instruction_map = apply_instruction_status(rules, findings) + summary = summarize_findings(findings) + metrics = metrics_from_findings(sources, rules, findings) + return AuditReport( + schema_version="1.0", + workspace_root=str(workspace_root), + source_metadata=source_metadata(workspace_root, sources), + instruction_map=instruction_map, + findings=[finding.to_dict() for finding in findings], + metrics=metrics, + summary=summary, + ) + + +def markdown_report(report: AuditReport) -> str: + payload = report.to_dict() + lines: list[str] = [ + "## MemoryLint Drift Report", + "", + "### Instruction Map", + "", + "| rule_id | source | line_range | summary | category | status |", + "|---------|--------|------------|---------|----------|--------|", + ] + for item in report.instruction_map: + lines.append( + f"| {item['rule_id']} | {item['source']} | {item['line_range']} | {item['summary']} | {item['category']} | {item['status']} |" + ) + + lines.extend(["", "### Findings", ""]) + if not report.findings: + lines.append("No findings.") + for finding in report.findings: + lines.extend( + [ + f"#### {finding['id']}", + f"- **drift_type**: {finding['drift_type']}", + f"- **severity**: {finding['severity']}", + f"- **confidence**: {finding['confidence']}", + f"- **source**: {finding['source']}", + f"- **evidence**: {finding['evidence']}", + f"- **recommended_destination**: {finding['recommended_destination']}", + f"- **suggested_action**: {finding['suggested_action']}", + f"- **detail**: {finding['detail']}", + ] + ) + if "manual_handoff" in finding: + handoff = finding["manual_handoff"] + lines.append(f"- **manual_handoff**: {json.dumps(handoff, ensure_ascii=False)}") + lines.append("") + + lines.extend( + [ + "### Summary", + "", + "| Drift Type | Critical | Warning | Info | Total |", + "|------------|----------|---------|------|-------|", + ] + ) + for drift in ("boundary", "reality", "conflict", "redundancy", "total"): + bucket = report.summary[drift] + name = "**Total**" if drift == "total" else drift + lines.append( + f"| {name} | {bucket['critical']} | {bucket['warning']} | {bucket['info']} | {bucket['total']} |" + ) + + lines.extend( + [ + "", + "### Metrics", + "", + "| Metric | Value |", + "|--------|-------|", + f"| Total instruction sources scanned | {report.metrics['total_instruction_sources_scanned']} |", + f"| Total rules catalogued | {report.metrics['total_rules_catalogued']} |", + f"| Total findings | {report.metrics['total_findings']} |", + f"| High-confidence findings | {report.metrics['high_confidence_findings']} |", + f"| Medium-confidence findings | {report.metrics['medium_confidence_findings']} |", + f"| Low-confidence findings | {report.metrics['low_confidence_findings']} |", + f"| Files that would be modified by suggested actions | {', '.join(report.metrics['files_that_would_be_modified']) or 'None'} |", + "", + "### Source Metadata", + "", + "| File Path | Content Hash (SHA-256) |", + "|-----------|------------------------|", + ] + ) + for item in report.source_metadata: + lines.append(f"| {item['path']} | {item['sha256']} |") + + lines.extend( + [ + "", + "### Machine-Readable Report", + "", + "```memorylint-report.json", + json.dumps(payload, indent=2, ensure_ascii=False), + "```", + ] + ) + return "\n".join(lines) + "\n" + + +def extract_report_payload(text: str) -> dict[str, object]: + stripped = text.strip() + if stripped.startswith("{"): + return json.loads(stripped) + match = re.search(r"```memorylint-report\.json\n(.*?)\n```", text, re.DOTALL) + if not match: + raise ValueError("memorylint-report.json artifact not found") + return json.loads(match.group(1)) + + +def safe_mode_eligible(finding: dict[str, object]) -> bool: + if finding["confidence"] != "high": + return False + if finding["severity"] not in {"info", "warning"}: + return False + if finding["suggested_action"] == "move": + return False + if finding.get("category") in {"architecture", "domain"} and finding["suggested_action"] != "rewrite": + return False + return bool(finding.get("edits")) + + +def approved_findings(report_payload: dict[str, object], mode: str, approved_ids: set[str] | None = None) -> list[dict[str, object]]: + findings = list(report_payload.get("findings", [])) + if mode == "report-only": + return [] + if mode == "apply-safe-fixes": + return [finding for finding in findings if safe_mode_eligible(finding)] + approved_ids = approved_ids or set() + return [finding for finding in findings if finding["id"] in approved_ids or safe_mode_eligible(finding)] + + +def apply_edits_to_lines(lines: list[str], edits: list[dict[str, object]]) -> list[str]: + updated = list(lines) + for edit in sorted(edits, key=lambda item: (item["start_line"], item["end_line"]), reverse=True): + start = int(edit["start_line"]) - 1 + end = int(edit["end_line"]) + replacement = list(edit.get("replacement", [])) + updated[start:end] = replacement + return updated + + +def agents_headings(lines: list[str]) -> list[str]: + headings = [] + for line in lines: + match = re.match(r"^\s{0,3}##\s+(.+?)\s*$", line) + if match: + headings.append(match.group(1).strip()) + return headings + + +def validate_agents_integrity(before_text: str, after_text: str) -> list[str]: + issues: list[str] = [] + before_headings = agents_headings(before_text.splitlines()) + after_headings = agents_headings(after_text.splitlines()) + before_normalized = [heading.lower() for heading in before_headings] + after_normalized = [heading.lower() for heading in after_headings] + for family in CRITICAL_AGENTS_SECTION_FAMILIES: + if any(keyword in heading for heading in before_normalized for keyword in family): + if not any(keyword in heading for heading in after_normalized for keyword in family): + issues.append(f"Missing AGENTS.md critical section family after apply: {family[0]}") + for index, line in enumerate(after_text.splitlines(), start=1): + if re.match(r"^\s*[-*]\s+", line): + has_heading_above = any( + re.match(r"^\s{0,3}##\s+", earlier) + for earlier in after_text.splitlines()[: index - 1] + ) + if not has_heading_above: + issues.append(f"Found orphaned list item in AGENTS.md at line {index}") + break + return issues + + +def count_constitution_rules(text: str) -> int: + return sum(1 for line in text.splitlines() if re.match(r"^\s*(?:[-*]|\d+\.)\s+", line)) + + +def validate_constitution_integrity(before_text: str, after_text: str, finding_map: dict[str, dict[str, object]]) -> list[str]: + issues: list[str] = [] + if not before_text: + return issues + deleted_constitution = any( + edit["path"] == ".specify/memory/constitution.md" and edit["action"] == "delete" + for finding in finding_map.values() + for edit in finding.get("edits", []) + ) + if count_constitution_rules(after_text) < count_constitution_rules(before_text) and not deleted_constitution: + issues.append("Constitution rule count decreased without an explicit delete finding.") + return issues + + +def validate_hook_consistency(workspace_root: Path) -> list[str]: + issues: list[str] = [] + for manifest_path in discover_sources(workspace_root): + if path_kind(manifest_path) != "manifest": + continue + text = read_text(manifest_path) + declared = parse_extension_commands(text) + for hook_name, command_name in parse_extension_hooks(text).items(): + if command_name not in declared: + issues.append( + f"{relative_path(manifest_path, workspace_root)} hook `{hook_name}` references undeclared command `{command_name}`" + ) + return issues + + +def format_apply_summary(applied: list[dict[str, object]], files_modified: list[str], validation_issues: list[str]) -> str: + lines = [ + "## Apply Summary", + "", + "| Metric | Value |", + "|--------|-------|", + f"| Findings applied | {len(applied)} |", + f"| Files modified | {len(files_modified)} |", + f"| Lines changed | {sum(sum(edit['end_line'] - edit['start_line'] + 1 for edit in finding.get('edits', [])) for finding in applied)} |", + f"| Validations passed | {0 if validation_issues else 4} |", + f"| Validations failed | {len(validation_issues)} |", + "", + "### Changes Applied", + ] + if not applied: + lines.append("- None") + for finding in applied: + lines.append(f"- {finding['id']}: {finding['detail']}") + return "\n".join(lines) + "\n" + + +def format_apply_failure(validation_issues: list[str], reverted_files: list[str]) -> str: + lines = [ + "## Apply Failed — All Changes Reverted", + "", + "### Validation Failures", + ] + for issue in validation_issues: + lines.append(f"- {issue}") + lines.extend(["", "### Reverted Files"]) + for file_path in reverted_files: + lines.append(f"- {file_path}") + lines.extend( + [ + "", + "### Recommendation", + "- Fix the underlying issue, regenerate the audit report if needed, and retry the apply run.", + ] + ) + return "\n".join(lines) + "\n" + + +def deep_copy_report(payload: dict[str, object]) -> dict[str, object]: + return copy.deepcopy(payload) diff --git a/memorylint/scripts/scan_fixtures.py b/memorylint/scripts/scan_fixtures.py old mode 100755 new mode 100644 index 58a540f..25e8143 --- a/memorylint/scripts/scan_fixtures.py +++ b/memorylint/scripts/scan_fixtures.py @@ -1,395 +1,77 @@ #!/usr/bin/env python3 -"""Deterministic MemoryLint fixture scanner. - -This script intentionally covers the regression fixture corpus, not the full -interactive audit command. It turns the design contract into executable evidence -by proving the bundled fixtures produce the expected drift findings. -""" +"""Run MemoryLint's executable audit core against the regression fixture corpus.""" from __future__ import annotations import argparse -import hashlib import json -import re -import sys -from dataclasses import dataclass from pathlib import Path -from typing import Iterable - - -@dataclass(frozen=True) -class Finding: - drift_type: str - severity: str - confidence: str - description: str - source: str - evidence: str - recommended_destination: str - suggested_action: str - - def expected_dict(self) -> dict[str, str]: - return { - "drift_type": self.drift_type, - "severity": self.severity, - "confidence": self.confidence, - "description": self.description, - "source": self.source, - "evidence": self.evidence, - "recommended_destination": self.recommended_destination, - "suggested_action": self.suggested_action, - } - - def as_dict(self) -> dict[str, str]: - return { - "id": "", - "drift_type": self.drift_type, - "severity": self.severity, - "confidence": self.confidence, - "description": self.description, - "source": self.source, - "evidence": self.evidence, - "recommended_destination": self.recommended_destination, - "suggested_action": self.suggested_action, - } - - -def read_text(path: Path) -> str: - if not path.exists(): - return "" - return path.read_text(encoding="utf-8") - - -def sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def fixture_dirs(fixtures_dir: Path) -> Iterable[Path]: - for child in sorted(fixtures_dir.iterdir()): - if child.is_dir() and not child.name.startswith((".", "_")): - yield child - - -def source_files(fixture: Path) -> list[Path]: - return sorted( - path - for path in fixture.rglob("*") - if path.is_file() and path.name != "expected-findings.json" - ) - - -def rel(path: Path, root: Path) -> str: - return path.relative_to(root).as_posix() - - -def has_constitution(fixture: Path) -> bool: - return (fixture / ".specify/memory/constitution.md").exists() - - -def scan_boundary_drift(fixture: Path) -> list[Finding]: - findings: list[Finding] = [] - agents = fixture / "AGENTS.md" - agents_text = read_text(agents) - - boundary_patterns = [ - ( - "Use MVC pattern", - "Architecture rule 'Use MVC pattern' in AGENTS.md belongs in constitution.md", - "AGENTS.md contains an architecture pattern rule outside the constitution.", - ), - ( - "State management must be handled via Redux", - "Architecture rule 'State management via Redux' in AGENTS.md belongs in constitution.md", - "AGENTS.md contains a state-management architecture rule outside the constitution.", - ), - ( - "API design must follow REST principles", - "Architecture rule 'API design must follow REST principles' in AGENTS.md belongs in constitution.md", - "AGENTS.md contains an API design rule outside the constitution.", - ), - ] - for needle, description, evidence in boundary_patterns: - if needle in agents_text: - findings.append( - Finding( - drift_type="boundary", - severity="warning", - confidence="high", - description=description, - source="AGENTS.md", - evidence=evidence, - recommended_destination=".specify/memory/constitution.md", - suggested_action="move", - ) - ) - - cursor_rules = fixture / ".cursor/rules/project.md" - cursor_text = read_text(cursor_rules) - if "## Architecture" in cursor_text and "hexagonal architecture" in cursor_text: - findings.append( - Finding( - drift_type="boundary", - severity="warning", - confidence="high", - description="Architecture rules in .cursor/rules/project.md belong in constitution.md, not in editor-specific config", - source=".cursor/rules/project.md", - evidence=".cursor/rules/project.md contains an Architecture section with architecture-boundary rules.", - recommended_destination=".specify/memory/constitution.md", - suggested_action="move", - ) - ) - - return findings +from memorylint_core import generate_report -def scan_reality_drift(fixture: Path) -> list[Finding]: - findings: list[Finding] = [] - agents_text = read_text(fixture / "AGENTS.md") - if "scripts/deploy.sh" in agents_text and not (fixture / "scripts/deploy.sh").exists(): - findings.append( - Finding( - drift_type="reality", - severity="warning", - confidence="high", - description="AGENTS.md references 'scripts/deploy.sh' but the file does not exist in the workspace", - source="AGENTS.md", - evidence="AGENTS.md names scripts/deploy.sh and the fixture has no scripts/deploy.sh file.", - recommended_destination="N/A", - suggested_action="delete", - ) - ) +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Scan MemoryLint regression fixtures.") + parser.add_argument("--fixtures", type=Path, required=True, help="Fixture corpus root.") + parser.add_argument("--check", action="store_true", help="Compare actual findings to expected-findings.json.") + parser.add_argument("--dump", action="store_true", help="Print actual findings for each fixture.") + return parser.parse_args() - if "npm run e2e" in agents_text: - package_json = fixture / "package.json" - if not package_json.exists() or '"e2e"' not in read_text(package_json): - findings.append( - Finding( - drift_type="reality", - severity="warning", - confidence="high", - description="AGENTS.md references 'npm run e2e' but no package.json with e2e script exists in the workspace", - source="AGENTS.md", - evidence="AGENTS.md names npm run e2e and no package.json e2e script exists in the fixture.", - recommended_destination="N/A", - suggested_action="delete", - ) - ) - - if ( - not has_constitution(fixture) - and "## CI Entry Points" in agents_text - and "Workflow files live in `.github/workflows/`" not in agents_text - ): - findings.append( - Finding( - drift_type="reality", - severity="info", - confidence="medium", - description="constitution.md does not exist in the workspace — architecture rules have no canonical home", - source=".specify/memory/constitution.md", - evidence="The fixture does not contain .specify/memory/constitution.md.", - recommended_destination=".specify/memory/constitution.md", - suggested_action="keep", - ) - ) - - extension_text = read_text(fixture / "extension.yml") - declared_commands = set(re.findall(r'name:\s*["\']?([^"\'\s]+)["\']?', extension_text)) - hook_commands = re.findall(r'command:\s*["\']?([^"\'\s]+)["\']?', extension_text) - for command in hook_commands: - if command not in declared_commands: - findings.append( - Finding( - drift_type="reality", - severity="critical", - confidence="high", - description="Hook before_plan references command 'speckit.memorylint.run' which is not declared in provides.commands — applying a rename without updating extension.yml would break hooks", - source="extension.yml", - evidence=f"extension.yml hook references {command}, but provides.commands declares {sorted(declared_commands)}.", - recommended_destination="extension.yml", - suggested_action="rewrite", - ) - ) - - return findings - - -def scan_conflict_drift(fixture: Path) -> list[Finding]: - agents_text = read_text(fixture / "AGENTS.md") - constitution_text = read_text(fixture / ".specify/memory/constitution.md") - if "Always use Conventional Commits" in agents_text and "Never use type prefixes" in constitution_text: - return [ - Finding( - drift_type="conflict", - severity="critical", - confidence="high", - description="AGENTS.md mandates Conventional Commits while constitution.md mandates a custom [JIRA-ID] format — these are mutually exclusive commit message policies", - source="AGENTS.md + .specify/memory/constitution.md", - evidence="AGENTS.md requires Conventional Commits while constitution.md forbids feat:/fix: prefixes.", - recommended_destination="AGENTS.md", - suggested_action="rewrite", - ) - ] - return [] - - -def scan_redundancy_drift(fixture: Path) -> list[Finding]: - findings: list[Finding] = [] - agents_text = read_text(fixture / "AGENTS.md") - constitution_text = read_text(fixture / ".specify/memory/constitution.md") - claude_text = read_text(fixture / "CLAUDE.md") - nested_agents_text = read_text(fixture / "packages/frontend/AGENTS.md") - - if ( - "speckit.{extension-id}.{command-name}" in agents_text - and "speckit.{extension-id}.{command-name}" in constitution_text - ): - findings.append( - Finding( - drift_type="redundancy", - severity="info", - confidence="high", - description="Command naming convention rule appears in both AGENTS.md and constitution.md with near-identical wording", - source="AGENTS.md + .specify/memory/constitution.md", - evidence="Both files contain the same speckit command naming pattern.", - recommended_destination="AGENTS.md", - suggested_action="merge", - ) - ) - - if "Run `make build`" in agents_text and "Run `make build`" in claude_text: - findings.append( - Finding( - drift_type="redundancy", - severity="warning", - confidence="high", - description="Build command rules in CLAUDE.md duplicate AGENTS.md — risks divergence if only one file is updated", - source="AGENTS.md + CLAUDE.md", - evidence="AGENTS.md and CLAUDE.md both list make build and make test.", - recommended_destination="AGENTS.md", - suggested_action="merge", - ) - ) - - if "Use focused commits with Conventional Commit style" in agents_text and ( - "Use focused commits with Conventional Commit style" in nested_agents_text - ): - findings.append( - Finding( - drift_type="redundancy", - severity="warning", - confidence="medium", - description="Git workflow rules in packages/frontend/AGENTS.md duplicate the root AGENTS.md — nested rules risk diverging from root policy", - source="AGENTS.md + packages/frontend/AGENTS.md", - evidence="Root and nested AGENTS.md repeat the focused commits / Conventional Commit policy.", - recommended_destination="AGENTS.md", - suggested_action="merge", - ) - ) - - return findings - - -def scan_fixture(fixture: Path) -> dict: - findings = ( - scan_reality_drift(fixture) - + scan_conflict_drift(fixture) - + scan_redundancy_drift(fixture) - + scan_boundary_drift(fixture) - ) - - finding_dicts = [] - for index, finding in enumerate(findings, start=1): - item = finding.as_dict() - item["id"] = f"ML-{index:03d}" - finding_dicts.append(item) - - sources = [ - { - "path": rel(path, fixture), - "sha256": sha256(path), - } - for path in source_files(fixture) - ] +def normalize_finding(finding: dict[str, object]) -> dict[str, object]: + source = __import__("re").sub(r":\d+(?:-\d+)?", "", str(finding["source"])) return { - "schema_version": "1.0", - "fixture": fixture.name, - "source_metadata": sources, - "findings": finding_dicts, - "metrics": { - "total_instruction_sources_scanned": len(sources), - "total_findings": len(finding_dicts), - "high_confidence_findings": sum(1 for f in finding_dicts if f["confidence"] == "high"), - "medium_confidence_findings": sum(1 for f in finding_dicts if f["confidence"] == "medium"), - "low_confidence_findings": sum(1 for f in finding_dicts if f["confidence"] == "low"), - }, + "drift_type": finding["drift_type"], + "severity": finding["severity"], + "confidence": finding["confidence"], + "description": finding["detail"], + "source": source, + "evidence": finding["evidence"], + "recommended_destination": finding["recommended_destination"], + "suggested_action": finding["suggested_action"], } -def check(fixtures_dir: Path, reports: list[dict]) -> int: - failures: list[str] = [] - for report in reports: - fixture = fixtures_dir / report["fixture"] - expected_path = fixture / "expected-findings.json" - expected = json.loads(expected_path.read_text(encoding="utf-8")) - actual = [ - { - "drift_type": item["drift_type"], - "severity": item["severity"], - "confidence": item["confidence"], - "description": item["description"], - "source": item["source"], - "evidence": item["evidence"], - "recommended_destination": item["recommended_destination"], - "suggested_action": item["suggested_action"], - } - for item in report["findings"] - ] - if actual != expected: - failures.append( - "\n".join( - [ - f"FAIL: {fixture.name} findings mismatch", - "Expected:", - json.dumps(expected, indent=2, ensure_ascii=False), - "Actual:", - json.dumps(actual, indent=2, ensure_ascii=False), - ] - ) - ) - - if failures: - print("\n\n".join(failures), file=sys.stderr) - return 1 - - total = sum(len(report["findings"]) for report in reports) - print(f"fixture scanner passed ({len(reports)} fixtures, {total} findings checked)") - return 0 +def fixture_findings(fixtures_dir: Path) -> list[tuple[str, list[dict[str, object]]]]: + findings_by_fixture: list[tuple[str, list[dict[str, object]]]] = [] + for fixture in sorted(fixtures_dir.iterdir()): + if not fixture.is_dir() or fixture.name.startswith(".") or fixture.name.startswith("_"): + continue + report = generate_report(fixture.resolve()) + normalized = [normalize_finding(finding) for finding in report.findings] + findings_by_fixture.append((fixture.name, normalized)) + return findings_by_fixture + + +def compare(expected: list[dict[str, object]], actual: list[dict[str, object]]) -> tuple[bool, str]: + if expected == actual: + return True, "" + return ( + False, + "Expected findings do not match actual findings.\n" + f"Expected:\n{json.dumps(expected, indent=2, ensure_ascii=False)}\n" + f"Actual:\n{json.dumps(actual, indent=2, ensure_ascii=False)}", + ) def main() -> int: - parser = argparse.ArgumentParser(description="Scan MemoryLint regression fixtures") - parser.add_argument("--fixtures", required=True, type=Path, help="Path to memorylint/tests/fixtures") - parser.add_argument("--check", action="store_true", help="Compare generated findings with expected-findings.json") - parser.add_argument("--pretty", action="store_true", help="Pretty-print JSON output") - args = parser.parse_args() - - fixtures_dir = args.fixtures - reports = [scan_fixture(fixture) for fixture in fixture_dirs(fixtures_dir)] + args = parse_args() + fixtures_dir = args.fixtures.resolve() + findings_by_fixture = fixture_findings(fixtures_dir) + + for fixture_name, findings in findings_by_fixture: + if args.dump: + print(f"## {fixture_name}") + print(json.dumps(findings, indent=2, ensure_ascii=False)) + if args.check: + manifest = fixtures_dir / fixture_name / "expected-findings.json" + expected = json.loads(manifest.read_text(encoding="utf-8")) + ok, message = compare(expected, findings) + if not ok: + raise SystemExit(f"FAIL: {fixture_name}\n{message}") if args.check: - return check(fixtures_dir, reports) - - json.dump( - {"schema_version": "1.0", "fixtures": reports}, - sys.stdout, - indent=2 if args.pretty else None, - ensure_ascii=False, - ) - sys.stdout.write("\n") + print(f"fixture scanner passed ({len(findings_by_fixture)} fixtures checked)") return 0 diff --git a/memorylint/tests/fixtures/bloated-agents/expected-findings.json b/memorylint/tests/fixtures/bloated-agents/expected-findings.json index cfabab1..f1a1892 100644 --- a/memorylint/tests/fixtures/bloated-agents/expected-findings.json +++ b/memorylint/tests/fixtures/bloated-agents/expected-findings.json @@ -3,9 +3,9 @@ "drift_type": "boundary", "severity": "warning", "confidence": "high", - "description": "Architecture rule 'Use MVC pattern' in AGENTS.md belongs in constitution.md", + "description": "Rule 'Use MVC pattern for all user-facing modules.' belongs in .specify/memory/constitution.md, not in AGENTS.md.", "source": "AGENTS.md", - "evidence": "AGENTS.md contains an architecture pattern rule outside the constitution.", + "evidence": "AGENTS.md contains a architecture rule under heading 'Architecture (MISPLACED — should be in constitution)'.", "recommended_destination": ".specify/memory/constitution.md", "suggested_action": "move" }, @@ -13,9 +13,9 @@ "drift_type": "boundary", "severity": "warning", "confidence": "high", - "description": "Architecture rule 'State management via Redux' in AGENTS.md belongs in constitution.md", + "description": "Rule 'State management must be handled via Redux and only Redux.' belongs in .specify/memory/constitution.md, not in AGENTS.md.", "source": "AGENTS.md", - "evidence": "AGENTS.md contains a state-management architecture rule outside the constitution.", + "evidence": "AGENTS.md contains a architecture rule under heading 'Architecture (MISPLACED — should be in constitution)'.", "recommended_destination": ".specify/memory/constitution.md", "suggested_action": "move" }, @@ -23,9 +23,29 @@ "drift_type": "boundary", "severity": "warning", "confidence": "high", - "description": "Architecture rule 'API design must follow REST principles' in AGENTS.md belongs in constitution.md", + "description": "Rule 'API design must follow REST principles with JSON:API response format.' belongs in .specify/memory/constitution.md, not in AGENTS.md.", "source": "AGENTS.md", - "evidence": "AGENTS.md contains an API design rule outside the constitution.", + "evidence": "AGENTS.md contains a architecture rule under heading 'Architecture (MISPLACED — should be in constitution)'.", + "recommended_destination": ".specify/memory/constitution.md", + "suggested_action": "move" + }, + { + "drift_type": "boundary", + "severity": "warning", + "confidence": "high", + "description": "Rule 'Prefer composition over inheritance in all service layers.' belongs in .specify/memory/constitution.md, not in AGENTS.md.", + "source": "AGENTS.md", + "evidence": "AGENTS.md contains a architecture rule under heading 'Code Conventions (MISPLACED — should be in constitution)'.", + "recommended_destination": ".specify/memory/constitution.md", + "suggested_action": "move" + }, + { + "drift_type": "boundary", + "severity": "warning", + "confidence": "high", + "description": "Rule 'All public interfaces must include JSDoc annotations.' belongs in .specify/memory/constitution.md, not in AGENTS.md.", + "source": "AGENTS.md", + "evidence": "AGENTS.md contains a architecture rule under heading 'Code Conventions (MISPLACED — should be in constitution)'.", "recommended_destination": ".specify/memory/constitution.md", "suggested_action": "move" } diff --git a/memorylint/tests/fixtures/conflicting-rules/expected-findings.json b/memorylint/tests/fixtures/conflicting-rules/expected-findings.json index c9955f7..4d3e8fa 100644 --- a/memorylint/tests/fixtures/conflicting-rules/expected-findings.json +++ b/memorylint/tests/fixtures/conflicting-rules/expected-findings.json @@ -1,11 +1,31 @@ [ + { + "drift_type": "boundary", + "severity": "warning", + "confidence": "high", + "description": "Rule 'Commit messages should follow the project's custom format: [JIRA-ID] description.' belongs in AGENTS.md, not in .specify/memory/constitution.md.", + "source": ".specify/memory/constitution.md", + "evidence": ".specify/memory/constitution.md contains a infrastructure rule under heading 'Commit Message Policy'.", + "recommended_destination": "AGENTS.md", + "suggested_action": "move" + }, + { + "drift_type": "boundary", + "severity": "warning", + "confidence": "high", + "description": "Rule 'Never use type prefixes like `feat:` or `fix:` in commit messages.' belongs in AGENTS.md, not in .specify/memory/constitution.md.", + "source": ".specify/memory/constitution.md", + "evidence": ".specify/memory/constitution.md contains a infrastructure rule under heading 'Commit Message Policy'.", + "recommended_destination": "AGENTS.md", + "suggested_action": "move" + }, { "drift_type": "conflict", "severity": "critical", "confidence": "high", - "description": "AGENTS.md mandates Conventional Commits while constitution.md mandates a custom [JIRA-ID] format — these are mutually exclusive commit message policies", + "description": "The commit message policies are mutually exclusive and must be reconciled in one canonical source.", "source": "AGENTS.md + .specify/memory/constitution.md", - "evidence": "AGENTS.md requires Conventional Commits while constitution.md forbids feat:/fix: prefixes.", + "evidence": "AGENTS.md requires Conventional Commits while .specify/memory/constitution.md forbids type prefixes and requires a JIRA format.", "recommended_destination": "AGENTS.md", "suggested_action": "rewrite" } diff --git a/memorylint/tests/fixtures/missing-constitution/expected-findings.json b/memorylint/tests/fixtures/missing-constitution/expected-findings.json index 73c29b7..fe51488 100644 --- a/memorylint/tests/fixtures/missing-constitution/expected-findings.json +++ b/memorylint/tests/fixtures/missing-constitution/expected-findings.json @@ -1,12 +1 @@ -[ - { - "drift_type": "reality", - "severity": "info", - "confidence": "medium", - "description": "constitution.md does not exist in the workspace — architecture rules have no canonical home", - "source": ".specify/memory/constitution.md", - "evidence": "The fixture does not contain .specify/memory/constitution.md.", - "recommended_destination": ".specify/memory/constitution.md", - "suggested_action": "keep" - } -] +[] diff --git a/memorylint/tests/fixtures/monorepo-nested/expected-findings.json b/memorylint/tests/fixtures/monorepo-nested/expected-findings.json index 01f0918..6b930f2 100644 --- a/memorylint/tests/fixtures/monorepo-nested/expected-findings.json +++ b/memorylint/tests/fixtures/monorepo-nested/expected-findings.json @@ -1,11 +1,21 @@ [ + { + "drift_type": "boundary", + "severity": "warning", + "confidence": "high", + "description": "Rule 'Prefer functional components over class components.' belongs in AGENTS.md, not in .specify/memory/constitution.md.", + "source": ".specify/memory/constitution.md", + "evidence": ".specify/memory/constitution.md contains a personal_preference rule under heading 'Code Conventions'.", + "recommended_destination": "AGENTS.md", + "suggested_action": "move" + }, { "drift_type": "redundancy", "severity": "warning", "confidence": "medium", - "description": "Git workflow rules in packages/frontend/AGENTS.md duplicate the root AGENTS.md — nested rules risk diverging from root policy", + "description": "Nested workflow rules duplicate the root policy and risk divergence.", "source": "AGENTS.md + packages/frontend/AGENTS.md", - "evidence": "Root and nested AGENTS.md repeat the focused commits / Conventional Commit policy.", + "evidence": "Root and nested AGENTS.md repeat the focused commits policy.", "recommended_destination": "AGENTS.md", "suggested_action": "merge" } diff --git a/memorylint/tests/fixtures/monorepo-nested/packages/frontend/package.json b/memorylint/tests/fixtures/monorepo-nested/packages/frontend/package.json new file mode 100644 index 0000000..7b4c99b --- /dev/null +++ b/memorylint/tests/fixtures/monorepo-nested/packages/frontend/package.json @@ -0,0 +1,8 @@ +{ + "name": "frontend", + "private": true, + "scripts": { + "build": "echo build", + "test": "echo test" + } +} diff --git a/memorylint/tests/fixtures/multi-source/expected-findings.json b/memorylint/tests/fixtures/multi-source/expected-findings.json index df87a7d..298dbd0 100644 --- a/memorylint/tests/fixtures/multi-source/expected-findings.json +++ b/memorylint/tests/fixtures/multi-source/expected-findings.json @@ -1,22 +1,52 @@ [ { - "drift_type": "redundancy", + "drift_type": "boundary", "severity": "warning", "confidence": "high", - "description": "Build command rules in CLAUDE.md duplicate AGENTS.md — risks divergence if only one file is updated", - "source": "AGENTS.md + CLAUDE.md", - "evidence": "AGENTS.md and CLAUDE.md both list make build and make test.", - "recommended_destination": "AGENTS.md", - "suggested_action": "merge" + "description": "Rule 'Use repository pattern for all data access.' belongs in .specify/memory/constitution.md, not in .cursor/rules/project.md.", + "source": ".cursor/rules/project.md", + "evidence": ".cursor/rules/project.md contains a architecture rule under heading 'Architecture'.", + "recommended_destination": ".specify/memory/constitution.md", + "suggested_action": "move" }, { "drift_type": "boundary", "severity": "warning", "confidence": "high", - "description": "Architecture rules in .cursor/rules/project.md belong in constitution.md, not in editor-specific config", + "description": "Rule 'Services must not depend on concrete implementations.' belongs in .specify/memory/constitution.md, not in .cursor/rules/project.md.", "source": ".cursor/rules/project.md", - "evidence": ".cursor/rules/project.md contains an Architecture section with architecture-boundary rules.", + "evidence": ".cursor/rules/project.md contains a architecture rule under heading 'Architecture'.", "recommended_destination": ".specify/memory/constitution.md", "suggested_action": "move" + }, + { + "drift_type": "boundary", + "severity": "warning", + "confidence": "high", + "description": "Rule 'All modules follow hexagonal architecture boundaries.' belongs in .specify/memory/constitution.md, not in .cursor/rules/project.md.", + "source": ".cursor/rules/project.md", + "evidence": ".cursor/rules/project.md contains a architecture rule under heading 'Architecture'.", + "recommended_destination": ".specify/memory/constitution.md", + "suggested_action": "move" + }, + { + "drift_type": "reality", + "severity": "info", + "confidence": "medium", + "description": "The canonical constitution file is missing. Create or restore it before consolidating architecture/domain rules.", + "source": ".specify/memory/constitution.md", + "evidence": "The workspace does not contain .specify/memory/constitution.md, so architecture/domain rules do not have their canonical home.", + "recommended_destination": ".specify/memory/constitution.md", + "suggested_action": "keep" + }, + { + "drift_type": "redundancy", + "severity": "warning", + "confidence": "high", + "description": "Build command rules are duplicated across AGENTS.md and CLAUDE.md, risking divergence.", + "source": "AGENTS.md + CLAUDE.md", + "evidence": "AGENTS.md and CLAUDE.md both list make build and make test.", + "recommended_destination": "AGENTS.md", + "suggested_action": "merge" } ] diff --git a/memorylint/tests/fixtures/post-apply-breakage/expected-findings.json b/memorylint/tests/fixtures/post-apply-breakage/expected-findings.json index 8098e0c..4e9b5f2 100644 --- a/memorylint/tests/fixtures/post-apply-breakage/expected-findings.json +++ b/memorylint/tests/fixtures/post-apply-breakage/expected-findings.json @@ -1,11 +1,31 @@ [ + { + "drift_type": "reality", + "severity": "warning", + "confidence": "high", + "description": "Rewrite the stale command reference to `speckit.memorylint.load-agents`.", + "source": "AGENTS.md", + "evidence": "AGENTS.md still references removed command `speckit.memorylint.run`.", + "recommended_destination": "AGENTS.md", + "suggested_action": "rewrite" + }, + { + "drift_type": "reality", + "severity": "warning", + "confidence": "high", + "description": "Rewrite the stale command reference to `speckit.memorylint.audit`.", + "source": "AGENTS.md", + "evidence": "AGENTS.md still references removed command `speckit.memorylint.run`.", + "recommended_destination": "AGENTS.md", + "suggested_action": "rewrite" + }, { "drift_type": "reality", "severity": "critical", "confidence": "high", - "description": "Hook before_plan references command 'speckit.memorylint.run' which is not declared in provides.commands — applying a rename without updating extension.yml would break hooks", + "description": "Rewrite hook `before_plan` to use a declared command.", "source": "extension.yml", - "evidence": "extension.yml hook references speckit.memorylint.run, but provides.commands declares ['MemoryLint', 'speckit.memorylint.audit'].", + "evidence": "extension.yml hook `before_plan` references `speckit.memorylint.run`, but provides.commands declares ['speckit.memorylint.audit'].", "recommended_destination": "extension.yml", "suggested_action": "rewrite" } diff --git a/memorylint/tests/fixtures/redundant-rules/expected-findings.json b/memorylint/tests/fixtures/redundant-rules/expected-findings.json index fa8482e..2dfb0e5 100644 --- a/memorylint/tests/fixtures/redundant-rules/expected-findings.json +++ b/memorylint/tests/fixtures/redundant-rules/expected-findings.json @@ -1,12 +1,22 @@ [ + { + "drift_type": "boundary", + "severity": "warning", + "confidence": "high", + "description": "Rule 'Command names must follow `speckit.{extension-id}.{command-name}` pattern.' belongs in .specify/memory/constitution.md, not in AGENTS.md.", + "source": "AGENTS.md", + "evidence": "AGENTS.md contains a domain rule under heading 'Command Naming'.", + "recommended_destination": ".specify/memory/constitution.md", + "suggested_action": "move" + }, { "drift_type": "redundancy", "severity": "info", "confidence": "high", - "description": "Command naming convention rule appears in both AGENTS.md and constitution.md with near-identical wording", + "description": "Keep one canonical copy of the command naming rule to avoid future divergence.", "source": "AGENTS.md + .specify/memory/constitution.md", - "evidence": "Both files contain the same speckit command naming pattern.", - "recommended_destination": "AGENTS.md", + "evidence": "Both sources contain the same speckit command naming pattern.", + "recommended_destination": ".specify/memory/constitution.md", "suggested_action": "merge" } ] diff --git a/memorylint/tests/fixtures/stale-command/expected-findings.json b/memorylint/tests/fixtures/stale-command/expected-findings.json index 7768f92..9a4316d 100644 --- a/memorylint/tests/fixtures/stale-command/expected-findings.json +++ b/memorylint/tests/fixtures/stale-command/expected-findings.json @@ -3,9 +3,9 @@ "drift_type": "reality", "severity": "warning", "confidence": "high", - "description": "AGENTS.md references 'scripts/deploy.sh' but the file does not exist in the workspace", + "description": "Remove or replace the stale reference to `scripts/deploy.sh`.", "source": "AGENTS.md", - "evidence": "AGENTS.md names scripts/deploy.sh and the fixture has no scripts/deploy.sh file.", + "evidence": "AGENTS.md references `scripts/deploy.sh` but that path does not exist in the workspace.", "recommended_destination": "N/A", "suggested_action": "delete" }, @@ -13,9 +13,9 @@ "drift_type": "reality", "severity": "warning", "confidence": "high", - "description": "AGENTS.md references 'npm run e2e' but no package.json with e2e script exists in the workspace", + "description": "Remove or update the stale npm script reference `e2e`.", "source": "AGENTS.md", - "evidence": "AGENTS.md names npm run e2e and no package.json e2e script exists in the fixture.", + "evidence": "AGENTS.md references `npm run e2e` but the workspace has no package.json script named `e2e`.", "recommended_destination": "N/A", "suggested_action": "delete" } diff --git a/memorylint/tests/test-apply-workflow.sh b/memorylint/tests/test-apply-workflow.sh new file mode 100644 index 0000000..22a2be8 --- /dev/null +++ b/memorylint/tests/test-apply-workflow.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +AUDIT_SCRIPT="$ROOT_DIR/memorylint/scripts/audit_workspace.py" +APPLY_SCRIPT="$ROOT_DIR/memorylint/scripts/apply_report.py" +FIXTURES_DIR="$ROOT_DIR/memorylint/tests/fixtures" + +find_python3() { + if command -v python3 >/dev/null 2>&1; then + echo "python3" + elif command -v python >/dev/null 2>&1 && python -c 'import sys; sys.exit(0 if sys.version_info[0] >= 3 else 1)' >/dev/null 2>&1; then + echo "python" + else + echo "ERROR: test-apply-workflow.sh requires Python 3 on PATH" >&2 + exit 1 + fi +} + +PYTHON_BIN=$(find_python3) +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +cp -R "$FIXTURES_DIR/stale-command" "$TMP_DIR/stale-command" +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/stale-command" --json-out "$TMP_DIR/stale-command-report.json" >/dev/null +"$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/stale-command-report.json" --mode apply-safe-fixes >"$TMP_DIR/stale-apply.txt" +if grep -q "scripts/deploy.sh" "$TMP_DIR/stale-command/AGENTS.md"; then + echo "FAIL: apply-safe-fixes should remove stale script reference" >&2 + exit 1 +fi +if grep -q "npm run e2e" "$TMP_DIR/stale-command/AGENTS.md"; then + echo "FAIL: apply-safe-fixes should remove stale npm script reference" >&2 + exit 1 +fi + +cp -R "$FIXTURES_DIR/stale-command" "$TMP_DIR/stale-stale" +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/stale-stale" --json-out "$TMP_DIR/stale-stale-report.json" >/dev/null +printf '\n- manual mutation after audit\n' >> "$TMP_DIR/stale-stale/AGENTS.md" +if "$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/stale-stale-report.json" --mode apply-safe-fixes >"$TMP_DIR/staleness.txt" 2>&1; then + echo "FAIL: apply should reject stale reports" >&2 + exit 1 +fi +grep -q "Staleness check failed" "$TMP_DIR/staleness.txt" || { + echo "FAIL: staleness failure output missing" >&2 + exit 1 +} + +cp -R "$FIXTURES_DIR/post-apply-breakage" "$TMP_DIR/post-apply-breakage" +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/post-apply-breakage" --json-out "$TMP_DIR/post-apply-report.json" >/dev/null +if "$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/post-apply-report.json" --mode apply-all-approved --approve ML-003 >"$TMP_DIR/rollback.txt" 2>&1; then + echo "FAIL: apply-all-approved should fail validation and rollback" >&2 + exit 1 +fi +grep -q "All Changes Reverted" "$TMP_DIR/rollback.txt" || { + echo "FAIL: rollback report missing" >&2 + exit 1 +} +grep -q "speckit.memorylint.run" "$TMP_DIR/post-apply-breakage/extension.yml" || { + echo "FAIL: rollback should restore original extension.yml" >&2 + exit 1 +} + +echo "apply workflow checks passed" diff --git a/memorylint/tests/test-load-agents-proof.sh b/memorylint/tests/test-load-agents-proof.sh new file mode 100644 index 0000000..c98d3be --- /dev/null +++ b/memorylint/tests/test-load-agents-proof.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LOAD_SCRIPT="$ROOT_DIR/memorylint/scripts/load_agents_state.py" +FIXTURES_DIR="$ROOT_DIR/memorylint/tests/fixtures" + +find_python3() { + if command -v python3 >/dev/null 2>&1; then + echo "python3" + elif command -v python >/dev/null 2>&1 && python -c 'import sys; sys.exit(0 if sys.version_info[0] >= 3 else 1)' >/dev/null 2>&1; then + echo "python" + else + echo "ERROR: test-load-agents-proof.sh requires Python 3 on PATH" >&2 + exit 1 + fi +} + +PYTHON_BIN=$(find_python3) + +"$PYTHON_BIN" - "$LOAD_SCRIPT" "$FIXTURES_DIR/clean-repo" <<'PY' +import json +import subprocess +import sys + +payload = json.loads( + subprocess.check_output([sys.executable, sys.argv[1], sys.argv[2]], text=True) +) + +required = {"workspace_root", "agents_path", "agents_sha256", "rule_count", "sections", "rule_summaries"} +missing = required.difference(payload.keys()) +if missing: + raise SystemExit(f"FAIL: load-agents payload missing keys: {sorted(missing)}") + +if payload["agents_path"] != "AGENTS.md": + raise SystemExit("FAIL: load-agents should target the root AGENTS.md") + +if payload["rule_count"] <= 0 or not payload["rule_summaries"]: + raise SystemExit("FAIL: load-agents payload should include extracted rule summaries") + +print("load-agents proof checks passed") +PY diff --git a/memorylint/tests/test-memorylint-regressions.sh b/memorylint/tests/test-memorylint-regressions.sh index e2d4c87..e065c29 100644 --- a/memorylint/tests/test-memorylint-regressions.sh +++ b/memorylint/tests/test-memorylint-regressions.sh @@ -32,6 +32,12 @@ design = design_path.read_text(encoding="utf-8") if design_path.exists() else "" check_boundaries_path = root / "memorylint/commands/check-boundaries.md" fixture_scanner_path = root / "memorylint/scripts/scan_fixtures.py" fixture_scanner_test_path = root / "memorylint/tests/test-fixture-scanner.sh" +workspace_audit_script = root / "memorylint/scripts/audit_workspace.py" +apply_report_script = root / "memorylint/scripts/apply_report.py" +load_agents_script = root / "memorylint/scripts/load_agents_state.py" +workspace_audit_test_path = root / "memorylint/tests/test-workspace-audit.sh" +apply_workflow_test_path = root / "memorylint/tests/test-apply-workflow.sh" +load_agents_test_path = root / "memorylint/tests/test-load-agents-proof.sh" def require(condition: bool, message: str) -> None: @@ -117,11 +123,15 @@ require( # ── audit.md ───────────────────────────────────────────────────────────────── require("$ARGUMENTS" in audit, "audit.md must include the $ARGUMENTS context block") +require("scripts:" in audit, "audit.md must declare scripts frontmatter") require("Instruction Inventory" in audit, "audit.md must include Instruction Inventory") require("Rule Classification" in audit, "audit.md must include Rule Classification") require("Evidence Binding" in audit, "audit.md must include Evidence Binding") require("Drift Detection" in audit, "audit.md must include Drift Detection") +require("Canonical Ownership / Precedence Matrix" in audit, "audit.md must include canonical ownership / precedence matrix") +require("manual_handoff" in audit, "audit.md must define constitution handoff output") +require("edits" in audit, "audit.md must describe executable edits") # All 8 categories for cat in [ @@ -166,6 +176,7 @@ require( # ── apply.md ───────────────────────────────────────────────────────────────── require("$ARGUMENTS" in apply_cmd, "apply.md must include the $ARGUMENTS context block") +require("scripts:" in apply_cmd, "apply.md must declare scripts frontmatter") # Three modes require("report-only" in apply_cmd, "apply.md must include report-only mode") @@ -178,6 +189,7 @@ require("Post-Apply Validation" in apply_cmd, "apply.md must include Post-Apply # Rollback require("Rollback" in apply_cmd, "apply.md must mention Rollback") +require("manual_handoff" in apply_cmd or "handoff artifact" in apply_cmd, "apply.md must mention constitution handoff artifact") # AGENTS.md integrity check require("AGENTS.md" in apply_cmd and "Integrity" in apply_cmd, "apply.md must include AGENTS.md integrity check") @@ -198,6 +210,7 @@ require( "$ARGUMENTS" in load_agents, "load-agents.md must include the $ARGUMENTS context block", ) +require("scripts:" in load_agents, "load-agents.md must declare scripts frontmatter") # Mandatory failure / fail-fast require( @@ -212,6 +225,7 @@ require( "Read-Only" in load_agents or "read-only" in load_agents.lower(), "load-agents.md must mention Read-Only constraint", ) +require("agents_sha256" in load_agents and "rule_summaries" in load_agents, "load-agents.md must define structured proof output") # ── check-boundaries.md must NOT exist (old command removed) ───────────────── @@ -230,6 +244,12 @@ require( fixture_scanner_test_path.exists(), "memorylint/tests/test-fixture-scanner.sh must exist", ) +require(workspace_audit_script.exists(), "memorylint/scripts/audit_workspace.py must exist") +require(apply_report_script.exists(), "memorylint/scripts/apply_report.py must exist") +require(load_agents_script.exists(), "memorylint/scripts/load_agents_state.py must exist") +require(workspace_audit_test_path.exists(), "memorylint/tests/test-workspace-audit.sh must exist") +require(apply_workflow_test_path.exists(), "memorylint/tests/test-apply-workflow.sh must exist") +require(load_agents_test_path.exists(), "memorylint/tests/test-load-agents-proof.sh must exist") # ── design document ───────────────────────────────────────────────────────── @@ -238,7 +258,9 @@ for phrase in [ "Product Boundary", "Trust Model", "Audit Pipeline", + "Source Ownership Matrix", "Apply Gate", + "Planning Gate", "Machine-Readable Report", "Regression Corpus", ]: diff --git a/memorylint/tests/test-workspace-audit.sh b/memorylint/tests/test-workspace-audit.sh new file mode 100644 index 0000000..75ee54c --- /dev/null +++ b/memorylint/tests/test-workspace-audit.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +AUDIT_SCRIPT="$ROOT_DIR/memorylint/scripts/audit_workspace.py" +FIXTURES_DIR="$ROOT_DIR/memorylint/tests/fixtures" + +find_python3() { + if command -v python3 >/dev/null 2>&1; then + echo "python3" + elif command -v python >/dev/null 2>&1 && python -c 'import sys; sys.exit(0 if sys.version_info[0] >= 3 else 1)' >/dev/null 2>&1; then + echo "python" + else + echo "ERROR: test-workspace-audit.sh requires Python 3 on PATH" >&2 + exit 1 + fi +} + +PYTHON_BIN=$(find_python3) +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$FIXTURES_DIR/clean-repo" --json-out "$TMP_DIR/clean.json" >/dev/null +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$FIXTURES_DIR/bloated-agents" --json-out "$TMP_DIR/bloated.json" >/dev/null + +"$PYTHON_BIN" - "$TMP_DIR/clean.json" "$TMP_DIR/bloated.json" <<'PY' +import json +import sys +from pathlib import Path + +clean = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +bloated = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + +if clean["metrics"]["total_findings"] != 0: + raise SystemExit("FAIL: clean-repo workspace audit should produce zero findings") + +required_top_level = {"schema_version", "workspace_root", "source_metadata", "instruction_map", "findings", "metrics", "summary"} +missing = required_top_level.difference(bloated.keys()) +if missing: + raise SystemExit(f"FAIL: bloated-agents report missing keys: {sorted(missing)}") + +if not bloated["metrics"]["files_that_would_be_modified"]: + raise SystemExit("FAIL: bloated-agents report should list files_that_would_be_modified") + +handoffs = [finding for finding in bloated["findings"] if finding.get("manual_handoff")] +if not handoffs: + raise SystemExit("FAIL: bloated-agents should emit at least one constitution manual handoff") + +print("workspace audit checks passed") +PY From 85c0fba41322ea898a5c8971a617c073404d5896 Mon Sep 17 00:00:00 2001 From: rbbtsn0w Date: Fri, 29 May 2026 10:42:00 +0800 Subject: [PATCH 2/7] fix(memorylint): include hidden post-apply fixture Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../post-apply-breakage/.specify/memory/constitution.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 memorylint/tests/fixtures/post-apply-breakage/.specify/memory/constitution.md diff --git a/memorylint/tests/fixtures/post-apply-breakage/.specify/memory/constitution.md b/memorylint/tests/fixtures/post-apply-breakage/.specify/memory/constitution.md new file mode 100644 index 0000000..c4cbc6c --- /dev/null +++ b/memorylint/tests/fixtures/post-apply-breakage/.specify/memory/constitution.md @@ -0,0 +1,5 @@ +# Constitution + +## Module Boundaries + +- Extensions must keep hook wiring consistent with declared commands. From d8f7e9ef0b8f683fd4366f722cf08b1be1a460e6 Mon Sep 17 00:00:00 2001 From: rbbtsn0w Date: Fri, 29 May 2026 11:05:41 +0800 Subject: [PATCH 3/7] fix(memorylint): block workspace path escapes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- memorylint/scripts/apply_report.py | 17 ++++++--- memorylint/scripts/memorylint_core.py | 43 +++++++++++++++++++++-- memorylint/tests/test-apply-workflow.sh | 44 ++++++++++++++++++++++++ memorylint/tests/test-workspace-audit.sh | 15 +++++++- 4 files changed, 111 insertions(+), 8 deletions(-) diff --git a/memorylint/scripts/apply_report.py b/memorylint/scripts/apply_report.py index c05ba68..a9fa809 100644 --- a/memorylint/scripts/apply_report.py +++ b/memorylint/scripts/apply_report.py @@ -14,6 +14,7 @@ format_apply_failure, format_apply_summary, read_text, + resolve_workspace_path, validate_agents_integrity, validate_constitution_integrity, validate_hook_consistency, @@ -73,8 +74,14 @@ def main() -> int: source_hashes = {item["path"]: item["sha256"] for item in report_copy.get("source_metadata", [])} target_paths = sorted({edit["path"] for finding in approved for edit in finding.get("edits", [])}) + resolved_targets: dict[str, Path] = {} for relative in target_paths: - target_path = workspace_root / relative + try: + target_path = resolve_workspace_path(workspace_root, relative) + except ValueError as exc: + print(format_apply_failure([str(exc)], []), end="") + return 1 + resolved_targets[relative] = target_path if not target_path.exists(): print(format_apply_failure([f"Target file disappeared after audit: {relative}"], []), end="") return 1 @@ -83,16 +90,16 @@ def main() -> int: print(format_apply_failure([f"Staleness check failed for {relative}"], []), end="") return 1 - originals = {relative: read_text(workspace_root / relative) for relative in target_paths} + originals = {relative: read_text(resolved_targets[relative]) for relative in target_paths} edits_by_file = grouped_edits(approved) for relative, edits in edits_by_file.items(): - target_path = workspace_root / relative + target_path = resolved_targets[relative] updated = apply_edits_to_lines(read_text(target_path).splitlines(), edits) target_path.write_text("\n".join(updated).rstrip() + "\n", encoding="utf-8") validation_issues: list[str] = [] for relative, before_text in originals.items(): - after_text = read_text(workspace_root / relative) + after_text = read_text(resolved_targets[relative]) if relative.endswith("AGENTS.md"): validation_issues.extend(validate_agents_integrity(before_text, after_text)) if relative.endswith(".specify/memory/constitution.md"): @@ -102,7 +109,7 @@ def main() -> int: if validation_issues: for relative, before_text in originals.items(): - (workspace_root / relative).write_text(before_text, encoding="utf-8") + resolved_targets[relative].write_text(before_text, encoding="utf-8") print(format_apply_failure(validation_issues, sorted(originals)), end="") return 1 diff --git a/memorylint/scripts/memorylint_core.py b/memorylint/scripts/memorylint_core.py index a2da6ca..b8271d0 100644 --- a/memorylint/scripts/memorylint_core.py +++ b/memorylint/scripts/memorylint_core.py @@ -126,6 +126,15 @@ def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def resolve_workspace_path(workspace_root: Path, relative: str) -> Path: + candidate = (workspace_root / relative).resolve() + try: + candidate.relative_to(workspace_root) + except ValueError as exc: + raise ValueError(f"Path escapes workspace: {relative}") from exc + return candidate + + def relative_path(path: Path, workspace_root: Path) -> str: return path.relative_to(workspace_root).as_posix() @@ -465,9 +474,39 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin line_number = int(rule.line_range.split("-")[0]) for reference in detect_path_references(rule): - candidate = (workspace_root / reference).resolve() + key = (rule.source, reference) + try: + candidate = resolve_workspace_path(workspace_root, reference) + except ValueError: + if key in seen_sources: + continue + seen_sources.add(key) + findings.append( + Finding( + id="", + drift_type="reality", + severity="warning", + confidence="high", + source=f"{rule.source}:{rule.line_range}", + evidence=f"{rule.source} references `{reference}` but that path escapes the workspace boundary.", + recommended_destination="N/A", + suggested_action="delete", + detail=f"Remove or replace the out-of-workspace reference `{reference}`.", + rule_ids=[rule.rule_id], + category=rule.category, + edits=[ + Edit( + path=rule.source, + action="delete", + start_line=line_number, + end_line=line_number, + reason=f"Delete out-of-workspace reference {reference}.", + ) + ], + ) + ) + continue if not candidate.exists(): - key = (rule.source, reference) if key in seen_sources: continue seen_sources.add(key) diff --git a/memorylint/tests/test-apply-workflow.sh b/memorylint/tests/test-apply-workflow.sh index 22a2be8..ccf0713 100644 --- a/memorylint/tests/test-apply-workflow.sh +++ b/memorylint/tests/test-apply-workflow.sh @@ -61,4 +61,48 @@ grep -q "speckit.memorylint.run" "$TMP_DIR/post-apply-breakage/extension.yml" || exit 1 } +cp -R "$FIXTURES_DIR/stale-command" "$TMP_DIR/path-escape" +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/path-escape" --json-out "$TMP_DIR/path-escape-report.json" >/dev/null +printf 'outside\n' > "$TMP_DIR/escaped.txt" +"$PYTHON_BIN" - "$TMP_DIR/path-escape-report.json" <<'PY' +import json +import sys +from pathlib import Path + +report_path = Path(sys.argv[1]) +report = json.loads(report_path.read_text(encoding="utf-8")) +report["findings"] = [{ + "id": "ML-escape", + "drift_type": "reality", + "severity": "warning", + "confidence": "high", + "source": "AGENTS.md:3-3", + "evidence": "malicious report", + "recommended_destination": "AGENTS.md", + "suggested_action": "delete", + "detail": "attempt escape", + "edits": [{ + "path": "../escaped.txt", + "action": "delete", + "start_line": 1, + "end_line": 1, + "reason": "escape test" + }] +}] +report["source_metadata"] = [{"path": "../escaped.txt", "sha256": "ignored"}] +report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") +PY +if "$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/path-escape-report.json" --mode apply-all-approved --approve ML-escape >"$TMP_DIR/path-escape.txt" 2>&1; then + echo "FAIL: apply should reject report paths outside the workspace" >&2 + exit 1 +fi +grep -q "Path escapes workspace" "$TMP_DIR/path-escape.txt" || { + echo "FAIL: workspace escape failure output missing" >&2 + exit 1 +} +grep -q '^outside$' "$TMP_DIR/escaped.txt" || { + echo "FAIL: escaped target should remain unchanged" >&2 + exit 1 +} + echo "apply workflow checks passed" diff --git a/memorylint/tests/test-workspace-audit.sh b/memorylint/tests/test-workspace-audit.sh index 75ee54c..d05a44e 100644 --- a/memorylint/tests/test-workspace-audit.sh +++ b/memorylint/tests/test-workspace-audit.sh @@ -23,14 +23,23 @@ trap 'rm -rf "$TMP_DIR"' EXIT "$PYTHON_BIN" "$AUDIT_SCRIPT" "$FIXTURES_DIR/clean-repo" --json-out "$TMP_DIR/clean.json" >/dev/null "$PYTHON_BIN" "$AUDIT_SCRIPT" "$FIXTURES_DIR/bloated-agents" --json-out "$TMP_DIR/bloated.json" >/dev/null +mkdir -p "$TMP_DIR/escape-ref" +cat >"$TMP_DIR/escape-ref/AGENTS.md" <<'EOF' +# Workspace Rules -"$PYTHON_BIN" - "$TMP_DIR/clean.json" "$TMP_DIR/bloated.json" <<'PY' +- Run `../outside.sh` before deploy. +EOF +printf '#!/usr/bin/env bash\n' >"$TMP_DIR/outside.sh" +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/escape-ref" --json-out "$TMP_DIR/escape.json" >/dev/null + +"$PYTHON_BIN" - "$TMP_DIR/clean.json" "$TMP_DIR/bloated.json" "$TMP_DIR/escape.json" <<'PY' import json import sys from pathlib import Path clean = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) bloated = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) +escaped = json.loads(Path(sys.argv[3]).read_text(encoding="utf-8")) if clean["metrics"]["total_findings"] != 0: raise SystemExit("FAIL: clean-repo workspace audit should produce zero findings") @@ -47,5 +56,9 @@ handoffs = [finding for finding in bloated["findings"] if finding.get("manual_ha if not handoffs: raise SystemExit("FAIL: bloated-agents should emit at least one constitution manual handoff") +escape_findings = [finding for finding in escaped["findings"] if "../outside.sh" in finding.get("evidence", "")] +if not escape_findings: + raise SystemExit("FAIL: workspace audit should flag out-of-workspace path references") + print("workspace audit checks passed") PY From b3e89da83654eefa81daac4dd75f49d0b2c9fc4d Mon Sep 17 00:00:00 2001 From: rbbtsn0w Date: Fri, 29 May 2026 11:09:48 +0800 Subject: [PATCH 4/7] fix(memorylint): address PR review findings Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 3 + memorylint/scripts/apply_report.py | 8 +- memorylint/scripts/memorylint_core.py | 99 +++++++++++-- memorylint/scripts/scan_fixtures.py | 3 +- memorylint/tests/test-apply-workflow.sh | 137 ++++++++++++++++++ .../tests/test-memorylint-regressions.sh | 13 ++ memorylint/tests/test-workspace-audit.sh | 59 +++++++- 7 files changed, 307 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 665b67c..034ff9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,9 @@ jobs: bash memorylint/tests/test-memorylint-regressions.sh bash memorylint/tests/test-fixture-validation.sh bash memorylint/tests/test-fixture-scanner.sh + bash memorylint/tests/test-workspace-audit.sh + bash memorylint/tests/test-apply-workflow.sh + bash memorylint/tests/test-load-agents-proof.sh powershell-bridge-tests: name: powershell-bridge-tests diff --git a/memorylint/scripts/apply_report.py b/memorylint/scripts/apply_report.py index a9fa809..593819b 100644 --- a/memorylint/scripts/apply_report.py +++ b/memorylint/scripts/apply_report.py @@ -4,6 +4,8 @@ from __future__ import annotations import argparse +import hashlib +import json from collections import defaultdict from pathlib import Path @@ -18,6 +20,7 @@ validate_agents_integrity, validate_constitution_integrity, validate_hook_consistency, + validate_repository_diff, apply_edits_to_lines, ) @@ -64,7 +67,7 @@ def main() -> int: manual_handoffs = [finding for finding in report_copy.get("findings", []) if finding.get("manual_handoff")] if args.handoff_out: args.handoff_out.write_text( - __import__("json").dumps(manual_handoffs, indent=2, ensure_ascii=False) + "\n", + json.dumps(manual_handoffs, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", ) @@ -85,7 +88,7 @@ def main() -> int: if not target_path.exists(): print(format_apply_failure([f"Target file disappeared after audit: {relative}"], []), end="") return 1 - current_hash = __import__("hashlib").sha256(target_path.read_bytes()).hexdigest() + current_hash = hashlib.sha256(target_path.read_bytes()).hexdigest() if source_hashes.get(relative) != current_hash: print(format_apply_failure([f"Staleness check failed for {relative}"], []), end="") return 1 @@ -106,6 +109,7 @@ def main() -> int: finding_map = {finding["id"]: finding for finding in approved} validation_issues.extend(validate_constitution_integrity(before_text, after_text, finding_map)) validation_issues.extend(validate_hook_consistency(workspace_root)) + validation_issues.extend(validate_repository_diff(workspace_root)) if validation_issues: for relative, before_text in originals.items(): diff --git a/memorylint/scripts/memorylint_core.py b/memorylint/scripts/memorylint_core.py index b8271d0..26371d9 100644 --- a/memorylint/scripts/memorylint_core.py +++ b/memorylint/scripts/memorylint_core.py @@ -7,6 +7,7 @@ import hashlib import json import re +import subprocess from collections import defaultdict from dataclasses import asdict, dataclass, field from pathlib import Path @@ -126,8 +127,9 @@ def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def resolve_workspace_path(workspace_root: Path, relative: str) -> Path: - candidate = (workspace_root / relative).resolve() +def resolve_workspace_path(workspace_root: Path, relative: str, *, base_path: Path | None = None) -> Path: + candidate_root = base_path if base_path is not None else workspace_root + candidate = (candidate_root / relative).resolve() try: candidate.relative_to(workspace_root) except ValueError as exc: @@ -272,6 +274,20 @@ def parse_extension_hooks(text: str) -> dict[str, str]: return hooks +def find_hook_command_line(text: str, hook_name: str) -> int | None: + current_hook: str | None = None + for index, line in enumerate(text.splitlines(), start=1): + hook_name_match = re.match(r"^\s{2}([a-z_]+):\s*$", line) + if hook_name_match: + current_hook = hook_name_match.group(1) + continue + if current_hook != hook_name: + continue + if re.match(r'^\s{4}command:\s*["\']?([^"\']+)["\']?\s*$', line): + return index + return None + + def manifest_rules(path: Path, workspace_root: Path, next_rule_id: int) -> tuple[list[Rule], int]: text = read_text(path) rules: list[Rule] = [] @@ -280,7 +296,7 @@ def manifest_rules(path: Path, workspace_root: Path, next_rule_id: int) -> tuple extension_id = extension_id_match.group(1) if extension_id_match else "" for hook_name, command_name in parse_extension_hooks(text).items(): - line_number = find_line_number(text, f"command: \"{command_name}\"") or find_line_number(text, f"command: {command_name}") or 1 + line_number = find_hook_command_line(text, hook_name) or 1 rule_id = f"R-{next_rule_id:03d}" next_rule_id += 1 summary = f"Hook {hook_name} uses command {command_name}" @@ -476,7 +492,7 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin for reference in detect_path_references(rule): key = (rule.source, reference) try: - candidate = resolve_workspace_path(workspace_root, reference) + candidate = resolve_workspace_path(workspace_root, reference, base_path=rule_path.parent) except ValueError: if key in seen_sources: continue @@ -539,7 +555,34 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin if npm_match: script_name = npm_match.group(1) package_json = nearest_package_json(rule_path, workspace_root) - package_data = json.loads(package_json.read_text(encoding="utf-8")) if package_json and package_json.exists() else {} + package_data: dict[str, object] = {} + if package_json and package_json.exists(): + try: + loaded = json.loads(package_json.read_text(encoding="utf-8")) + except json.JSONDecodeError: + manifest_reference = relative_path(package_json, workspace_root) + key = (rule.source, manifest_reference) + if key in seen_sources: + continue + seen_sources.add(key) + findings.append( + Finding( + id="", + drift_type="reality", + severity="warning", + confidence="high", + source=f"{rule.source}:{rule.line_range}", + evidence=f"{rule.source} references `npm run {script_name}`, but `{manifest_reference}` is not valid JSON.", + recommended_destination=manifest_reference, + suggested_action="rewrite", + detail=f"Fix `{manifest_reference}` so MemoryLint can verify the `{script_name}` script.", + rule_ids=[rule.rule_id], + category=rule.category, + ) + ) + continue + if isinstance(loaded, dict): + package_data = loaded scripts = package_data.get("scripts", {}) if isinstance(package_data, dict) else {} if script_name not in scripts: findings.append( @@ -595,7 +638,11 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin ) ) + scanned_manifests: set[str] = set() for manifest in [rule for rule in rules if path_kind(Path(rule.source)) == "manifest"]: + if manifest.source in scanned_manifests: + continue + scanned_manifests.add(manifest.source) manifest_path = workspace_root / manifest.source manifest_text = read_text(manifest_path) extension_id_match = re.search(r'^\s*id:\s*["\']?([^"\']+)["\']?\s*$', manifest_text, re.MULTILINE) @@ -605,7 +652,7 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin if command_name in declared: continue replacement = MEMORYLINT_EXPECTED_HOOKS.get(hook_name, command_name) if extension_id == "memorylint" else command_name - line_number = find_line_number(manifest_text, f'command: "{command_name}"') or find_line_number(manifest_text, f"command: {command_name}") or 1 + line_number = find_hook_command_line(manifest_text, hook_name) or 1 findings.append( Finding( id="", @@ -1023,7 +1070,7 @@ def approved_findings(report_payload: dict[str, object], mode: str, approved_ids if mode == "apply-safe-fixes": return [finding for finding in findings if safe_mode_eligible(finding)] approved_ids = approved_ids or set() - return [finding for finding in findings if finding["id"] in approved_ids or safe_mode_eligible(finding)] + return [finding for finding in findings if finding["id"] in approved_ids] def apply_edits_to_lines(lines: list[str], edits: list[dict[str, object]]) -> list[str]: @@ -1055,12 +1102,12 @@ def validate_agents_integrity(before_text: str, after_text: str) -> list[str]: if any(keyword in heading for heading in before_normalized for keyword in family): if not any(keyword in heading for heading in after_normalized for keyword in family): issues.append(f"Missing AGENTS.md critical section family after apply: {family[0]}") + has_heading_above = False for index, line in enumerate(after_text.splitlines(), start=1): + if re.match(r"^\s{0,3}##\s+", line): + has_heading_above = True + continue if re.match(r"^\s*[-*]\s+", line): - has_heading_above = any( - re.match(r"^\s{0,3}##\s+", earlier) - for earlier in after_text.splitlines()[: index - 1] - ) if not has_heading_above: issues.append(f"Found orphaned list item in AGENTS.md at line {index}") break @@ -1100,6 +1147,36 @@ def validate_hook_consistency(workspace_root: Path) -> list[str]: return issues +def validate_repository_diff(workspace_root: Path) -> list[str]: + try: + repo_check = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=workspace_root, + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError: + return [] + if repo_check.returncode != 0: + return [] + + diff_check = subprocess.run( + ["git", "diff", "--check", "--", "."], + cwd=workspace_root, + capture_output=True, + text=True, + check=False, + ) + if diff_check.returncode == 0: + return [] + + output = (diff_check.stdout + diff_check.stderr).strip() + if not output: + return ["git diff --check failed after apply."] + return [f"git diff --check failed: {line}" for line in output.splitlines()] + + def format_apply_summary(applied: list[dict[str, object]], files_modified: list[str], validation_issues: list[str]) -> str: lines = [ "## Apply Summary", diff --git a/memorylint/scripts/scan_fixtures.py b/memorylint/scripts/scan_fixtures.py index 25e8143..8ec51dd 100644 --- a/memorylint/scripts/scan_fixtures.py +++ b/memorylint/scripts/scan_fixtures.py @@ -5,6 +5,7 @@ import argparse import json +import re from pathlib import Path from memorylint_core import generate_report @@ -19,7 +20,7 @@ def parse_args() -> argparse.Namespace: def normalize_finding(finding: dict[str, object]) -> dict[str, object]: - source = __import__("re").sub(r":\d+(?:-\d+)?", "", str(finding["source"])) + source = re.sub(r":\d+(?:-\d+)?", "", str(finding["source"])) return { "drift_type": finding["drift_type"], "severity": finding["severity"], diff --git a/memorylint/tests/test-apply-workflow.sh b/memorylint/tests/test-apply-workflow.sh index ccf0713..8840727 100644 --- a/memorylint/tests/test-apply-workflow.sh +++ b/memorylint/tests/test-apply-workflow.sh @@ -105,4 +105,141 @@ grep -q '^outside$' "$TMP_DIR/escaped.txt" || { exit 1 } +mkdir -p "$TMP_DIR/apply-approved" +cat >"$TMP_DIR/apply-approved/AGENTS.md" <<'EOF' +# Workspace Rules + +## Commands + +- Remove stale script reference. +- Remove stale npm script reference. +EOF +"$PYTHON_BIN" - "$TMP_DIR/apply-approved/approved-report.json" "$TMP_DIR/apply-approved" <<'PY' +import hashlib +import json +import sys +from pathlib import Path + +report_path = Path(sys.argv[1]) +workspace = Path(sys.argv[2]).resolve() +agents_path = workspace / "AGENTS.md" +report = { + "schema_version": "1.0", + "workspace_root": str(workspace), + "source_metadata": [{ + "path": "AGENTS.md", + "sha256": hashlib.sha256(agents_path.read_bytes()).hexdigest(), + }], + "instruction_map": [], + "findings": [ + { + "id": "ML-safe", + "drift_type": "reality", + "severity": "warning", + "confidence": "high", + "source": "AGENTS.md:5", + "evidence": "safe finding", + "recommended_destination": "AGENTS.md", + "suggested_action": "delete", + "detail": "delete first line", + "edits": [{"path": "AGENTS.md", "action": "delete", "start_line": 5, "end_line": 5, "reason": "safe"}], + }, + { + "id": "ML-approved", + "drift_type": "reality", + "severity": "critical", + "confidence": "high", + "source": "AGENTS.md:6", + "evidence": "approved finding", + "recommended_destination": "AGENTS.md", + "suggested_action": "delete", + "detail": "delete second line", + "edits": [{"path": "AGENTS.md", "action": "delete", "start_line": 6, "end_line": 6, "reason": "approved"}], + }, + ], + "metrics": {}, + "summary": {}, +} +report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") +PY +"$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/apply-approved/approved-report.json" --mode apply-all-approved --approve ML-approved >/dev/null +grep -q "Remove stale script reference" "$TMP_DIR/apply-approved/AGENTS.md" || { + echo "FAIL: apply-all-approved should not apply unapproved safe findings" >&2 + exit 1 +} +if grep -q "Remove stale npm script reference" "$TMP_DIR/apply-approved/AGENTS.md"; then + echo "FAIL: apply-all-approved should apply explicitly approved findings" >&2 + exit 1 +fi + +mkdir -p "$TMP_DIR/git-diff-check" +cat >"$TMP_DIR/git-diff-check/AGENTS.md" <<'EOF' +# Workspace Rules + +## Commands + +- Clean command +- Stable command +EOF +git -C "$TMP_DIR/git-diff-check" init -q +git -C "$TMP_DIR/git-diff-check" config user.name "MemoryLint Test" +git -C "$TMP_DIR/git-diff-check" config user.email "memorylint@example.com" +git -C "$TMP_DIR/git-diff-check" add AGENTS.md +git -C "$TMP_DIR/git-diff-check" commit -qm "init" +"$PYTHON_BIN" - "$TMP_DIR/git-diff-check/git-report.json" "$TMP_DIR/git-diff-check" <<'PY' +import hashlib +import json +import sys +from pathlib import Path + +report_path = Path(sys.argv[1]) +workspace = Path(sys.argv[2]).resolve() +agents_path = workspace / "AGENTS.md" +report = { + "schema_version": "1.0", + "workspace_root": str(workspace), + "source_metadata": [{ + "path": "AGENTS.md", + "sha256": hashlib.sha256(agents_path.read_bytes()).hexdigest(), + }], + "instruction_map": [], + "findings": [ + { + "id": "ML-diff", + "drift_type": "reality", + "severity": "warning", + "confidence": "high", + "source": "AGENTS.md:5", + "evidence": "git diff check", + "recommended_destination": "AGENTS.md", + "suggested_action": "rewrite", + "detail": "introduce trailing whitespace", + "edits": [{ + "path": "AGENTS.md", + "action": "replace", + "start_line": 5, + "end_line": 5, + "replacement": ["- Dirty command "], + "reason": "diff check", + }], + }, + ], + "metrics": {}, + "summary": {}, +} +report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") +PY +if "$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/git-diff-check/git-report.json" --mode apply-all-approved --approve ML-diff >"$TMP_DIR/git-diff-check/output.txt" 2>&1; then + echo "FAIL: apply should rollback when git diff --check fails" >&2 + exit 1 +fi +grep -q "git diff --check failed" "$TMP_DIR/git-diff-check/output.txt" || { + echo "FAIL: git diff validation failure missing" >&2 + exit 1 +} +grep -q "Clean command" "$TMP_DIR/git-diff-check/AGENTS.md" || { + echo "FAIL: git diff validation rollback should restore original file" >&2 + exit 1 +} + echo "apply workflow checks passed" diff --git a/memorylint/tests/test-memorylint-regressions.sh b/memorylint/tests/test-memorylint-regressions.sh index e065c29..203c7bf 100644 --- a/memorylint/tests/test-memorylint-regressions.sh +++ b/memorylint/tests/test-memorylint-regressions.sh @@ -38,6 +38,7 @@ load_agents_script = root / "memorylint/scripts/load_agents_state.py" workspace_audit_test_path = root / "memorylint/tests/test-workspace-audit.sh" apply_workflow_test_path = root / "memorylint/tests/test-apply-workflow.sh" load_agents_test_path = root / "memorylint/tests/test-load-agents-proof.sh" +ci_workflow = (root / ".github/workflows/ci.yml").read_text(encoding="utf-8") def require(condition: bool, message: str) -> None: @@ -250,6 +251,18 @@ require(load_agents_script.exists(), "memorylint/scripts/load_agents_state.py mu require(workspace_audit_test_path.exists(), "memorylint/tests/test-workspace-audit.sh must exist") require(apply_workflow_test_path.exists(), "memorylint/tests/test-apply-workflow.sh must exist") require(load_agents_test_path.exists(), "memorylint/tests/test-load-agents-proof.sh must exist") +require( + "bash memorylint/tests/test-workspace-audit.sh" in ci_workflow, + "CI must run memorylint/tests/test-workspace-audit.sh", +) +require( + "bash memorylint/tests/test-apply-workflow.sh" in ci_workflow, + "CI must run memorylint/tests/test-apply-workflow.sh", +) +require( + "bash memorylint/tests/test-load-agents-proof.sh" in ci_workflow, + "CI must run memorylint/tests/test-load-agents-proof.sh", +) # ── design document ───────────────────────────────────────────────────────── diff --git a/memorylint/tests/test-workspace-audit.sh b/memorylint/tests/test-workspace-audit.sh index d05a44e..a1ff417 100644 --- a/memorylint/tests/test-workspace-audit.sh +++ b/memorylint/tests/test-workspace-audit.sh @@ -31,8 +31,48 @@ cat >"$TMP_DIR/escape-ref/AGENTS.md" <<'EOF' EOF printf '#!/usr/bin/env bash\n' >"$TMP_DIR/outside.sh" "$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/escape-ref" --json-out "$TMP_DIR/escape.json" >/dev/null +mkdir -p "$TMP_DIR/nested-ref/packages/frontend/scripts" +cat >"$TMP_DIR/nested-ref/packages/frontend/AGENTS.md" <<'EOF' +# Frontend Rules -"$PYTHON_BIN" - "$TMP_DIR/clean.json" "$TMP_DIR/bloated.json" "$TMP_DIR/escape.json" <<'PY' +## Commands + +- Run `scripts/build.sh` before release. +EOF +printf '#!/usr/bin/env bash\n' >"$TMP_DIR/nested-ref/packages/frontend/scripts/build.sh" +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/nested-ref" --json-out "$TMP_DIR/nested.json" >/dev/null +mkdir -p "$TMP_DIR/invalid-package" +cat >"$TMP_DIR/invalid-package/AGENTS.md" <<'EOF' +# Package Rules + +## Commands + +- Run `npm run build` before release. +EOF +cat >"$TMP_DIR/invalid-package/package.json" <<'EOF' +{"scripts": +EOF +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/invalid-package" --json-out "$TMP_DIR/invalid-package.json" >/dev/null +mkdir -p "$TMP_DIR/single-quote-hooks" +cat >"$TMP_DIR/single-quote-hooks/extension.yml" <<'EOF' +schema_version: "1.0" +extension: + id: memorylint + version: "0.1.0" + description: "Fixture for single quoted hooks." +hooks: + before_plan: + command: 'speckit.memorylint.run' + after_constitution: + command: 'speckit.memorylint.run' +provides: + commands: + - name: speckit.memorylint.audit + description: "Audit" +EOF +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/single-quote-hooks" --json-out "$TMP_DIR/single-quote.json" >/dev/null + +"$PYTHON_BIN" - "$TMP_DIR/clean.json" "$TMP_DIR/bloated.json" "$TMP_DIR/escape.json" "$TMP_DIR/nested.json" "$TMP_DIR/invalid-package.json" "$TMP_DIR/single-quote.json" <<'PY' import json import sys from pathlib import Path @@ -40,6 +80,9 @@ from pathlib import Path clean = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) bloated = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) escaped = json.loads(Path(sys.argv[3]).read_text(encoding="utf-8")) +nested = json.loads(Path(sys.argv[4]).read_text(encoding="utf-8")) +invalid_package = json.loads(Path(sys.argv[5]).read_text(encoding="utf-8")) +single_quote = json.loads(Path(sys.argv[6]).read_text(encoding="utf-8")) if clean["metrics"]["total_findings"] != 0: raise SystemExit("FAIL: clean-repo workspace audit should produce zero findings") @@ -60,5 +103,19 @@ escape_findings = [finding for finding in escaped["findings"] if "../outside.sh" if not escape_findings: raise SystemExit("FAIL: workspace audit should flag out-of-workspace path references") +nested_stale = [finding for finding in nested["findings"] if "scripts/build.sh" in finding.get("evidence", "")] +if nested_stale: + raise SystemExit("FAIL: nested package-local script paths should resolve relative to their rule file") + +invalid_json = [finding for finding in invalid_package["findings"] if "not valid JSON" in finding.get("evidence", "")] +if not invalid_json: + raise SystemExit("FAIL: malformed package.json should produce an explicit finding instead of crashing audit") + +hook_findings = [finding for finding in single_quote["findings"] if "hook `" in finding.get("evidence", "")] +if len(hook_findings) != 2: + raise SystemExit(f"FAIL: single-quoted hook fixture should produce exactly 2 hook findings, got {len(hook_findings)}") +if any(finding["source"].endswith(":1") for finding in hook_findings): + raise SystemExit("FAIL: single-quoted hook findings should point at the hook command line, not line 1") + print("workspace audit checks passed") PY From 2d7e3928678a5216817f91fb95f34aac3496fa90 Mon Sep 17 00:00:00 2001 From: rbbtsn0w Date: Fri, 29 May 2026 11:19:21 +0800 Subject: [PATCH 5/7] fix(memorylint): harden apply and hook parsing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- memorylint/scripts/apply_report.py | 43 +++++- memorylint/scripts/memorylint_core.py | 69 ++++++--- memorylint/tests/test-apply-workflow.sh | 179 +++++++++++++++++++++++ memorylint/tests/test-workspace-audit.sh | 48 +++++- 4 files changed, 319 insertions(+), 20 deletions(-) diff --git a/memorylint/scripts/apply_report.py b/memorylint/scripts/apply_report.py index 593819b..6319259 100644 --- a/memorylint/scripts/apply_report.py +++ b/memorylint/scripts/apply_report.py @@ -45,6 +45,11 @@ def parse_args() -> argparse.Namespace: type=Path, help="Optional path to write manual handoff findings as JSON.", ) + parser.add_argument( + "--workspace", + type=Path, + help="Optional workspace root override. Defaults to the path stored in the report.", + ) return parser.parse_args() @@ -56,12 +61,39 @@ def grouped_edits(findings: list[dict[str, object]]) -> dict[str, list[dict[str, return grouped +def protected_edit_issues(findings: list[dict[str, object]]) -> list[str]: + issues: list[str] = [] + for finding in findings: + for edit in finding.get("edits", []): + if str(edit["path"]).endswith(".specify/memory/constitution.md"): + issues.append( + f"Finding {finding['id']} targets .specify/memory/constitution.md. Constitution edits must stay manual handoffs." + ) + return issues + + +def overlapping_edit_issues(edits_by_file: dict[str, list[dict[str, object]]]) -> list[str]: + issues: list[str] = [] + for relative, edits in edits_by_file.items(): + previous: tuple[int, int] | None = None + for edit in sorted(edits, key=lambda item: (int(item["start_line"]), int(item["end_line"]))): + start = int(edit["start_line"]) + end = int(edit["end_line"]) + if previous and start <= previous[1]: + issues.append( + f"Overlapping edits detected in {relative}: {previous[0]}-{previous[1]} conflicts with {start}-{end}." + ) + break + previous = (start, end) + return issues + + def main() -> int: args = parse_args() report_path = args.report.resolve() report_payload = extract_report_payload(report_path.read_text(encoding="utf-8")) report_copy = deep_copy_report(report_payload) - workspace_root = Path(report_copy["workspace_root"]).resolve() + workspace_root = args.workspace.resolve() if args.workspace else Path(report_copy["workspace_root"]).resolve() approved = approved_findings(report_copy, args.mode, set(args.approve)) manual_handoffs = [finding for finding in report_copy.get("findings", []) if finding.get("manual_handoff")] @@ -75,6 +107,11 @@ def main() -> int: print(format_apply_summary([], [], []), end="") return 0 + policy_issues = protected_edit_issues(approved) + if policy_issues: + print(format_apply_failure(policy_issues, []), end="") + return 1 + source_hashes = {item["path"]: item["sha256"] for item in report_copy.get("source_metadata", [])} target_paths = sorted({edit["path"] for finding in approved for edit in finding.get("edits", [])}) resolved_targets: dict[str, Path] = {} @@ -95,6 +132,10 @@ def main() -> int: originals = {relative: read_text(resolved_targets[relative]) for relative in target_paths} edits_by_file = grouped_edits(approved) + overlap_issues = overlapping_edit_issues(edits_by_file) + if overlap_issues: + print(format_apply_failure(overlap_issues, sorted(originals)), end="") + return 1 for relative, edits in edits_by_file.items(): target_path = resolved_targets[relative] updated = apply_edits_to_lines(read_text(target_path).splitlines(), edits) diff --git a/memorylint/scripts/memorylint_core.py b/memorylint/scripts/memorylint_core.py index 26371d9..8604e2d 100644 --- a/memorylint/scripts/memorylint_core.py +++ b/memorylint/scripts/memorylint_core.py @@ -7,6 +7,7 @@ import hashlib import json import re +import shlex import subprocess from collections import defaultdict from dataclasses import asdict, dataclass, field @@ -268,12 +269,26 @@ def parse_extension_hooks(text: str) -> dict[str, str]: if hook_name_match: current_hook = hook_name_match.group(1) continue - command_match = re.match(r'^\s{4}command:\s*["\']?([^"\']+)["\']?\s*$', line) - if current_hook and command_match: - hooks[current_hook] = command_match.group(1) + command_value = parse_hook_command_value(line) + if current_hook and command_value: + hooks[current_hook] = command_value return hooks +def parse_hook_command_value(line: str) -> str | None: + match = re.match(r"^\s{4}command:\s*(.+?)\s*$", line) + if not match: + return None + value = match.group(1).strip() + if value.startswith('"'): + quoted = re.match(r'^"([^"]+)"(?:\s+#.*)?$', value) + return quoted.group(1) if quoted else None + if value.startswith("'"): + quoted = re.match(r"^'([^']+)'(?:\s+#.*)?$", value) + return quoted.group(1) if quoted else None + return re.sub(r"\s+#.*$", "", value).strip() or None + + def find_hook_command_line(text: str, hook_name: str) -> int | None: current_hook: str | None = None for index, line in enumerate(text.splitlines(), start=1): @@ -283,7 +298,7 @@ def find_hook_command_line(text: str, hook_name: str) -> int | None: continue if current_hook != hook_name: continue - if re.match(r'^\s{4}command:\s*["\']?([^"\']+)["\']?\s*$', line): + if parse_hook_command_value(line): return index return None @@ -473,11 +488,25 @@ def find_line_number(text: str, snippet: str) -> int | None: def detect_path_references(rule: Rule) -> Iterable[str]: + def looks_like_path_reference(value: str) -> bool: + return value.startswith(("scripts/", "bin/", "tools/", "./", "../")) or value.endswith((".sh", ".py", ".rb", ".js", ".ts")) + for match in re.findall(r"`([^`]+)`", rule.text): if "://" in match or "{" in match: continue - if match.startswith(("scripts/", "bin/", "tools/")) or match.endswith((".sh", ".py", ".rb", ".js", ".ts")): - yield match + normalized = match.strip().rstrip(".,;:") + if " " not in normalized and looks_like_path_reference(normalized): + yield normalized + continue + try: + tokens = shlex.split(match) + except ValueError: + tokens = match.split() + for token in tokens[1:] if len(tokens) > 1 else tokens: + normalized_token = token.strip().rstrip(".,;:") + if looks_like_path_reference(normalized_token): + yield normalized_token + break def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Finding]: @@ -653,6 +682,21 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin continue replacement = MEMORYLINT_EXPECTED_HOOKS.get(hook_name, command_name) if extension_id == "memorylint" else command_name line_number = find_hook_command_line(manifest_text, hook_name) or 1 + edits: list[Edit] = [] + detail = f"Rewrite hook `{hook_name}` to use a declared command." + if replacement != command_name: + edits = [ + Edit( + path=manifest.source, + action="replace", + start_line=line_number, + end_line=line_number, + replacement=[re.sub(r'command:\s*(?:"[^"]+"|\'[^\']+\'|[^#\n]+)', f'command: "{replacement}"', manifest_text.splitlines()[line_number - 1])], + reason=f"Rewrite hook {hook_name} to the declared command {replacement}.", + ) + ] + else: + detail = f"Declare `{command_name}` under provides.commands or replace hook `{hook_name}` with a declared command." findings.append( Finding( id="", @@ -663,19 +707,10 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin evidence=f"{manifest.source} hook `{hook_name}` references `{command_name}`, but provides.commands declares {sorted(declared)}.", recommended_destination=manifest.source, suggested_action="rewrite", - detail=f"Rewrite hook `{hook_name}` to use a declared command.", + detail=detail, rule_ids=[manifest.rule_id], category="domain", - edits=[ - Edit( - path=manifest.source, - action="replace", - start_line=line_number, - end_line=line_number, - replacement=[re.sub(r'command:\s*["\']?[^"\']+["\']?', f'command: "{replacement}"', manifest_text.splitlines()[line_number - 1])], - reason=f"Rewrite hook {hook_name} to the declared command {replacement}.", - ) - ], + edits=edits, ) ) diff --git a/memorylint/tests/test-apply-workflow.sh b/memorylint/tests/test-apply-workflow.sh index 8840727..eb89a1e 100644 --- a/memorylint/tests/test-apply-workflow.sh +++ b/memorylint/tests/test-apply-workflow.sh @@ -172,6 +172,115 @@ if grep -q "Remove stale npm script reference" "$TMP_DIR/apply-approved/AGENTS.m exit 1 fi +mkdir -p "$TMP_DIR/workspace-override" +cat >"$TMP_DIR/workspace-override/AGENTS.md" <<'EOF' +# Workspace Rules + +## Commands + +- Override target +EOF +"$PYTHON_BIN" - "$TMP_DIR/workspace-override/override-report.json" "$TMP_DIR/workspace-override" <<'PY' +import hashlib +import json +import sys +from pathlib import Path + +report_path = Path(sys.argv[1]) +workspace = Path(sys.argv[2]).resolve() +agents_path = workspace / "AGENTS.md" +report = { + "schema_version": "1.0", + "workspace_root": "/tmp/elsewhere", + "source_metadata": [{ + "path": "AGENTS.md", + "sha256": hashlib.sha256(agents_path.read_bytes()).hexdigest(), + }], + "instruction_map": [], + "findings": [{ + "id": "ML-override", + "drift_type": "reality", + "severity": "warning", + "confidence": "high", + "source": "AGENTS.md:5", + "evidence": "workspace override", + "recommended_destination": "AGENTS.md", + "suggested_action": "delete", + "detail": "delete target", + "edits": [{"path": "AGENTS.md", "action": "delete", "start_line": 5, "end_line": 5, "reason": "override"}], + }], + "metrics": {}, + "summary": {}, +} +report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") +PY +"$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/workspace-override/override-report.json" --workspace "$TMP_DIR/workspace-override" --mode apply-all-approved --approve ML-override >/dev/null +if grep -q "Override target" "$TMP_DIR/workspace-override/AGENTS.md"; then + echo "FAIL: --workspace override should apply against the overridden workspace root" >&2 + exit 1 +fi + +mkdir -p "$TMP_DIR/constitution-edit/.specify/memory" +cat >"$TMP_DIR/constitution-edit/.specify/memory/constitution.md" <<'EOF' +# Constitution + +## Rules + +- Immutable rule +EOF +"$PYTHON_BIN" - "$TMP_DIR/constitution-edit/constitution-report.json" "$TMP_DIR/constitution-edit" <<'PY' +import hashlib +import json +import sys +from pathlib import Path + +report_path = Path(sys.argv[1]) +workspace = Path(sys.argv[2]).resolve() +constitution_path = workspace / ".specify/memory/constitution.md" +report = { + "schema_version": "1.0", + "workspace_root": str(workspace), + "source_metadata": [{ + "path": ".specify/memory/constitution.md", + "sha256": hashlib.sha256(constitution_path.read_bytes()).hexdigest(), + }], + "instruction_map": [], + "findings": [{ + "id": "ML-constitution", + "drift_type": "reality", + "severity": "warning", + "confidence": "high", + "source": ".specify/memory/constitution.md:5", + "evidence": "constitution edit", + "recommended_destination": ".specify/memory/constitution.md", + "suggested_action": "delete", + "detail": "should remain manual", + "edits": [{ + "path": ".specify/memory/constitution.md", + "action": "delete", + "start_line": 5, + "end_line": 5, + "reason": "constitution", + }], + }], + "metrics": {}, + "summary": {}, +} +report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") +PY +if "$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/constitution-edit/constitution-report.json" --mode apply-all-approved --approve ML-constitution >"$TMP_DIR/constitution-edit/output.txt" 2>&1; then + echo "FAIL: apply should reject constitution edit targets" >&2 + exit 1 +fi +grep -q "Constitution edits must stay manual handoffs" "$TMP_DIR/constitution-edit/output.txt" || { + echo "FAIL: constitution protection failure output missing" >&2 + exit 1 +} +grep -q "Immutable rule" "$TMP_DIR/constitution-edit/.specify/memory/constitution.md" || { + echo "FAIL: constitution file should remain unchanged" >&2 + exit 1 +} + mkdir -p "$TMP_DIR/git-diff-check" cat >"$TMP_DIR/git-diff-check/AGENTS.md" <<'EOF' # Workspace Rules @@ -242,4 +351,74 @@ grep -q "Clean command" "$TMP_DIR/git-diff-check/AGENTS.md" || { exit 1 } +mkdir -p "$TMP_DIR/overlap-check" +cat >"$TMP_DIR/overlap-check/AGENTS.md" <<'EOF' +# Workspace Rules + +## Commands + +- First rule +- Second rule +EOF +"$PYTHON_BIN" - "$TMP_DIR/overlap-check/overlap-report.json" "$TMP_DIR/overlap-check" <<'PY' +import hashlib +import json +import sys +from pathlib import Path + +report_path = Path(sys.argv[1]) +workspace = Path(sys.argv[2]).resolve() +agents_path = workspace / "AGENTS.md" +report = { + "schema_version": "1.0", + "workspace_root": str(workspace), + "source_metadata": [{ + "path": "AGENTS.md", + "sha256": hashlib.sha256(agents_path.read_bytes()).hexdigest(), + }], + "instruction_map": [], + "findings": [ + { + "id": "ML-overlap-a", + "drift_type": "reality", + "severity": "warning", + "confidence": "high", + "source": "AGENTS.md:5", + "evidence": "overlap a", + "recommended_destination": "AGENTS.md", + "suggested_action": "delete", + "detail": "first overlap", + "edits": [{"path": "AGENTS.md", "action": "delete", "start_line": 5, "end_line": 5, "reason": "overlap a"}], + }, + { + "id": "ML-overlap-b", + "drift_type": "reality", + "severity": "warning", + "confidence": "high", + "source": "AGENTS.md:5", + "evidence": "overlap b", + "recommended_destination": "AGENTS.md", + "suggested_action": "delete", + "detail": "second overlap", + "edits": [{"path": "AGENTS.md", "action": "delete", "start_line": 5, "end_line": 5, "reason": "overlap b"}], + }, + ], + "metrics": {}, + "summary": {}, +} +report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") +PY +if "$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/overlap-check/overlap-report.json" --mode apply-all-approved --approve ML-overlap-a --approve ML-overlap-b >"$TMP_DIR/overlap-check/output.txt" 2>&1; then + echo "FAIL: apply should reject overlapping edits" >&2 + exit 1 +fi +grep -q "Overlapping edits detected" "$TMP_DIR/overlap-check/output.txt" || { + echo "FAIL: overlapping edit failure output missing" >&2 + exit 1 +} +grep -q "First rule" "$TMP_DIR/overlap-check/AGENTS.md" || { + echo "FAIL: overlapping edit rejection should leave source unchanged" >&2 + exit 1 +} + echo "apply workflow checks passed" diff --git a/memorylint/tests/test-workspace-audit.sh b/memorylint/tests/test-workspace-audit.sh index a1ff417..458a6ca 100644 --- a/memorylint/tests/test-workspace-audit.sh +++ b/memorylint/tests/test-workspace-audit.sh @@ -37,7 +37,7 @@ cat >"$TMP_DIR/nested-ref/packages/frontend/AGENTS.md" <<'EOF' ## Commands -- Run `scripts/build.sh` before release. +- Run `bash scripts/build.sh` before release. EOF printf '#!/usr/bin/env bash\n' >"$TMP_DIR/nested-ref/packages/frontend/scripts/build.sh" "$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/nested-ref" --json-out "$TMP_DIR/nested.json" >/dev/null @@ -71,8 +71,40 @@ provides: description: "Audit" EOF "$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/single-quote-hooks" --json-out "$TMP_DIR/single-quote.json" >/dev/null +mkdir -p "$TMP_DIR/comment-hook" +cat >"$TMP_DIR/comment-hook/extension.yml" <<'EOF' +schema_version: "1.0" +extension: + id: memorylint + version: "0.1.0" + description: "Fixture for commented hook commands." +hooks: + before_plan: + command: speckit.memorylint.load-agents # planning gate +provides: + commands: + - name: speckit.memorylint.load-agents + description: "Load agents" +EOF +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/comment-hook" --json-out "$TMP_DIR/comment-hook.json" >/dev/null +mkdir -p "$TMP_DIR/other-extension-hook" +cat >"$TMP_DIR/other-extension-hook/extension.yml" <<'EOF' +schema_version: "1.0" +extension: + id: demo-extension + version: "0.1.0" + description: "Fixture for external hook remediation." +hooks: + before_plan: + command: speckit.demo.audit +provides: + commands: + - name: speckit.demo.other + description: "Other command" +EOF +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/other-extension-hook" --json-out "$TMP_DIR/other-extension.json" >/dev/null -"$PYTHON_BIN" - "$TMP_DIR/clean.json" "$TMP_DIR/bloated.json" "$TMP_DIR/escape.json" "$TMP_DIR/nested.json" "$TMP_DIR/invalid-package.json" "$TMP_DIR/single-quote.json" <<'PY' +"$PYTHON_BIN" - "$TMP_DIR/clean.json" "$TMP_DIR/bloated.json" "$TMP_DIR/escape.json" "$TMP_DIR/nested.json" "$TMP_DIR/invalid-package.json" "$TMP_DIR/single-quote.json" "$TMP_DIR/comment-hook.json" "$TMP_DIR/other-extension.json" <<'PY' import json import sys from pathlib import Path @@ -83,6 +115,8 @@ escaped = json.loads(Path(sys.argv[3]).read_text(encoding="utf-8")) nested = json.loads(Path(sys.argv[4]).read_text(encoding="utf-8")) invalid_package = json.loads(Path(sys.argv[5]).read_text(encoding="utf-8")) single_quote = json.loads(Path(sys.argv[6]).read_text(encoding="utf-8")) +comment_hook = json.loads(Path(sys.argv[7]).read_text(encoding="utf-8")) +other_extension = json.loads(Path(sys.argv[8]).read_text(encoding="utf-8")) if clean["metrics"]["total_findings"] != 0: raise SystemExit("FAIL: clean-repo workspace audit should produce zero findings") @@ -117,5 +151,15 @@ if len(hook_findings) != 2: if any(finding["source"].endswith(":1") for finding in hook_findings): raise SystemExit("FAIL: single-quoted hook findings should point at the hook command line, not line 1") +comment_hook_findings = [finding for finding in comment_hook["findings"] if "hook `" in finding.get("evidence", "")] +if comment_hook_findings: + raise SystemExit("FAIL: inline YAML comments should not become part of hook command names") + +external_hook_findings = [finding for finding in other_extension["findings"] if "hook `" in finding.get("evidence", "")] +if len(external_hook_findings) != 1: + raise SystemExit(f"FAIL: external hook fixture should produce exactly 1 hook finding, got {len(external_hook_findings)}") +if external_hook_findings[0].get("edits"): + raise SystemExit("FAIL: non-memorylint hook findings without a known replacement should not emit no-op edits") + print("workspace audit checks passed") PY From db0dad10aa212e82d3ce3be14db87d9840637660 Mon Sep 17 00:00:00 2001 From: rbbtsn0w Date: Fri, 29 May 2026 11:24:44 +0800 Subject: [PATCH 6/7] fix(memorylint): preserve hook comment rewrites Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- memorylint/scripts/memorylint_core.py | 18 +++++++++++++- memorylint/tests/test-apply-workflow.sh | 33 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/memorylint/scripts/memorylint_core.py b/memorylint/scripts/memorylint_core.py index 8604e2d..d9a0e40 100644 --- a/memorylint/scripts/memorylint_core.py +++ b/memorylint/scripts/memorylint_core.py @@ -303,6 +303,22 @@ def find_hook_command_line(text: str, hook_name: str) -> int | None: return None +def rewrite_hook_command_line(line: str, replacement: str) -> str: + prefix_match = re.match(r"^(\s*command:\s*)(.+?)\s*$", line) + if not prefix_match: + return line + + original_value = prefix_match.group(2).strip() + if original_value.startswith('"'): + comment_match = re.match(r'^"[^"]+"(\s+#.*)?$', original_value) + elif original_value.startswith("'"): + comment_match = re.match(r"^'[^']+'(\s+#.*)?$", original_value) + else: + comment_match = re.match(r"^[^#\n]+?(\s+#.*)?$", original_value) + comment = comment_match.group(1) if comment_match and comment_match.group(1) else "" + return f'{prefix_match.group(1)}"{replacement}"{comment}' + + def manifest_rules(path: Path, workspace_root: Path, next_rule_id: int) -> tuple[list[Rule], int]: text = read_text(path) rules: list[Rule] = [] @@ -691,7 +707,7 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin action="replace", start_line=line_number, end_line=line_number, - replacement=[re.sub(r'command:\s*(?:"[^"]+"|\'[^\']+\'|[^#\n]+)', f'command: "{replacement}"', manifest_text.splitlines()[line_number - 1])], + replacement=[rewrite_hook_command_line(manifest_text.splitlines()[line_number - 1], replacement)], reason=f"Rewrite hook {hook_name} to the declared command {replacement}.", ) ] diff --git a/memorylint/tests/test-apply-workflow.sh b/memorylint/tests/test-apply-workflow.sh index eb89a1e..7f393b8 100644 --- a/memorylint/tests/test-apply-workflow.sh +++ b/memorylint/tests/test-apply-workflow.sh @@ -421,4 +421,37 @@ grep -q "First rule" "$TMP_DIR/overlap-check/AGENTS.md" || { exit 1 } +mkdir -p "$TMP_DIR/hook-comment-rewrite" +cat >"$TMP_DIR/hook-comment-rewrite/extension.yml" <<'EOF' +schema_version: "1.0" +extension: + id: memorylint + version: "0.1.0" + description: "Fixture for hook rewrite comments." +hooks: + before_plan: + command: speckit.memorylint.run # planning gate +provides: + commands: + - name: speckit.memorylint.load-agents + description: "Load agents" +EOF +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/hook-comment-rewrite" --json-out "$TMP_DIR/hook-comment-rewrite/report.json" >/dev/null +"$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/hook-comment-rewrite/report.json" --mode apply-all-approved --approve ML-001 >/dev/null +grep -q 'command: "speckit.memorylint.load-agents" # planning gate' "$TMP_DIR/hook-comment-rewrite/extension.yml" || { + echo "FAIL: hook rewrite should preserve spacing before inline comments" >&2 + exit 1 +} +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/hook-comment-rewrite" --json-out "$TMP_DIR/hook-comment-rewrite/recheck.json" >/dev/null +"$PYTHON_BIN" - "$TMP_DIR/hook-comment-rewrite/recheck.json" <<'PY' +import json +import sys +from pathlib import Path + +report = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +hook_findings = [finding for finding in report["findings"] if "hook `" in finding.get("evidence", "")] +if hook_findings: + raise SystemExit("FAIL: rewritten hook command with inline comment should remain parseable") +PY + echo "apply workflow checks passed" From 38ee88a5701bc86787ea750570e892826a56f052 Mon Sep 17 00:00:00 2001 From: rbbtsn0w Date: Fri, 29 May 2026 11:31:42 +0800 Subject: [PATCH 7/7] fix(memorylint): tighten rule parsing and apply validation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- memorylint/scripts/memorylint_core.py | 38 +++++++++++++++---- memorylint/tests/test-apply-workflow.sh | 47 ++++++++++++++++++++++++ memorylint/tests/test-workspace-audit.sh | 39 +++++++++++++++++++- 3 files changed, 116 insertions(+), 8 deletions(-) diff --git a/memorylint/scripts/memorylint_core.py b/memorylint/scripts/memorylint_core.py index d9a0e40..70290d0 100644 --- a/memorylint/scripts/memorylint_core.py +++ b/memorylint/scripts/memorylint_core.py @@ -251,12 +251,26 @@ def parse_extension_commands(text: str) -> set[str]: if in_commands and re.match(r"^\s*[a-z_]+:\s*$", line): break if in_commands: - command_match = re.match(r'^\s*-\s+name:\s*["\']?([^"\']+)["\']?\s*$', line) - if command_match: - commands.add(command_match.group(1)) + command_value = parse_declared_command_value(line) + if command_value: + commands.add(command_value) return commands +def parse_declared_command_value(line: str) -> str | None: + match = re.match(r"^\s*-\s+name:\s*(.+?)\s*$", line) + if not match: + return None + value = match.group(1).strip() + if value.startswith('"'): + quoted = re.match(r'^"([^"]+)"(?:\s+#.*)?$', value) + return quoted.group(1) if quoted else None + if value.startswith("'"): + quoted = re.match(r"^'([^']+)'(?:\s+#.*)?$", value) + return quoted.group(1) if quoted else None + return re.sub(r"\s+#.*$", "", value).strip() or None + + def parse_extension_hooks(text: str) -> dict[str, str]: hooks: dict[str, str] = {} hook_match = re.search(r"^\s*hooks:\s*$", text, re.MULTILINE) @@ -529,6 +543,11 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin findings: list[Finding] = [] seen_sources: set[tuple[str, str]] = set() + def needs_manual_stale_path_rewrite(rule_text: str, reference: str) -> bool: + lowered = rule_text.lower() + stripped = lowered.replace(f"`{reference.lower()}`", "") + return any(token in stripped for token in (",", ";", " and ", " then ", " while ")) + for rule in rules: rule_path = workspace_root / rule.source rule_text_normalized = normalize_rule_text(rule.text) @@ -571,6 +590,7 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin if key in seen_sources: continue seen_sources.add(key) + manual_rewrite = needs_manual_stale_path_rewrite(rule.text, reference) findings.append( Finding( id="", @@ -580,11 +600,15 @@ def detect_reality_findings(workspace_root: Path, rules: list[Rule]) -> list[Fin source=f"{rule.source}:{rule.line_range}", evidence=f"{rule.source} references `{reference}` but that path does not exist in the workspace.", recommended_destination="N/A", - suggested_action="delete", - detail=f"Remove or replace the stale reference to `{reference}`.", + suggested_action="rewrite" if manual_rewrite else "delete", + detail=( + f"Rewrite the mixed-content rule in {rule.source} to remove the stale reference `{reference}` without dropping the remaining guidance." + if manual_rewrite + else f"Remove or replace the stale reference to `{reference}`." + ), rule_ids=[rule.rule_id], category=rule.category, - edits=[ + edits=[] if manual_rewrite else [ Edit( path=rule.source, action="delete", @@ -1155,7 +1179,7 @@ def validate_agents_integrity(before_text: str, after_text: str) -> list[str]: issues.append(f"Missing AGENTS.md critical section family after apply: {family[0]}") has_heading_above = False for index, line in enumerate(after_text.splitlines(), start=1): - if re.match(r"^\s{0,3}##\s+", line): + if re.match(r"^\s{0,3}#{1,2}\s+", line): has_heading_above = True continue if re.match(r"^\s*[-*]\s+", line): diff --git a/memorylint/tests/test-apply-workflow.sh b/memorylint/tests/test-apply-workflow.sh index 7f393b8..9e09414 100644 --- a/memorylint/tests/test-apply-workflow.sh +++ b/memorylint/tests/test-apply-workflow.sh @@ -454,4 +454,51 @@ if hook_findings: raise SystemExit("FAIL: rewritten hook command with inline comment should remain parseable") PY +mkdir -p "$TMP_DIR/top-level-heading" +cat >"$TMP_DIR/top-level-heading/AGENTS.md" <<'EOF' +# Workspace Rules + +- Keep this bullet. +- Remove this bullet. +EOF +"$PYTHON_BIN" - "$TMP_DIR/top-level-heading/top-level-report.json" "$TMP_DIR/top-level-heading" <<'PY' +import hashlib +import json +import sys +from pathlib import Path + +report_path = Path(sys.argv[1]) +workspace = Path(sys.argv[2]).resolve() +agents_path = workspace / "AGENTS.md" +report = { + "schema_version": "1.0", + "workspace_root": str(workspace), + "source_metadata": [{ + "path": "AGENTS.md", + "sha256": hashlib.sha256(agents_path.read_bytes()).hexdigest(), + }], + "instruction_map": [], + "findings": [{ + "id": "ML-top-level", + "drift_type": "reality", + "severity": "warning", + "confidence": "high", + "source": "AGENTS.md:4", + "evidence": "top-level heading", + "recommended_destination": "AGENTS.md", + "suggested_action": "delete", + "detail": "delete second bullet", + "edits": [{"path": "AGENTS.md", "action": "delete", "start_line": 4, "end_line": 4, "reason": "top-level"}], + }], + "metrics": {}, + "summary": {}, +} +report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") +PY +"$PYTHON_BIN" "$APPLY_SCRIPT" "$TMP_DIR/top-level-heading/top-level-report.json" --mode apply-all-approved --approve ML-top-level >/dev/null +grep -q "Keep this bullet." "$TMP_DIR/top-level-heading/AGENTS.md" || { + echo "FAIL: bullets under a top-level heading should remain valid after apply" >&2 + exit 1 +} + echo "apply workflow checks passed" diff --git a/memorylint/tests/test-workspace-audit.sh b/memorylint/tests/test-workspace-audit.sh index 458a6ca..fe8af57 100644 --- a/memorylint/tests/test-workspace-audit.sh +++ b/memorylint/tests/test-workspace-audit.sh @@ -103,8 +103,33 @@ provides: description: "Other command" EOF "$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/other-extension-hook" --json-out "$TMP_DIR/other-extension.json" >/dev/null +mkdir -p "$TMP_DIR/commented-command" +cat >"$TMP_DIR/commented-command/extension.yml" <<'EOF' +schema_version: "1.0" +extension: + id: demo-extension + version: "0.1.0" + description: "Fixture for commented command declarations." +hooks: + before_plan: + command: speckit.demo.audit +provides: + commands: + - name: speckit.demo.audit # primary command + description: "Audit" +EOF +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/commented-command" --json-out "$TMP_DIR/commented-command.json" >/dev/null +mkdir -p "$TMP_DIR/mixed-content" +cat >"$TMP_DIR/mixed-content/AGENTS.md" <<'EOF' +# Workspace Rules + +## Commands -"$PYTHON_BIN" - "$TMP_DIR/clean.json" "$TMP_DIR/bloated.json" "$TMP_DIR/escape.json" "$TMP_DIR/nested.json" "$TMP_DIR/invalid-package.json" "$TMP_DIR/single-quote.json" "$TMP_DIR/comment-hook.json" "$TMP_DIR/other-extension.json" <<'PY' +- Run `make test`, then remove old helper `scripts/old.sh`. +EOF +"$PYTHON_BIN" "$AUDIT_SCRIPT" "$TMP_DIR/mixed-content" --json-out "$TMP_DIR/mixed-content.json" >/dev/null + +"$PYTHON_BIN" - "$TMP_DIR/clean.json" "$TMP_DIR/bloated.json" "$TMP_DIR/escape.json" "$TMP_DIR/nested.json" "$TMP_DIR/invalid-package.json" "$TMP_DIR/single-quote.json" "$TMP_DIR/comment-hook.json" "$TMP_DIR/other-extension.json" "$TMP_DIR/commented-command.json" "$TMP_DIR/mixed-content.json" <<'PY' import json import sys from pathlib import Path @@ -117,6 +142,8 @@ invalid_package = json.loads(Path(sys.argv[5]).read_text(encoding="utf-8")) single_quote = json.loads(Path(sys.argv[6]).read_text(encoding="utf-8")) comment_hook = json.loads(Path(sys.argv[7]).read_text(encoding="utf-8")) other_extension = json.loads(Path(sys.argv[8]).read_text(encoding="utf-8")) +commented_command = json.loads(Path(sys.argv[9]).read_text(encoding="utf-8")) +mixed_content = json.loads(Path(sys.argv[10]).read_text(encoding="utf-8")) if clean["metrics"]["total_findings"] != 0: raise SystemExit("FAIL: clean-repo workspace audit should produce zero findings") @@ -161,5 +188,15 @@ if len(external_hook_findings) != 1: if external_hook_findings[0].get("edits"): raise SystemExit("FAIL: non-memorylint hook findings without a known replacement should not emit no-op edits") +commented_command_findings = [finding for finding in commented_command["findings"] if "hook `" in finding.get("evidence", "")] +if commented_command_findings: + raise SystemExit("FAIL: inline comments in provides.commands should not break hook declaration matching") + +mixed_content_findings = [finding for finding in mixed_content["findings"] if "scripts/old.sh" in finding.get("evidence", "")] +if len(mixed_content_findings) != 1: + raise SystemExit(f"FAIL: mixed-content fixture should produce exactly 1 stale-path finding, got {len(mixed_content_findings)}") +if mixed_content_findings[0].get("edits"): + raise SystemExit("FAIL: mixed-content stale-path findings should not auto-delete the entire rule line") + print("workspace audit checks passed") PY