From e9e5a480b8d8632fd5ef73453daa645f37862773 Mon Sep 17 00:00:00 2001 From: Alex <45095641+runsonmypc@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:05:20 -0400 Subject: [PATCH 01/14] fix(verify): do not report unverified dimensions as passing Step 5 gates task and spec coverage on `contextFiles.tasks` and `contextFiles.specs`. `contextFiles` is an artifact-id map and artifact ids come from the active schema, so on a schema that defines neither, both branches are no-ops: nothing is checked, no issues are raised, and step 8 concludes "All checks passed. Ready for archive." The Graceful Degradation guardrail already asks the agent to note skipped checks, but nothing stopped the all-clear verdict. Mark an unchecked dimension `Not verified` in the scorecard and require the final assessment to name it. --- .changeset/verify-report-skipped-dimensions.md | 5 +++++ skills/openspec-verify-change/SKILL.md | 7 ++++++- src/core/templates/workflows/verify-change.ts | 14 ++++++++++++-- test/core/templates/skill-templates-parity.test.ts | 6 +++--- 4 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 .changeset/verify-report-skipped-dimensions.md diff --git a/.changeset/verify-report-skipped-dimensions.md b/.changeset/verify-report-skipped-dimensions.md new file mode 100644 index 0000000000..70dd472d40 --- /dev/null +++ b/.changeset/verify-report-skipped-dimensions.md @@ -0,0 +1,5 @@ +--- +"@fission-ai/openspec": patch +--- + +Stop `/opsx:verify` from reporting "All checks passed" for a dimension it never checked. `contextFiles` is keyed by artifact id, so a schema without a `tasks` or `specs` artifact makes those branches no-ops; the scorecard and final assessment now mark such a dimension `Not verified` instead of clean. diff --git a/skills/openspec-verify-change/SKILL.md b/skills/openspec-verify-change/SKILL.md index 2165a6a910..fd29b83732 100644 --- a/skills/openspec-verify-change/SKILL.md +++ b/skills/openspec-verify-change/SKILL.md @@ -59,6 +59,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design 5. **Verify Completeness** **Task Completion**: + - If `contextFiles` has no `tasks` entry, this dimension is **not verified** - record it as skipped, same as above - If `contextFiles.tasks` exists, read every file path in it - Parse checkboxes: `- [ ]` (incomplete) vs `- [x]` (complete) - Count complete vs total tasks @@ -67,6 +68,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: + - `contextFiles` is keyed by artifact id, and artifact ids come from the active schema. If it has no `specs` entry, or the entry is empty, this dimension is **not verified** - record it as skipped and carry that through to the report rather than treating it as clean. - If delta specs exist in `contextFiles.specs`: - Extract all requirements (marked with "### Requirement:") - For each requirement: @@ -127,6 +129,8 @@ Verify that an implementation matches the change artifacts (specs, tasks, design | Coherence | Followed/Issues | ``` + Write `Not verified ()` in the Status cell of any dimension whose artifacts were absent. Never leave it blank or score it as passing. + **Issues by Priority**: 1. **CRITICAL** (Must fix before archive): @@ -147,7 +151,8 @@ Verify that an implementation matches the change artifacts (specs, tasks, design **Final Assessment**: - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." - If only warnings: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." - - If all clear: "All checks passed. Ready for archive." + - If all clear and every dimension was verified: "All checks passed. Ready for archive." + - If all clear but a dimension was skipped: say so - "No issues found in the checks that ran. not verified: ." A dimension that checked nothing has not passed. **Verification Heuristics** diff --git a/src/core/templates/workflows/verify-change.ts b/src/core/templates/workflows/verify-change.ts index 1aa540c76b..ff2b0f3664 100644 --- a/src/core/templates/workflows/verify-change.ts +++ b/src/core/templates/workflows/verify-change.ts @@ -61,6 +61,7 @@ ${STORE_SELECTION_GUIDANCE} 5. **Verify Completeness** **Task Completion**: + - If \`contextFiles\` has no \`tasks\` entry, this dimension is **not verified** - record it as skipped, same as above - If \`contextFiles.tasks\` exists, read every file path in it - Parse checkboxes: \`- [ ]\` (incomplete) vs \`- [x]\` (complete) - Count complete vs total tasks @@ -69,6 +70,7 @@ ${STORE_SELECTION_GUIDANCE} - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: + - \`contextFiles\` is keyed by artifact id, and artifact ids come from the active schema. If it has no \`specs\` entry, or the entry is empty, this dimension is **not verified** - record it as skipped and carry that through to the report rather than treating it as clean. - If delta specs exist in \`contextFiles.specs\`: - Extract all requirements (marked with "### Requirement:") - For each requirement: @@ -129,6 +131,8 @@ ${STORE_SELECTION_GUIDANCE} | Coherence | Followed/Issues | \`\`\` + Write \`Not verified ()\` in the Status cell of any dimension whose artifacts were absent. Never leave it blank or score it as passing. + **Issues by Priority**: 1. **CRITICAL** (Must fix before archive): @@ -149,7 +153,8 @@ ${STORE_SELECTION_GUIDANCE} **Final Assessment**: - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." - If only warnings: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." - - If all clear: "All checks passed. Ready for archive." + - If all clear and every dimension was verified: "All checks passed. Ready for archive." + - If all clear but a dimension was skipped: say so - "No issues found in the checks that ran. not verified: ." A dimension that checked nothing has not passed. **Verification Heuristics** @@ -236,6 +241,7 @@ ${STORE_SELECTION_GUIDANCE} 5. **Verify Completeness** **Task Completion**: + - If \`contextFiles\` has no \`tasks\` entry, this dimension is **not verified** - record it as skipped, same as above - If \`contextFiles.tasks\` exists, read every file path in it - Parse checkboxes: \`- [ ]\` (incomplete) vs \`- [x]\` (complete) - Count complete vs total tasks @@ -244,6 +250,7 @@ ${STORE_SELECTION_GUIDANCE} - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: + - \`contextFiles\` is keyed by artifact id, and artifact ids come from the active schema. If it has no \`specs\` entry, or the entry is empty, this dimension is **not verified** - record it as skipped and carry that through to the report rather than treating it as clean. - If delta specs exist in \`contextFiles.specs\`: - Extract all requirements (marked with "### Requirement:") - For each requirement: @@ -304,6 +311,8 @@ ${STORE_SELECTION_GUIDANCE} | Coherence | Followed/Issues | \`\`\` + Write \`Not verified ()\` in the Status cell of any dimension whose artifacts were absent. Never leave it blank or score it as passing. + **Issues by Priority**: 1. **CRITICAL** (Must fix before archive): @@ -324,7 +333,8 @@ ${STORE_SELECTION_GUIDANCE} **Final Assessment**: - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." - If only warnings: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." - - If all clear: "All checks passed. Ready for archive." + - If all clear and every dimension was verified: "All checks passed. Ready for archive." + - If all clear but a dimension was skipped: say so - "No issues found in the checks that ran. not verified: ." A dimension that checked nothing has not passed. **Verification Heuristics** diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index c7a8b44b5c..694cdcf568 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -53,11 +53,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '56bfada1a5f35a127791b70de9d428a75b5aedd1584d6c9803a1ecb1fd1b4a23', getBulkArchiveChangeSkillTemplate: '93875998cade5322d95b43299fba794bc1da754e917dd63a770406386a6d295d', getOpsxSyncCommandTemplate: '0d2427efb79986e8fff3f96bd075a739c80d45eb29159fae717e950030da8202', - getVerifyChangeSkillTemplate: '223b7ffd99299a7d430e13092b9a0a3421b39f0d3217232f46c39d79b5f619ff', + getVerifyChangeSkillTemplate: 'b1d4f57ea164d69409bf0137b051d27f9ff8255967af01e3f1dbd5bbeeb7b0f2', getOpsxArchiveCommandTemplate: '9f973c819b11620985b03322945f0e0a92a02a2ef455b94e74482f5e6292ac5d', getOpsxOnboardCommandTemplate: 'ee99aa99252c602720fbb8c63fb3ac438a5bd4e952fd961ddf1ae956cbfc2c8f', getOpsxBulkArchiveCommandTemplate: '9fa8cdebe2f5667ebfc37bdc023396762c59d5b038c771dac2d8fd2c19e2627b', - getOpsxVerifyCommandTemplate: '1efcf7eff0671f48e9d9420f50865c563dd3079ee60f8c380bb7a90dd0102696', + getOpsxVerifyCommandTemplate: 'ff2c701634454c96b6b5e66dbef499f7000d456c30e48a8534e445342a797ff7', getOpsxProposeSkillTemplate: '24623c066f97e34b957d448d1f9a9e8b8a13da3dfce45d45671f6226a2534848', getOpsxProposeCommandTemplate: 'e67ba591efb0fecacb2229d06dfa84af18b825fab8a7b01377279e4f09a06ce4', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -74,7 +74,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': 'd933d8856584d6c1253de91e652e7aee9e85c77ad4d3531f6476f79d84e6e5e8', 'openspec-archive-change': '7c65053d674ba4e1e20e2bf73ba7e5a7f94baef2eaa9b33cee48d4cadea51b7a', 'openspec-bulk-archive-change': '2039b9ecf6e64339dffe0e16272507a386d9fe326f419ff758315aa736fdd96c', - 'openspec-verify-change': 'af9be013dcbe8c6d8f6d9ab10c893fbd03f4c62933c384d82f63894dd0ceb84f', + 'openspec-verify-change': '7d9a26a1cffe55d0d9d0d5116063dc35d0d9ac232e5e0d67ed1548e2512ede46', 'openspec-onboard': 'f6f59476acaf5e4d65dbb180da4cef62432612f3cecf207d471a951295e2003a', 'openspec-propose': '25d08ed4f031770cea219604167d76bca9f3e89fe0c2f545263674482c6f13f0', 'openspec-update-change': '586547406aca94422dfeb3ffedce6c01049429b743f57ce829baa79ebc714d51', From cf41bc85e676f4983d90231aeca9ad3a2834ff6a Mon Sep 17 00:00:00 2001 From: Alex <45095641+runsonmypc@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:07:52 -0400 Subject: [PATCH 02/14] fix(verify): map skipped checks to report outcomes --- .../verify-report-skipped-dimensions.md | 2 +- skills/openspec-verify-change/SKILL.md | 28 ++++------ src/core/templates/workflows/verify-change.ts | 56 +++++++------------ .../templates/skill-templates-parity.test.ts | 6 +- test/core/templates/verify-change.test.ts | 46 +++++++++++++++ 5 files changed, 80 insertions(+), 58 deletions(-) create mode 100644 test/core/templates/verify-change.test.ts diff --git a/.changeset/verify-report-skipped-dimensions.md b/.changeset/verify-report-skipped-dimensions.md index 70dd472d40..24d60d9ea8 100644 --- a/.changeset/verify-report-skipped-dimensions.md +++ b/.changeset/verify-report-skipped-dimensions.md @@ -2,4 +2,4 @@ "@fission-ai/openspec": patch --- -Stop `/opsx:verify` from reporting "All checks passed" for a dimension it never checked. `contextFiles` is keyed by artifact id, so a schema without a `tasks` or `specs` artifact makes those branches no-ops; the scorecard and final assessment now mark such a dimension `Not verified` instead of clean. +Stop `/opsx:verify` from reporting skipped checks as passing. Task completion now uses the schema-aware `tasks` and `progress` fields returned by apply instructions, while absent spec or design inputs are mapped to every check they prevent. The scorecard identifies each skipped check, and the final assessment does not claim archive readiness when any check did not run. diff --git a/skills/openspec-verify-change/SKILL.md b/skills/openspec-verify-change/SKILL.md index fd29b83732..16e0fa737a 100644 --- a/skills/openspec-verify-change/SKILL.md +++ b/skills/openspec-verify-change/SKILL.md @@ -45,7 +45,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design openspec instructions apply --change "" --json ``` - This returns the change directory and `contextFiles` (artifact ID -> array of concrete file paths). Read all available artifacts from `contextFiles`. + This returns the change directory, `contextFiles` (artifact ID -> array of concrete file paths), and top-level `tasks` and `progress` resolved from the schema's `apply.tracks` configuration. Read all available artifacts from `contextFiles`. 4. **Initialize verification report structure** @@ -59,16 +59,15 @@ Verify that an implementation matches the change artifacts (specs, tasks, design 5. **Verify Completeness** **Task Completion**: - - If `contextFiles` has no `tasks` entry, this dimension is **not verified** - record it as skipped, same as above - - If `contextFiles.tasks` exists, read every file path in it - - Parse checkboxes: `- [ ]` (incomplete) vs `- [x]` (complete) - - Count complete vs total tasks + - Use the top-level `tasks` and `progress` fields; do not look for a `contextFiles.tasks` artifact id. + - If `tasks` is empty and `progress.total` is 0, no tracked tasks can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply `state` and `instruction`. + - Report complete vs total tasks from `progress`. - If incomplete tasks exist: - Add CRITICAL issue for each incomplete task - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: - - `contextFiles` is keyed by artifact id, and artifact ids come from the active schema. If it has no `specs` entry, or the entry is empty, this dimension is **not verified** - record it as skipped and carry that through to the report rather than treating it as clean. + - `contextFiles` is keyed by artifact id, and artifact ids come from the active schema. If `contextFiles.specs` is absent or empty, mark **Spec Coverage**, **Requirement Implementation Mapping**, and **Scenario Coverage** as not verified; do not treat any of them as clean. - If delta specs exist in `contextFiles.specs`: - Extract all requirements (marked with "### Requirement:") - For each requirement: @@ -106,7 +105,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design - If contradiction detected: - Add WARNING: "Design decision not followed: " - Recommendation: "Update implementation or revise design.md to match reality" - - If no design.md: Skip design adherence check, note "No design.md to verify against" + - If `contextFiles.design` is absent or empty: mark **Design Adherence** as not verified. **Code Pattern Consistency** still runs. **Code Pattern Consistency**: - Review new code for consistency with project patterns @@ -129,7 +128,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design | Coherence | Followed/Issues | ``` - Write `Not verified ()` in the Status cell of any dimension whose artifacts were absent. Never leave it blank or score it as passing. + In each Status cell, report the results of checks that ran and `Not verified ()` for every skipped check. If all checks in a dimension were skipped, start the cell with `Not verified`. Never score a skipped check as passing. **Issues by Priority**: @@ -150,9 +149,9 @@ Verify that an implementation matches the change artifacts (specs, tasks, design **Final Assessment**: - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." - - If only warnings: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." - - If all clear and every dimension was verified: "All checks passed. Ready for archive." - - If all clear but a dimension was skipped: say so - "No issues found in the checks that ran. not verified: ." A dimension that checked nothing has not passed. + - If only warnings and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." + - If no issues and no checks were skipped: "All checks passed. Ready for archive." + - If any check was skipped and there are no CRITICAL issues: do not claim readiness. Say "No critical issues found in the checks that ran. not verified: ." Include the warning count when nonzero. **Verification Heuristics** @@ -162,13 +161,6 @@ Verify that an implementation matches the change artifacts (specs, tasks, design - **False Positives**: When uncertain, prefer SUGGESTION over WARNING, WARNING over CRITICAL - **Actionability**: Every issue must have a specific recommendation with file/line references where applicable -**Graceful Degradation** - -- If only tasks.md exists: verify task completion only, skip spec/design checks -- If tasks + specs exist: verify completeness and correctness, skip design -- If full artifacts: verify all three dimensions -- Always note which checks were skipped and why - **Output Format** Use clear markdown with: diff --git a/src/core/templates/workflows/verify-change.ts b/src/core/templates/workflows/verify-change.ts index ff2b0f3664..3bd5541527 100644 --- a/src/core/templates/workflows/verify-change.ts +++ b/src/core/templates/workflows/verify-change.ts @@ -47,7 +47,7 @@ ${STORE_SELECTION_GUIDANCE} openspec instructions apply --change "" --json \`\`\` - This returns the change directory and \`contextFiles\` (artifact ID -> array of concrete file paths). Read all available artifacts from \`contextFiles\`. + This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` resolved from the schema's \`apply.tracks\` configuration. Read all available artifacts from \`contextFiles\`. 4. **Initialize verification report structure** @@ -61,16 +61,15 @@ ${STORE_SELECTION_GUIDANCE} 5. **Verify Completeness** **Task Completion**: - - If \`contextFiles\` has no \`tasks\` entry, this dimension is **not verified** - record it as skipped, same as above - - If \`contextFiles.tasks\` exists, read every file path in it - - Parse checkboxes: \`- [ ]\` (incomplete) vs \`- [x]\` (complete) - - Count complete vs total tasks + - Use the top-level \`tasks\` and \`progress\` fields; do not look for a \`contextFiles.tasks\` artifact id. + - If \`tasks\` is empty and \`progress.total\` is 0, no tracked tasks can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply \`state\` and \`instruction\`. + - Report complete vs total tasks from \`progress\`. - If incomplete tasks exist: - Add CRITICAL issue for each incomplete task - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: - - \`contextFiles\` is keyed by artifact id, and artifact ids come from the active schema. If it has no \`specs\` entry, or the entry is empty, this dimension is **not verified** - record it as skipped and carry that through to the report rather than treating it as clean. + - \`contextFiles\` is keyed by artifact id, and artifact ids come from the active schema. If \`contextFiles.specs\` is absent or empty, mark **Spec Coverage**, **Requirement Implementation Mapping**, and **Scenario Coverage** as not verified; do not treat any of them as clean. - If delta specs exist in \`contextFiles.specs\`: - Extract all requirements (marked with "### Requirement:") - For each requirement: @@ -108,7 +107,7 @@ ${STORE_SELECTION_GUIDANCE} - If contradiction detected: - Add WARNING: "Design decision not followed: " - Recommendation: "Update implementation or revise design.md to match reality" - - If no design.md: Skip design adherence check, note "No design.md to verify against" + - If \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. **Code Pattern Consistency** still runs. **Code Pattern Consistency**: - Review new code for consistency with project patterns @@ -131,7 +130,7 @@ ${STORE_SELECTION_GUIDANCE} | Coherence | Followed/Issues | \`\`\` - Write \`Not verified ()\` in the Status cell of any dimension whose artifacts were absent. Never leave it blank or score it as passing. + In each Status cell, report the results of checks that ran and \`Not verified ()\` for every skipped check. If all checks in a dimension were skipped, start the cell with \`Not verified\`. Never score a skipped check as passing. **Issues by Priority**: @@ -152,9 +151,9 @@ ${STORE_SELECTION_GUIDANCE} **Final Assessment**: - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." - - If only warnings: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." - - If all clear and every dimension was verified: "All checks passed. Ready for archive." - - If all clear but a dimension was skipped: say so - "No issues found in the checks that ran. not verified: ." A dimension that checked nothing has not passed. + - If only warnings and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." + - If no issues and no checks were skipped: "All checks passed. Ready for archive." + - If any check was skipped and there are no CRITICAL issues: do not claim readiness. Say "No critical issues found in the checks that ran. not verified: ." Include the warning count when nonzero. **Verification Heuristics** @@ -164,13 +163,6 @@ ${STORE_SELECTION_GUIDANCE} - **False Positives**: When uncertain, prefer SUGGESTION over WARNING, WARNING over CRITICAL - **Actionability**: Every issue must have a specific recommendation with file/line references where applicable -**Graceful Degradation** - -- If only tasks.md exists: verify task completion only, skip spec/design checks -- If tasks + specs exist: verify completeness and correctness, skip design -- If full artifacts: verify all three dimensions -- Always note which checks were skipped and why - **Output Format** Use clear markdown with: @@ -227,7 +219,7 @@ ${STORE_SELECTION_GUIDANCE} openspec instructions apply --change "" --json \`\`\` - This returns the change directory and \`contextFiles\` (artifact ID -> array of concrete file paths). Read all available artifacts from \`contextFiles\`. + This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` resolved from the schema's \`apply.tracks\` configuration. Read all available artifacts from \`contextFiles\`. 4. **Initialize verification report structure** @@ -241,16 +233,15 @@ ${STORE_SELECTION_GUIDANCE} 5. **Verify Completeness** **Task Completion**: - - If \`contextFiles\` has no \`tasks\` entry, this dimension is **not verified** - record it as skipped, same as above - - If \`contextFiles.tasks\` exists, read every file path in it - - Parse checkboxes: \`- [ ]\` (incomplete) vs \`- [x]\` (complete) - - Count complete vs total tasks + - Use the top-level \`tasks\` and \`progress\` fields; do not look for a \`contextFiles.tasks\` artifact id. + - If \`tasks\` is empty and \`progress.total\` is 0, no tracked tasks can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply \`state\` and \`instruction\`. + - Report complete vs total tasks from \`progress\`. - If incomplete tasks exist: - Add CRITICAL issue for each incomplete task - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: - - \`contextFiles\` is keyed by artifact id, and artifact ids come from the active schema. If it has no \`specs\` entry, or the entry is empty, this dimension is **not verified** - record it as skipped and carry that through to the report rather than treating it as clean. + - \`contextFiles\` is keyed by artifact id, and artifact ids come from the active schema. If \`contextFiles.specs\` is absent or empty, mark **Spec Coverage**, **Requirement Implementation Mapping**, and **Scenario Coverage** as not verified; do not treat any of them as clean. - If delta specs exist in \`contextFiles.specs\`: - Extract all requirements (marked with "### Requirement:") - For each requirement: @@ -288,7 +279,7 @@ ${STORE_SELECTION_GUIDANCE} - If contradiction detected: - Add WARNING: "Design decision not followed: " - Recommendation: "Update implementation or revise design.md to match reality" - - If no design.md: Skip design adherence check, note "No design.md to verify against" + - If \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. **Code Pattern Consistency** still runs. **Code Pattern Consistency**: - Review new code for consistency with project patterns @@ -311,7 +302,7 @@ ${STORE_SELECTION_GUIDANCE} | Coherence | Followed/Issues | \`\`\` - Write \`Not verified ()\` in the Status cell of any dimension whose artifacts were absent. Never leave it blank or score it as passing. + In each Status cell, report the results of checks that ran and \`Not verified ()\` for every skipped check. If all checks in a dimension were skipped, start the cell with \`Not verified\`. Never score a skipped check as passing. **Issues by Priority**: @@ -332,9 +323,9 @@ ${STORE_SELECTION_GUIDANCE} **Final Assessment**: - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." - - If only warnings: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." - - If all clear and every dimension was verified: "All checks passed. Ready for archive." - - If all clear but a dimension was skipped: say so - "No issues found in the checks that ran. not verified: ." A dimension that checked nothing has not passed. + - If only warnings and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." + - If no issues and no checks were skipped: "All checks passed. Ready for archive." + - If any check was skipped and there are no CRITICAL issues: do not claim readiness. Say "No critical issues found in the checks that ran. not verified: ." Include the warning count when nonzero. **Verification Heuristics** @@ -344,13 +335,6 @@ ${STORE_SELECTION_GUIDANCE} - **False Positives**: When uncertain, prefer SUGGESTION over WARNING, WARNING over CRITICAL - **Actionability**: Every issue must have a specific recommendation with file/line references where applicable -**Graceful Degradation** - -- If only tasks.md exists: verify task completion only, skip spec/design checks -- If tasks + specs exist: verify completeness and correctness, skip design -- If full artifacts: verify all three dimensions -- Always note which checks were skipped and why - **Output Format** Use clear markdown with: diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index 694cdcf568..c43c6d2b63 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -53,11 +53,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '56bfada1a5f35a127791b70de9d428a75b5aedd1584d6c9803a1ecb1fd1b4a23', getBulkArchiveChangeSkillTemplate: '93875998cade5322d95b43299fba794bc1da754e917dd63a770406386a6d295d', getOpsxSyncCommandTemplate: '0d2427efb79986e8fff3f96bd075a739c80d45eb29159fae717e950030da8202', - getVerifyChangeSkillTemplate: 'b1d4f57ea164d69409bf0137b051d27f9ff8255967af01e3f1dbd5bbeeb7b0f2', + getVerifyChangeSkillTemplate: '3f845275ac43a4efec0d4dd8475f100e5783d3265c6a6ec68567f6c3d1052ad5', getOpsxArchiveCommandTemplate: '9f973c819b11620985b03322945f0e0a92a02a2ef455b94e74482f5e6292ac5d', getOpsxOnboardCommandTemplate: 'ee99aa99252c602720fbb8c63fb3ac438a5bd4e952fd961ddf1ae956cbfc2c8f', getOpsxBulkArchiveCommandTemplate: '9fa8cdebe2f5667ebfc37bdc023396762c59d5b038c771dac2d8fd2c19e2627b', - getOpsxVerifyCommandTemplate: 'ff2c701634454c96b6b5e66dbef499f7000d456c30e48a8534e445342a797ff7', + getOpsxVerifyCommandTemplate: '82a261d31fba50c04b018815759da1490281fd3e72a118dbc30f7fde9824e01a', getOpsxProposeSkillTemplate: '24623c066f97e34b957d448d1f9a9e8b8a13da3dfce45d45671f6226a2534848', getOpsxProposeCommandTemplate: 'e67ba591efb0fecacb2229d06dfa84af18b825fab8a7b01377279e4f09a06ce4', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -74,7 +74,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': 'd933d8856584d6c1253de91e652e7aee9e85c77ad4d3531f6476f79d84e6e5e8', 'openspec-archive-change': '7c65053d674ba4e1e20e2bf73ba7e5a7f94baef2eaa9b33cee48d4cadea51b7a', 'openspec-bulk-archive-change': '2039b9ecf6e64339dffe0e16272507a386d9fe326f419ff758315aa736fdd96c', - 'openspec-verify-change': '7d9a26a1cffe55d0d9d0d5116063dc35d0d9ac232e5e0d67ed1548e2512ede46', + 'openspec-verify-change': '9300d1f87781623bb8fad2742e4602a365323e264ff0903a128dbfec87862799', 'openspec-onboard': 'f6f59476acaf5e4d65dbb180da4cef62432612f3cecf207d471a951295e2003a', 'openspec-propose': '25d08ed4f031770cea219604167d76bca9f3e89fe0c2f545263674482c6f13f0', 'openspec-update-change': '586547406aca94422dfeb3ffedce6c01049429b743f57ce829baa79ebc714d51', diff --git a/test/core/templates/verify-change.test.ts b/test/core/templates/verify-change.test.ts new file mode 100644 index 0000000000..d23d044d36 --- /dev/null +++ b/test/core/templates/verify-change.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; + +import { + getOpsxVerifyCommandTemplate, + getVerifyChangeSkillTemplate, +} from '../../../src/core/templates/skill-templates.js'; + +const skill = getVerifyChangeSkillTemplate(); +const command = getOpsxVerifyCommandTemplate(); + +const bodies: Array<[string, string]> = [ + ['skill', skill.instructions], + ['command', command.content], +]; + +describe('verify-change templates', () => { + it('uses schema-aware apply task fields instead of a hardcoded tasks artifact id', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('top-level `tasks` and `progress`'); + expect(body, label).toContain("schema's `apply.tracks` configuration"); + expect(body, label).toContain('do not look for a `contextFiles.tasks` artifact id'); + expect(body, label).toContain('`tasks` is empty and `progress.total` is 0'); + expect(body, label).not.toContain('`contextFiles.tasks` exists'); + } + }); + + it('maps missing supporting artifacts to every check they prevent', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain( + 'mark **Spec Coverage**, **Requirement Implementation Mapping**, and **Scenario Coverage** as not verified' + ); + expect(body, label).toContain('mark **Design Adherence** as not verified'); + expect(body, label).toContain('**Code Pattern Consistency** still runs'); + } + }); + + it('never reports a skipped check as passing or archive-ready', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('`Not verified ()` for every skipped check'); + expect(body, label).toContain('Never score a skipped check as passing'); + expect(body, label).toContain('If any check was skipped and there are no CRITICAL issues'); + expect(body, label).toContain('do not claim readiness'); + expect(body, label).toContain('If no issues and no checks were skipped'); + } + }); +}); From 826f00b3284939cef2e109c7574a0ac884c2de05 Mon Sep 17 00:00:00 2001 From: Alex <45095641+runsonmypc@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:25:42 -0400 Subject: [PATCH 03/14] fix(verify): retain no-task and skipped-check context --- skills/openspec-verify-change/SKILL.md | 4 ++-- src/core/templates/workflows/verify-change.ts | 8 ++++---- test/core/templates/skill-templates-parity.test.ts | 6 +++--- test/core/templates/verify-change.test.ts | 13 +++++++++++++ 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/skills/openspec-verify-change/SKILL.md b/skills/openspec-verify-change/SKILL.md index 16e0fa737a..6345ed5f8a 100644 --- a/skills/openspec-verify-change/SKILL.md +++ b/skills/openspec-verify-change/SKILL.md @@ -24,7 +24,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design - Auto-select if only one active change exists - If ambiguous, run `openspec list --json` to get available changes and ask the user to select one - When prompting, show changes that have implementation tasks (tasks artifact exists). + When prompting, show all active changes returned by the list, including changes with `status: "no-tasks"`. Include the schema used for each change if available. Mark changes with incomplete tasks as "(In Progress)". @@ -148,7 +148,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design - Each with specific recommendation **Final Assessment**: - - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." + - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." If any check was skipped, also name every skipped check and its reason. - If only warnings and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." - If no issues and no checks were skipped: "All checks passed. Ready for archive." - If any check was skipped and there are no CRITICAL issues: do not claim readiness. Say "No critical issues found in the checks that ran. not verified: ." Include the warning count when nonzero. diff --git a/src/core/templates/workflows/verify-change.ts b/src/core/templates/workflows/verify-change.ts index 3bd5541527..80030dabdc 100644 --- a/src/core/templates/workflows/verify-change.ts +++ b/src/core/templates/workflows/verify-change.ts @@ -26,7 +26,7 @@ ${STORE_SELECTION_GUIDANCE} - Auto-select if only one active change exists - If ambiguous, run \`openspec list --json\` to get available changes and ask the user to select one - When prompting, show changes that have implementation tasks (tasks artifact exists). + When prompting, show all active changes returned by the list, including changes with \`status: "no-tasks"\`. Include the schema used for each change if available. Mark changes with incomplete tasks as "(In Progress)". @@ -150,7 +150,7 @@ ${STORE_SELECTION_GUIDANCE} - Each with specific recommendation **Final Assessment**: - - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." + - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." If any check was skipped, also name every skipped check and its reason. - If only warnings and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." - If no issues and no checks were skipped: "All checks passed. Ready for archive." - If any check was skipped and there are no CRITICAL issues: do not claim readiness. Say "No critical issues found in the checks that ran. not verified: ." Include the warning count when nonzero. @@ -198,7 +198,7 @@ ${STORE_SELECTION_GUIDANCE} - Auto-select if only one active change exists - If ambiguous, run \`openspec list --json\` to get available changes and ask the user to select one - When prompting, show changes that have implementation tasks (tasks artifact exists). + When prompting, show all active changes returned by the list, including changes with \`status: "no-tasks"\`. Include the schema used for each change if available. Mark changes with incomplete tasks as "(In Progress)". @@ -322,7 +322,7 @@ ${STORE_SELECTION_GUIDANCE} - Each with specific recommendation **Final Assessment**: - - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." + - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." If any check was skipped, also name every skipped check and its reason. - If only warnings and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." - If no issues and no checks were skipped: "All checks passed. Ready for archive." - If any check was skipped and there are no CRITICAL issues: do not claim readiness. Say "No critical issues found in the checks that ran. not verified: ." Include the warning count when nonzero. diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index c43c6d2b63..f88868df78 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -53,11 +53,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '56bfada1a5f35a127791b70de9d428a75b5aedd1584d6c9803a1ecb1fd1b4a23', getBulkArchiveChangeSkillTemplate: '93875998cade5322d95b43299fba794bc1da754e917dd63a770406386a6d295d', getOpsxSyncCommandTemplate: '0d2427efb79986e8fff3f96bd075a739c80d45eb29159fae717e950030da8202', - getVerifyChangeSkillTemplate: '3f845275ac43a4efec0d4dd8475f100e5783d3265c6a6ec68567f6c3d1052ad5', + getVerifyChangeSkillTemplate: 'c8a37f79d257d58e3c889e4d46f1bebf86c6852e9b8467364e66668913ca4efe', getOpsxArchiveCommandTemplate: '9f973c819b11620985b03322945f0e0a92a02a2ef455b94e74482f5e6292ac5d', getOpsxOnboardCommandTemplate: 'ee99aa99252c602720fbb8c63fb3ac438a5bd4e952fd961ddf1ae956cbfc2c8f', getOpsxBulkArchiveCommandTemplate: '9fa8cdebe2f5667ebfc37bdc023396762c59d5b038c771dac2d8fd2c19e2627b', - getOpsxVerifyCommandTemplate: '82a261d31fba50c04b018815759da1490281fd3e72a118dbc30f7fde9824e01a', + getOpsxVerifyCommandTemplate: 'f5ea1eb44d5b893441ed72ee50cee7560fe1b48d32e1b960a076065831061522', getOpsxProposeSkillTemplate: '24623c066f97e34b957d448d1f9a9e8b8a13da3dfce45d45671f6226a2534848', getOpsxProposeCommandTemplate: 'e67ba591efb0fecacb2229d06dfa84af18b825fab8a7b01377279e4f09a06ce4', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -74,7 +74,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': 'd933d8856584d6c1253de91e652e7aee9e85c77ad4d3531f6476f79d84e6e5e8', 'openspec-archive-change': '7c65053d674ba4e1e20e2bf73ba7e5a7f94baef2eaa9b33cee48d4cadea51b7a', 'openspec-bulk-archive-change': '2039b9ecf6e64339dffe0e16272507a386d9fe326f419ff758315aa736fdd96c', - 'openspec-verify-change': '9300d1f87781623bb8fad2742e4602a365323e264ff0903a128dbfec87862799', + 'openspec-verify-change': '4def773e6fc6b15ae95b01802acffa10dddf1cf7bb4735e3c8aa7089688991d8', 'openspec-onboard': 'f6f59476acaf5e4d65dbb180da4cef62432612f3cecf207d471a951295e2003a', 'openspec-propose': '25d08ed4f031770cea219604167d76bca9f3e89fe0c2f545263674482c6f13f0', 'openspec-update-change': '586547406aca94422dfeb3ffedce6c01049429b743f57ce829baa79ebc714d51', diff --git a/test/core/templates/verify-change.test.ts b/test/core/templates/verify-change.test.ts index d23d044d36..d6eee6cbe4 100644 --- a/test/core/templates/verify-change.test.ts +++ b/test/core/templates/verify-change.test.ts @@ -14,6 +14,16 @@ const bodies: Array<[string, string]> = [ ]; describe('verify-change templates', () => { + it('keeps active no-task changes eligible for ambiguous selection', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('show all active changes returned by the list'); + expect(body, label).toContain('including changes with `status: "no-tasks"`'); + expect(body, label).not.toContain( + 'show changes that have implementation tasks (tasks artifact exists)' + ); + } + }); + it('uses schema-aware apply task fields instead of a hardcoded tasks artifact id', () => { for (const [label, body] of bodies) { expect(body, label).toContain('top-level `tasks` and `progress`'); @@ -39,6 +49,9 @@ describe('verify-change templates', () => { expect(body, label).toContain('`Not verified ()` for every skipped check'); expect(body, label).toContain('Never score a skipped check as passing'); expect(body, label).toContain('If any check was skipped and there are no CRITICAL issues'); + expect(body, label).toContain( + 'If any check was skipped, also name every skipped check and its reason' + ); expect(body, label).toContain('do not claim readiness'); expect(body, label).toContain('If no issues and no checks were skipped'); } From 223d56460103a725b9100abfe4762dd9dc16a460 Mon Sep 17 00:00:00 2001 From: Clay Good Date: Fri, 28 Aug 2026 09:44:21 -0500 Subject: [PATCH 04/14] fix(verify): harden evidence gaps and final assessments --- skills/openspec-verify-change/SKILL.md | 19 ++++-- src/core/templates/workflows/verify-change.ts | 38 ++++++++---- .../commands/apply-instructions-tasks.test.ts | 60 +++++++++++++++++++ .../templates/skill-templates-parity.test.ts | 6 +- test/core/templates/verify-change.test.ts | 35 ++++++++++- 5 files changed, 136 insertions(+), 22 deletions(-) diff --git a/skills/openspec-verify-change/SKILL.md b/skills/openspec-verify-change/SKILL.md index 6345ed5f8a..ddb943dd17 100644 --- a/skills/openspec-verify-change/SKILL.md +++ b/skills/openspec-verify-change/SKILL.md @@ -47,6 +47,8 @@ Verify that an implementation matches the change artifacts (specs, tasks, design This returns the change directory, `contextFiles` (artifact ID -> array of concrete file paths), and top-level `tasks` and `progress` resolved from the schema's `apply.tracks` configuration. Read all available artifacts from `contextFiles`. + Treat apply `state` and `instruction` as context, not a verification verdict. Do not implement tasks or archive the change during verification. + 4. **Initialize verification report structure** Create a report structure with three dimensions: @@ -56,14 +58,16 @@ Verify that an implementation matches the change artifacts (specs, tasks, design Each dimension can have CRITICAL, WARNING, or SUGGESTION issues. + If artifacts cannot be read or contain no usable requirements, scenarios, or design decisions, mark the affected checks as not verified with the specific reason. Continue checks supported by the remaining evidence, but a partially checked input set is not a fully verified check. Missing requirements affect Spec Coverage and Requirement Implementation Mapping; missing scenarios affect Scenario Coverage; missing design decisions affect Design Adherence. + 5. **Verify Completeness** **Task Completion**: - Use the top-level `tasks` and `progress` fields; do not look for a `contextFiles.tasks` artifact id. - - If `tasks` is empty and `progress.total` is 0, no tracked tasks can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply `state` and `instruction`. + - If `tasks` is empty, even when `progress.total` is nonzero, no task descriptions can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply `state` and `instruction`. - Report complete vs total tasks from `progress`. - - If incomplete tasks exist: - - Add CRITICAL issue for each incomplete task + - If `progress.remaining` is greater than 0: + - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: @@ -108,7 +112,8 @@ Verify that an implementation matches the change artifacts (specs, tasks, design - If `contextFiles.design` is absent or empty: mark **Design Adherence** as not verified. **Code Pattern Consistency** still runs. **Code Pattern Consistency**: - - Review new code for consistency with project patterns + - If implementation changes cannot be identified, mark **Code Pattern Consistency** as not verified and explain the missing evidence. + - Otherwise, review new code for consistency with project patterns - Check file naming, directory structure, coding style - If significant deviations found: - Add SUGGESTION: "Code pattern deviation:
" @@ -128,7 +133,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design | Coherence | Followed/Issues | ``` - In each Status cell, report the results of checks that ran and `Not verified ()` for every skipped check. If all checks in a dimension were skipped, start the cell with `Not verified`. Never score a skipped check as passing. + In each Status cell, report the results of checks that ran and `Not verified ()` for every skipped check. If all checks in a dimension were skipped, start the cell with `Not verified`. Never score a skipped check as passing. Treat every not verified or partially verified check as skipped in the final assessment. **Issues by Priority**: @@ -149,9 +154,11 @@ Verify that an implementation matches the change artifacts (specs, tasks, design **Final Assessment**: - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." If any check was skipped, also name every skipped check and its reason. - - If only warnings and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." + - If no CRITICAL issues, one or more warnings, and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." + - If only suggestions and no checks were skipped: "No critical issues or warnings. Z suggestion(s) to consider. Ready for archive (with noted improvements)." - If no issues and no checks were skipped: "All checks passed. Ready for archive." - If any check was skipped and there are no CRITICAL issues: do not claim readiness. Say "No critical issues found in the checks that ran. not verified: ." Include the warning count when nonzero. + - Include the suggestion count when nonzero in every final assessment. **Verification Heuristics** diff --git a/src/core/templates/workflows/verify-change.ts b/src/core/templates/workflows/verify-change.ts index 80030dabdc..7c989623a7 100644 --- a/src/core/templates/workflows/verify-change.ts +++ b/src/core/templates/workflows/verify-change.ts @@ -49,6 +49,8 @@ ${STORE_SELECTION_GUIDANCE} This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` resolved from the schema's \`apply.tracks\` configuration. Read all available artifacts from \`contextFiles\`. + Treat apply \`state\` and \`instruction\` as context, not a verification verdict. Do not implement tasks or archive the change during verification. + 4. **Initialize verification report structure** Create a report structure with three dimensions: @@ -58,14 +60,16 @@ ${STORE_SELECTION_GUIDANCE} Each dimension can have CRITICAL, WARNING, or SUGGESTION issues. + If artifacts cannot be read or contain no usable requirements, scenarios, or design decisions, mark the affected checks as not verified with the specific reason. Continue checks supported by the remaining evidence, but a partially checked input set is not a fully verified check. Missing requirements affect Spec Coverage and Requirement Implementation Mapping; missing scenarios affect Scenario Coverage; missing design decisions affect Design Adherence. + 5. **Verify Completeness** **Task Completion**: - Use the top-level \`tasks\` and \`progress\` fields; do not look for a \`contextFiles.tasks\` artifact id. - - If \`tasks\` is empty and \`progress.total\` is 0, no tracked tasks can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply \`state\` and \`instruction\`. + - If \`tasks\` is empty, even when \`progress.total\` is nonzero, no task descriptions can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply \`state\` and \`instruction\`. - Report complete vs total tasks from \`progress\`. - - If incomplete tasks exist: - - Add CRITICAL issue for each incomplete task + - If \`progress.remaining\` is greater than 0: + - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: @@ -110,7 +114,8 @@ ${STORE_SELECTION_GUIDANCE} - If \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. **Code Pattern Consistency** still runs. **Code Pattern Consistency**: - - Review new code for consistency with project patterns + - If implementation changes cannot be identified, mark **Code Pattern Consistency** as not verified and explain the missing evidence. + - Otherwise, review new code for consistency with project patterns - Check file naming, directory structure, coding style - If significant deviations found: - Add SUGGESTION: "Code pattern deviation:
" @@ -130,7 +135,7 @@ ${STORE_SELECTION_GUIDANCE} | Coherence | Followed/Issues | \`\`\` - In each Status cell, report the results of checks that ran and \`Not verified ()\` for every skipped check. If all checks in a dimension were skipped, start the cell with \`Not verified\`. Never score a skipped check as passing. + In each Status cell, report the results of checks that ran and \`Not verified ()\` for every skipped check. If all checks in a dimension were skipped, start the cell with \`Not verified\`. Never score a skipped check as passing. Treat every not verified or partially verified check as skipped in the final assessment. **Issues by Priority**: @@ -151,9 +156,11 @@ ${STORE_SELECTION_GUIDANCE} **Final Assessment**: - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." If any check was skipped, also name every skipped check and its reason. - - If only warnings and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." + - If no CRITICAL issues, one or more warnings, and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." + - If only suggestions and no checks were skipped: "No critical issues or warnings. Z suggestion(s) to consider. Ready for archive (with noted improvements)." - If no issues and no checks were skipped: "All checks passed. Ready for archive." - If any check was skipped and there are no CRITICAL issues: do not claim readiness. Say "No critical issues found in the checks that ran. not verified: ." Include the warning count when nonzero. + - Include the suggestion count when nonzero in every final assessment. **Verification Heuristics** @@ -221,6 +228,8 @@ ${STORE_SELECTION_GUIDANCE} This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` resolved from the schema's \`apply.tracks\` configuration. Read all available artifacts from \`contextFiles\`. + Treat apply \`state\` and \`instruction\` as context, not a verification verdict. Do not implement tasks or archive the change during verification. + 4. **Initialize verification report structure** Create a report structure with three dimensions: @@ -230,14 +239,16 @@ ${STORE_SELECTION_GUIDANCE} Each dimension can have CRITICAL, WARNING, or SUGGESTION issues. + If artifacts cannot be read or contain no usable requirements, scenarios, or design decisions, mark the affected checks as not verified with the specific reason. Continue checks supported by the remaining evidence, but a partially checked input set is not a fully verified check. Missing requirements affect Spec Coverage and Requirement Implementation Mapping; missing scenarios affect Scenario Coverage; missing design decisions affect Design Adherence. + 5. **Verify Completeness** **Task Completion**: - Use the top-level \`tasks\` and \`progress\` fields; do not look for a \`contextFiles.tasks\` artifact id. - - If \`tasks\` is empty and \`progress.total\` is 0, no tracked tasks can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply \`state\` and \`instruction\`. + - If \`tasks\` is empty, even when \`progress.total\` is nonzero, no task descriptions can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply \`state\` and \`instruction\`. - Report complete vs total tasks from \`progress\`. - - If incomplete tasks exist: - - Add CRITICAL issue for each incomplete task + - If \`progress.remaining\` is greater than 0: + - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: @@ -282,7 +293,8 @@ ${STORE_SELECTION_GUIDANCE} - If \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. **Code Pattern Consistency** still runs. **Code Pattern Consistency**: - - Review new code for consistency with project patterns + - If implementation changes cannot be identified, mark **Code Pattern Consistency** as not verified and explain the missing evidence. + - Otherwise, review new code for consistency with project patterns - Check file naming, directory structure, coding style - If significant deviations found: - Add SUGGESTION: "Code pattern deviation:
" @@ -302,7 +314,7 @@ ${STORE_SELECTION_GUIDANCE} | Coherence | Followed/Issues | \`\`\` - In each Status cell, report the results of checks that ran and \`Not verified ()\` for every skipped check. If all checks in a dimension were skipped, start the cell with \`Not verified\`. Never score a skipped check as passing. + In each Status cell, report the results of checks that ran and \`Not verified ()\` for every skipped check. If all checks in a dimension were skipped, start the cell with \`Not verified\`. Never score a skipped check as passing. Treat every not verified or partially verified check as skipped in the final assessment. **Issues by Priority**: @@ -323,9 +335,11 @@ ${STORE_SELECTION_GUIDANCE} **Final Assessment**: - If CRITICAL issues: "X critical issue(s) found. Fix before archiving." If any check was skipped, also name every skipped check and its reason. - - If only warnings and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." + - If no CRITICAL issues, one or more warnings, and no checks were skipped: "No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements)." + - If only suggestions and no checks were skipped: "No critical issues or warnings. Z suggestion(s) to consider. Ready for archive (with noted improvements)." - If no issues and no checks were skipped: "All checks passed. Ready for archive." - If any check was skipped and there are no CRITICAL issues: do not claim readiness. Say "No critical issues found in the checks that ran. not verified: ." Include the warning count when nonzero. + - Include the suggestion count when nonzero in every final assessment. **Verification Heuristics** diff --git a/test/commands/apply-instructions-tasks.test.ts b/test/commands/apply-instructions-tasks.test.ts index f6f81e9c7a..5549087668 100644 --- a/test/commands/apply-instructions-tasks.test.ts +++ b/test/commands/apply-instructions-tasks.test.ts @@ -34,6 +34,65 @@ describe('generateApplyInstructions task list', () => { fs.writeFileSync(path.join(changeDir, 'tasks.md'), content); } + it.each([true, false])('resolves custom tracking configuration (enabled: %s)', async (tracked) => { + const schemaDir = path.join(tempDir, 'openspec', 'schemas', 'custom'); + fs.mkdirSync(schemaDir, { recursive: true }); + fs.writeFileSync( + path.join(schemaDir, 'schema.yaml'), + `name: custom +version: 1 +artifacts: + - id: implementation + generates: checklist.md + description: Implementation checklist + template: checklist.md + requires: [] +apply: + requires: [implementation] +${tracked ? ' tracks: checklist.md\n' : ''}` + ); + fs.writeFileSync(path.join(changeDir, '.openspec.yaml'), 'schema: custom\n'); + const checklist = path.join(changeDir, 'checklist.md'); + fs.writeFileSync(checklist, '- [x] Finished task\n- [ ] Pending task\n'); + + const instructions = await generateApplyInstructions(tempDir, 'my-change'); + + expect(instructions.contextFiles).toEqual({ implementation: [fs.realpathSync.native(checklist)] }); + expect(instructions.tasks).toEqual(tracked ? [ + { id: '1', description: 'Finished task', done: true }, + { id: '2', description: 'Pending task', done: false }, + ] : []); + expect(instructions.progress).toEqual(tracked + ? { total: 2, complete: 1, remaining: 1 } + : { total: 0, complete: 0, remaining: 0 }); + expect(instructions.state).toBe('ready'); + }); + + it.each(['missing', 'empty'])('returns no task evidence for a %s tracking file', async (kind) => { + if (kind === 'empty') writeTasks(''); + + const instructions = await generateApplyInstructions(tempDir, 'my-change'); + + expect(instructions.tasks).toEqual([]); + expect(instructions.progress).toEqual({ total: 0, complete: 0, remaining: 0 }); + expect(instructions.state).toBe('blocked'); + expect(instructions.instruction).toContain(kind === 'missing' ? 'Missing artifacts: tasks' : 'contains no tasks'); + }); + + it('returns existing spec and design paths even when their files contain no evidence', async () => { + writeTasks('- [x] Finished task\n'); + const spec = path.join(changeDir, 'specs', 'demo', 'spec.md'); + const design = path.join(changeDir, 'design.md'); + fs.writeFileSync(spec, ''); + fs.writeFileSync(design, ''); + + const instructions = await generateApplyInstructions(tempDir, 'my-change'); + + expect(instructions.contextFiles.specs).toEqual([fs.realpathSync.native(spec)]); + expect(instructions.contextFiles.design).toEqual([fs.realpathSync.native(design)]); + expect(instructions.state).toBe('all_done'); + }); + it('lists indented sub-tasks alongside their parents', async () => { writeTasks( [ @@ -84,6 +143,7 @@ describe('generateApplyInstructions task list', () => { // As before the shared parser: apply points at regenerating the file // rather than listing a blank row an agent cannot act on. expect(instructions.tasks).toEqual([]); + expect(instructions.progress).toEqual({ total: 1, complete: 1, remaining: 0 }); expect(instructions.state).toBe('blocked'); expect(instructions.instruction).toContain('contains no tasks'); }); diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index f88868df78..1f727d5d33 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -53,11 +53,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '56bfada1a5f35a127791b70de9d428a75b5aedd1584d6c9803a1ecb1fd1b4a23', getBulkArchiveChangeSkillTemplate: '93875998cade5322d95b43299fba794bc1da754e917dd63a770406386a6d295d', getOpsxSyncCommandTemplate: '0d2427efb79986e8fff3f96bd075a739c80d45eb29159fae717e950030da8202', - getVerifyChangeSkillTemplate: 'c8a37f79d257d58e3c889e4d46f1bebf86c6852e9b8467364e66668913ca4efe', + getVerifyChangeSkillTemplate: '7d9ed0264ec431e875837db373c1b419b86910918522c264331e50f97b12a60c', getOpsxArchiveCommandTemplate: '9f973c819b11620985b03322945f0e0a92a02a2ef455b94e74482f5e6292ac5d', getOpsxOnboardCommandTemplate: 'ee99aa99252c602720fbb8c63fb3ac438a5bd4e952fd961ddf1ae956cbfc2c8f', getOpsxBulkArchiveCommandTemplate: '9fa8cdebe2f5667ebfc37bdc023396762c59d5b038c771dac2d8fd2c19e2627b', - getOpsxVerifyCommandTemplate: 'f5ea1eb44d5b893441ed72ee50cee7560fe1b48d32e1b960a076065831061522', + getOpsxVerifyCommandTemplate: 'ce191e2dafabe720100eb75025da25369f7990f62321fcc62870c360b9cd4fde', getOpsxProposeSkillTemplate: '24623c066f97e34b957d448d1f9a9e8b8a13da3dfce45d45671f6226a2534848', getOpsxProposeCommandTemplate: 'e67ba591efb0fecacb2229d06dfa84af18b825fab8a7b01377279e4f09a06ce4', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -74,7 +74,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': 'd933d8856584d6c1253de91e652e7aee9e85c77ad4d3531f6476f79d84e6e5e8', 'openspec-archive-change': '7c65053d674ba4e1e20e2bf73ba7e5a7f94baef2eaa9b33cee48d4cadea51b7a', 'openspec-bulk-archive-change': '2039b9ecf6e64339dffe0e16272507a386d9fe326f419ff758315aa736fdd96c', - 'openspec-verify-change': '4def773e6fc6b15ae95b01802acffa10dddf1cf7bb4735e3c8aa7089688991d8', + 'openspec-verify-change': '1b7a2ddf025f163cdac00c2e7227858ae349c5703fd1f9be69b9edce6513e6cc', 'openspec-onboard': 'f6f59476acaf5e4d65dbb180da4cef62432612f3cecf207d471a951295e2003a', 'openspec-propose': '25d08ed4f031770cea219604167d76bca9f3e89fe0c2f545263674482c6f13f0', 'openspec-update-change': '586547406aca94422dfeb3ffedce6c01049429b743f57ce829baa79ebc714d51', diff --git a/test/core/templates/verify-change.test.ts b/test/core/templates/verify-change.test.ts index d6eee6cbe4..3a4ff37b92 100644 --- a/test/core/templates/verify-change.test.ts +++ b/test/core/templates/verify-change.test.ts @@ -29,11 +29,43 @@ describe('verify-change templates', () => { expect(body, label).toContain('top-level `tasks` and `progress`'); expect(body, label).toContain("schema's `apply.tracks` configuration"); expect(body, label).toContain('do not look for a `contextFiles.tasks` artifact id'); - expect(body, label).toContain('`tasks` is empty and `progress.total` is 0'); + expect(body, label).toContain('If `tasks` is empty, even when `progress.total` is nonzero'); expect(body, label).not.toContain('`contextFiles.tasks` exists'); } }); + it('does not lose incomplete checkboxes omitted from the task list', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('If `progress.remaining` is greater than 0'); + expect(body, label).toContain('incomplete checkboxes without descriptions'); + expect(body, label).toContain('Do not infer completion from the listed tasks alone'); + } + }); + + it('requires usable evidence rather than just existing artifact paths', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('cannot be read or contain no usable requirements, scenarios, or design decisions'); + expect(body, label).toContain('Continue checks supported by the remaining evidence'); + expect(body, label).toContain('a partially checked input set is not a fully verified check'); + expect(body, label).toContain('If implementation changes cannot be identified, mark **Code Pattern Consistency** as not verified'); + } + }); + + it('does not mistake apply readiness for verification or execute apply instructions', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('Treat apply `state` and `instruction` as context, not a verification verdict'); + expect(body, label).toContain('Do not implement tasks or archive the change during verification'); + } + }); + + it('covers warning and suggestion outcomes without claiming all checks passed', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('If no CRITICAL issues, one or more warnings, and no checks were skipped'); + expect(body, label).toContain('If only suggestions and no checks were skipped'); + expect(body, label).toContain('Include the suggestion count when nonzero'); + } + }); + it('maps missing supporting artifacts to every check they prevent', () => { for (const [label, body] of bodies) { expect(body, label).toContain( @@ -48,6 +80,7 @@ describe('verify-change templates', () => { for (const [label, body] of bodies) { expect(body, label).toContain('`Not verified ()` for every skipped check'); expect(body, label).toContain('Never score a skipped check as passing'); + expect(body, label).toContain('Treat every not verified or partially verified check as skipped in the final assessment'); expect(body, label).toContain('If any check was skipped and there are no CRITICAL issues'); expect(body, label).toContain( 'If any check was skipped, also name every skipped check and its reason' From 1872674ba82b123210038277c183dea8601a4a1e Mon Sep 17 00:00:00 2001 From: Clay Good Date: Fri, 28 Aug 2026 13:56:40 -0500 Subject: [PATCH 05/14] fix(verify): preserve optional workflows and task artifact fallback --- .../verify-report-skipped-dimensions.md | 2 +- openspec/specs/opsx-verify-skill/spec.md | 65 +++++++++++++++---- skills/openspec-verify-change/SKILL.md | 17 +++-- src/core/templates/workflows/verify-change.ts | 34 ++++++---- .../commands/apply-instructions-tasks.test.ts | 45 +++++++++++++ .../templates/skill-templates-parity.test.ts | 6 +- test/core/templates/verify-change.test.ts | 36 ++++++++-- 7 files changed, 167 insertions(+), 38 deletions(-) diff --git a/.changeset/verify-report-skipped-dimensions.md b/.changeset/verify-report-skipped-dimensions.md index 24d60d9ea8..d553c1a155 100644 --- a/.changeset/verify-report-skipped-dimensions.md +++ b/.changeset/verify-report-skipped-dimensions.md @@ -2,4 +2,4 @@ "@fission-ai/openspec": patch --- -Stop `/opsx:verify` from reporting skipped checks as passing. Task completion now uses the schema-aware `tasks` and `progress` fields returned by apply instructions, while absent spec or design inputs are mapped to every check they prevent. The scorecard identifies each skipped check, and the final assessment does not claim archive readiness when any check did not run. +Stop `/opsx:verify` from reporting skipped checks as passing. Task completion now uses the schema-aware `tasks` and `progress` fields returned by apply instructions, while absent spec or design inputs are mapped to every check they prevent. When apply metadata has no task entries, verification still reads available task artifact files, including glob outputs. Verification stays advisory and does not require optional or intentionally omitted artifacts. The scorecard identifies each skipped check, and the final assessment does not claim archive readiness when any check did not run. diff --git a/openspec/specs/opsx-verify-skill/spec.md b/openspec/specs/opsx-verify-skill/spec.md index 91562c0e55..b18202a334 100644 --- a/openspec/specs/opsx-verify-skill/spec.md +++ b/openspec/specs/opsx-verify-skill/spec.md @@ -15,22 +15,30 @@ The system SHALL provide an `/opsx:verify` skill that validates implementation a #### Scenario: Verify without change name - **WHEN** agent executes `/opsx:verify` without a change name - **THEN** the agent infers the change from conversation context, or auto-selects it when only one active change exists -- **AND** when ambiguous, prompts user to select from available changes, showing only changes that have implementation tasks +- **AND** when ambiguous, prompts user to select from all active changes, including changes with no tracked tasks - **AND** announces which change was selected and how to override -#### Scenario: Change has no tasks -- **WHEN** selected change has no tasks.md or tasks are empty -- **THEN** the agent reports "No tasks to verify" -- **AND** suggests running `/opsx:continue` to create tasks +#### Scenario: Change has no task descriptions +- **WHEN** neither the structured task list nor the resolved task artifact fallback provides usable task descriptions, even if task progress reports nonzero totals +- **THEN** the agent reports Task Completion as not verified with the reason +- **AND** continues checks supported by the remaining artifacts +- **AND** does not require tasks when the schema does not track them ### Requirement: Completeness Verification The agent SHALL verify that all required work has been completed. #### Scenario: Task completion check - **WHEN** verifying completeness -- **THEN** the agent reads tasks.md -- **AND** counts tasks marked `- [x]` (complete) vs `- [ ]` (incomplete) +- **THEN** the agent prefers the top-level `tasks` and `progress` from apply instructions, resolved from the active schema's `apply.tracks` configuration +- **AND** reports complete and total task counts from `progress` - **AND** reports completion status with specific incomplete tasks listed +- **AND** reports remaining checkboxes without descriptions when `progress.remaining` exceeds the listed incomplete tasks + +#### Scenario: Resolved task artifact fallback +- **WHEN** structured apply tasks are empty but resolved files exist in `contextFiles.tasks` +- **THEN** read every resolved task artifact and derive task descriptions and checkbox totals from those files +- **AND** identify the totals as artifact-derived rather than presenting zero apply progress as task completion +- **AND** report Task Completion as not verified if the files cannot be read or provide no usable task descriptions #### Scenario: Spec coverage check - **WHEN** verifying completeness @@ -42,7 +50,8 @@ The agent SHALL verify that all required work has been completed. #### Scenario: All tasks complete - **WHEN** all tasks are marked complete - **THEN** report "Tasks: N/N complete" -- **AND** mark completeness dimension as passed +- **AND** mark Task Completion as passed only when task descriptions are available +- **AND** mark the completeness dimension as passed only when all its checks ran and passed #### Scenario: Incomplete tasks found - **WHEN** some tasks are incomplete @@ -98,7 +107,7 @@ The agent SHALL verify that implementation is sensible and follows design decisi - **WHEN** verifying coherence - **AND** no design.md exists - **THEN** skip design adherence check -- **AND** note "No design.md to verify against" +- **AND** report "Design Adherence: Not verified (No design.md to verify against)" #### Scenario: Design decision followed - **WHEN** implementation follows a design decision @@ -113,8 +122,10 @@ The agent SHALL verify that implementation is sensible and follows design decisi #### Scenario: Code pattern consistency - **WHEN** verifying coherence +- **AND** available artifacts support identifying implementation changes beyond a tasks-only check - **THEN** check if new code follows existing project patterns - **AND** flag any significant deviations as suggestions +- **AND** report Code Pattern Consistency as not verified if implementation changes cannot be identified ### Requirement: Verification Report Format The agent SHALL produce a structured, prioritized report. @@ -132,6 +143,8 @@ The agent SHALL produce a structured, prioritized report. | Correctness | X/Y | | Coherence | Followed | ``` +- **AND** report `Not verified ()` for every skipped or partially verified check in its dimension's status +- **AND** never count a skipped check as passing #### Scenario: Issue prioritization - **WHEN** issues are found @@ -147,7 +160,7 @@ The agent SHALL produce a structured, prioritized report. - **AND** avoid vague suggestions like "consider reviewing" #### Scenario: All checks pass -- **WHEN** no issues found across all dimensions +- **WHEN** every check ran and no issues were found across all dimensions - **THEN** display: ```text All checks passed. Ready for archive. @@ -160,15 +173,31 @@ The agent SHALL produce a structured, prioritized report. X critical issue(s) found. Fix before archiving. ``` - **AND** do NOT suggest running archive +- **AND** name every skipped check and its reason, if any -#### Scenario: Only warnings/suggestions -- **WHEN** no CRITICAL issues but warnings exist +#### Scenario: Only warnings +- **WHEN** every check ran and no CRITICAL issues but warnings exist - **THEN** display: ```text No critical issues. Y warning(s) to consider. Ready for archive (with noted improvements). ``` +#### Scenario: Only suggestions +- **WHEN** every check ran and only suggestions exist +- **THEN** report "No critical issues or warnings. Z suggestion(s) to consider. Ready for archive (with noted improvements)." + +#### Scenario: Checks skipped +- **WHEN** any check was skipped or partially verified and no CRITICAL issues exist +- **THEN** report "No critical issues found in the checks that ran" +- **AND** name every unverified check and its reason +- **AND** include the warning count when nonzero +- **AND** do not claim archive readiness + +#### Scenario: Suggestions in final assessment +- **WHEN** suggestions exist +- **THEN** include their count in the final assessment, including assessments with critical issues or skipped checks + ### Requirement: Flexible Artifact Handling The agent SHALL gracefully handle changes with varying artifact completeness. @@ -188,3 +217,15 @@ The agent SHALL gracefully handle changes with varying artifact completeness. - **WHEN** change has proposal, design, specs, and tasks - **THEN** perform all verification checks - **AND** cross-reference artifacts for consistency + +#### Scenario: Unusable or partial artifact evidence +- **WHEN** an artifact cannot be read or lacks usable requirements, scenarios, or design decisions +- **THEN** mark each affected check as not verified with its reason +- **AND** continue checks supported by the remaining evidence without treating partial coverage as a fully verified check + +#### Scenario: Intentional artifact omissions +- **WHEN** a check has no supporting artifacts because the schema omits task tracking or optional artifacts, or the change declares `skip_specs: true` +- **THEN** report why the corresponding checks were not verified +- **AND** do not require or create optional or intentionally skipped artifacts to obtain a passing report +- **AND** treat verification as advisory: not verified describes a limit of the report, not a new archive gate +- **AND** leave archive checks and user-confirmation behavior unchanged diff --git a/skills/openspec-verify-change/SKILL.md b/skills/openspec-verify-change/SKILL.md index ddb943dd17..560a2064b9 100644 --- a/skills/openspec-verify-change/SKILL.md +++ b/skills/openspec-verify-change/SKILL.md @@ -58,16 +58,21 @@ Verify that an implementation matches the change artifacts (specs, tasks, design Each dimension can have CRITICAL, WARNING, or SUGGESTION issues. + Verification is advisory. Respect intentional omissions such as `skip_specs: true`, optional design documents, and schemas without task tracking. Do not require or invent optional or intentionally omitted artifacts to obtain a clean report. `Not verified` describes a limit of this report, not a new archive prerequisite. Archive retains its own checks and user-confirmation behavior. + + If only task evidence is available, verify task completion only and mark the remaining checks, including **Code Pattern Consistency**, as not verified with the reason "Only task evidence available". + If artifacts cannot be read or contain no usable requirements, scenarios, or design decisions, mark the affected checks as not verified with the specific reason. Continue checks supported by the remaining evidence, but a partially checked input set is not a fully verified check. Missing requirements affect Spec Coverage and Requirement Implementation Mapping; missing scenarios affect Scenario Coverage; missing design decisions affect Design Adherence. 5. **Verify Completeness** **Task Completion**: - - Use the top-level `tasks` and `progress` fields; do not look for a `contextFiles.tasks` artifact id. - - If `tasks` is empty, even when `progress.total` is nonzero, no task descriptions can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply `state` and `instruction`. - - Report complete vs total tasks from `progress`. - - If `progress.remaining` is greater than 0: - - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. + - Prefer the top-level `tasks` and `progress` fields; do not assume every schema uses a `tasks` artifact id. + - If `tasks` is empty but `contextFiles.tasks` contains paths, read every resolved file and count its checkboxes, including nested and blank checkboxes. This preserves verification for task globs that apply metadata does not represent. Label these as artifact-derived totals; do not report the zero apply totals as completion. If any file cannot be read, report the partial evidence and mark **Task Completion** as not verified. + - If neither source provides task descriptions, mark **Task Completion** as not verified and record the reason from the available evidence, including apply `state` and `instruction`. Nonzero totals alone do not establish evaluable tasks. + - Report complete vs total tasks from `progress`, or the artifact-derived totals when using the fallback. + - If `progress.remaining` is greater than 0 or the fallback files contain incomplete checkboxes: + - Add CRITICAL issue for each incomplete task from the selected source. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: @@ -109,7 +114,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design - If contradiction detected: - Add WARNING: "Design decision not followed: " - Recommendation: "Update implementation or revise design.md to match reality" - - If `contextFiles.design` is absent or empty: mark **Design Adherence** as not verified. **Code Pattern Consistency** still runs. + - If `contextFiles.design` is absent or empty: mark **Design Adherence** as not verified. With other supporting artifacts, **Code Pattern Consistency** still runs; the task-only case remains limited to task completion. **Code Pattern Consistency**: - If implementation changes cannot be identified, mark **Code Pattern Consistency** as not verified and explain the missing evidence. diff --git a/src/core/templates/workflows/verify-change.ts b/src/core/templates/workflows/verify-change.ts index 7c989623a7..5650197ec1 100644 --- a/src/core/templates/workflows/verify-change.ts +++ b/src/core/templates/workflows/verify-change.ts @@ -60,16 +60,21 @@ ${STORE_SELECTION_GUIDANCE} Each dimension can have CRITICAL, WARNING, or SUGGESTION issues. + Verification is advisory. Respect intentional omissions such as \`skip_specs: true\`, optional design documents, and schemas without task tracking. Do not require or invent optional or intentionally omitted artifacts to obtain a clean report. \`Not verified\` describes a limit of this report, not a new archive prerequisite. Archive retains its own checks and user-confirmation behavior. + + If only task evidence is available, verify task completion only and mark the remaining checks, including **Code Pattern Consistency**, as not verified with the reason "Only task evidence available". + If artifacts cannot be read or contain no usable requirements, scenarios, or design decisions, mark the affected checks as not verified with the specific reason. Continue checks supported by the remaining evidence, but a partially checked input set is not a fully verified check. Missing requirements affect Spec Coverage and Requirement Implementation Mapping; missing scenarios affect Scenario Coverage; missing design decisions affect Design Adherence. 5. **Verify Completeness** **Task Completion**: - - Use the top-level \`tasks\` and \`progress\` fields; do not look for a \`contextFiles.tasks\` artifact id. - - If \`tasks\` is empty, even when \`progress.total\` is nonzero, no task descriptions can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply \`state\` and \`instruction\`. - - Report complete vs total tasks from \`progress\`. - - If \`progress.remaining\` is greater than 0: - - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. + - Prefer the top-level \`tasks\` and \`progress\` fields; do not assume every schema uses a \`tasks\` artifact id. + - If \`tasks\` is empty but \`contextFiles.tasks\` contains paths, read every resolved file and count its checkboxes, including nested and blank checkboxes. This preserves verification for task globs that apply metadata does not represent. Label these as artifact-derived totals; do not report the zero apply totals as completion. If any file cannot be read, report the partial evidence and mark **Task Completion** as not verified. + - If neither source provides task descriptions, mark **Task Completion** as not verified and record the reason from the available evidence, including apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable tasks. + - Report complete vs total tasks from \`progress\`, or the artifact-derived totals when using the fallback. + - If \`progress.remaining\` is greater than 0 or the fallback files contain incomplete checkboxes: + - Add CRITICAL issue for each incomplete task from the selected source. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: @@ -111,7 +116,7 @@ ${STORE_SELECTION_GUIDANCE} - If contradiction detected: - Add WARNING: "Design decision not followed: " - Recommendation: "Update implementation or revise design.md to match reality" - - If \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. **Code Pattern Consistency** still runs. + - If \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. With other supporting artifacts, **Code Pattern Consistency** still runs; the task-only case remains limited to task completion. **Code Pattern Consistency**: - If implementation changes cannot be identified, mark **Code Pattern Consistency** as not verified and explain the missing evidence. @@ -239,16 +244,21 @@ ${STORE_SELECTION_GUIDANCE} Each dimension can have CRITICAL, WARNING, or SUGGESTION issues. + Verification is advisory. Respect intentional omissions such as \`skip_specs: true\`, optional design documents, and schemas without task tracking. Do not require or invent optional or intentionally omitted artifacts to obtain a clean report. \`Not verified\` describes a limit of this report, not a new archive prerequisite. Archive retains its own checks and user-confirmation behavior. + + If only task evidence is available, verify task completion only and mark the remaining checks, including **Code Pattern Consistency**, as not verified with the reason "Only task evidence available". + If artifacts cannot be read or contain no usable requirements, scenarios, or design decisions, mark the affected checks as not verified with the specific reason. Continue checks supported by the remaining evidence, but a partially checked input set is not a fully verified check. Missing requirements affect Spec Coverage and Requirement Implementation Mapping; missing scenarios affect Scenario Coverage; missing design decisions affect Design Adherence. 5. **Verify Completeness** **Task Completion**: - - Use the top-level \`tasks\` and \`progress\` fields; do not look for a \`contextFiles.tasks\` artifact id. - - If \`tasks\` is empty, even when \`progress.total\` is nonzero, no task descriptions can be evaluated. Mark **Task Completion** as not verified and record the reason from the apply \`state\` and \`instruction\`. - - Report complete vs total tasks from \`progress\`. - - If \`progress.remaining\` is greater than 0: - - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. + - Prefer the top-level \`tasks\` and \`progress\` fields; do not assume every schema uses a \`tasks\` artifact id. + - If \`tasks\` is empty but \`contextFiles.tasks\` contains paths, read every resolved file and count its checkboxes, including nested and blank checkboxes. This preserves verification for task globs that apply metadata does not represent. Label these as artifact-derived totals; do not report the zero apply totals as completion. If any file cannot be read, report the partial evidence and mark **Task Completion** as not verified. + - If neither source provides task descriptions, mark **Task Completion** as not verified and record the reason from the available evidence, including apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable tasks. + - Report complete vs total tasks from \`progress\`, or the artifact-derived totals when using the fallback. + - If \`progress.remaining\` is greater than 0 or the fallback files contain incomplete checkboxes: + - Add CRITICAL issue for each incomplete task from the selected source. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: @@ -290,7 +300,7 @@ ${STORE_SELECTION_GUIDANCE} - If contradiction detected: - Add WARNING: "Design decision not followed: " - Recommendation: "Update implementation or revise design.md to match reality" - - If \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. **Code Pattern Consistency** still runs. + - If \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. With other supporting artifacts, **Code Pattern Consistency** still runs; the task-only case remains limited to task completion. **Code Pattern Consistency**: - If implementation changes cannot be identified, mark **Code Pattern Consistency** as not verified and explain the missing evidence. diff --git a/test/commands/apply-instructions-tasks.test.ts b/test/commands/apply-instructions-tasks.test.ts index 5549087668..49df1c4a27 100644 --- a/test/commands/apply-instructions-tasks.test.ts +++ b/test/commands/apply-instructions-tasks.test.ts @@ -79,6 +79,51 @@ ${tracked ? ' tracks: checklist.md\n' : ''}` expect(instructions.instruction).toContain(kind === 'missing' ? 'Missing artifacts: tasks' : 'contains no tasks'); }); + it('exposes resolved task artifacts when apply cannot parse a tracking glob', async () => { + const schemaDir = path.join(tempDir, 'openspec', 'schemas', 'glob-tasks'); + fs.mkdirSync(schemaDir, { recursive: true }); + fs.writeFileSync( + path.join(schemaDir, 'schema.yaml'), + `name: glob-tasks +version: 1 +artifacts: + - id: tasks + generates: "**/tasks.md" + description: Implementation checklists + template: tasks.md + requires: [] +apply: + requires: [tasks] + tracks: "**/tasks.md" +` + ); + fs.writeFileSync(path.join(changeDir, '.openspec.yaml'), 'schema: glob-tasks\n'); + const backendTasks = path.join(changeDir, 'backend', 'tasks.md'); + const frontendTasks = path.join(changeDir, 'frontend', 'tasks.md'); + fs.mkdirSync(path.dirname(backendTasks), { recursive: true }); + fs.mkdirSync(path.dirname(frontendTasks), { recursive: true }); + fs.writeFileSync(backendTasks, '- [x] Finished backend task\n'); + fs.writeFileSync(frontendTasks, '- [x] Finished frontend task\n- [ ] Pending frontend task\n'); + + const instructions = await generateApplyInstructions(tempDir, 'my-change'); + const listProgress = await getTaskProgressForChange( + path.join(tempDir, 'openspec', 'changes'), + 'my-change', + tempDir + ); + + expect(instructions.contextFiles.tasks).toEqual([ + fs.realpathSync.native(backendTasks), + fs.realpathSync.native(frontendTasks), + ]); + // Apply reads tracks as one literal path; verify can still inspect the + // resolved artifacts instead of losing the unfinished task in those files. + expect(instructions.tasks).toEqual([]); + expect(instructions.progress).toEqual({ total: 0, complete: 0, remaining: 0 }); + expect(instructions.state).toBe('blocked'); + expect(listProgress).toEqual({ total: 3, completed: 2 }); + }); + it('returns existing spec and design paths even when their files contain no evidence', async () => { writeTasks('- [x] Finished task\n'); const spec = path.join(changeDir, 'specs', 'demo', 'spec.md'); diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index 1f727d5d33..c76dbd9384 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -53,11 +53,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '56bfada1a5f35a127791b70de9d428a75b5aedd1584d6c9803a1ecb1fd1b4a23', getBulkArchiveChangeSkillTemplate: '93875998cade5322d95b43299fba794bc1da754e917dd63a770406386a6d295d', getOpsxSyncCommandTemplate: '0d2427efb79986e8fff3f96bd075a739c80d45eb29159fae717e950030da8202', - getVerifyChangeSkillTemplate: '7d9ed0264ec431e875837db373c1b419b86910918522c264331e50f97b12a60c', + getVerifyChangeSkillTemplate: 'bc090496c01d64a8f48dc84340aa324ffebb20daa5f998f48e9980193088ce9b', getOpsxArchiveCommandTemplate: '9f973c819b11620985b03322945f0e0a92a02a2ef455b94e74482f5e6292ac5d', getOpsxOnboardCommandTemplate: 'ee99aa99252c602720fbb8c63fb3ac438a5bd4e952fd961ddf1ae956cbfc2c8f', getOpsxBulkArchiveCommandTemplate: '9fa8cdebe2f5667ebfc37bdc023396762c59d5b038c771dac2d8fd2c19e2627b', - getOpsxVerifyCommandTemplate: 'ce191e2dafabe720100eb75025da25369f7990f62321fcc62870c360b9cd4fde', + getOpsxVerifyCommandTemplate: '84dbb7d95ce08a4da72fa23b940a8ce05d4a2da9b56b169be97dd493f19dd1d3', getOpsxProposeSkillTemplate: '24623c066f97e34b957d448d1f9a9e8b8a13da3dfce45d45671f6226a2534848', getOpsxProposeCommandTemplate: 'e67ba591efb0fecacb2229d06dfa84af18b825fab8a7b01377279e4f09a06ce4', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -74,7 +74,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': 'd933d8856584d6c1253de91e652e7aee9e85c77ad4d3531f6476f79d84e6e5e8', 'openspec-archive-change': '7c65053d674ba4e1e20e2bf73ba7e5a7f94baef2eaa9b33cee48d4cadea51b7a', 'openspec-bulk-archive-change': '2039b9ecf6e64339dffe0e16272507a386d9fe326f419ff758315aa736fdd96c', - 'openspec-verify-change': '1b7a2ddf025f163cdac00c2e7227858ae349c5703fd1f9be69b9edce6513e6cc', + 'openspec-verify-change': 'a11d6b6d2ad1194999a946d781117850af3c79c4c3f34a8ca4a29ea72abada35', 'openspec-onboard': 'f6f59476acaf5e4d65dbb180da4cef62432612f3cecf207d471a951295e2003a', 'openspec-propose': '25d08ed4f031770cea219604167d76bca9f3e89fe0c2f545263674482c6f13f0', 'openspec-update-change': '586547406aca94422dfeb3ffedce6c01049429b743f57ce829baa79ebc714d51', diff --git a/test/core/templates/verify-change.test.ts b/test/core/templates/verify-change.test.ts index 3a4ff37b92..ead1cf8ce1 100644 --- a/test/core/templates/verify-change.test.ts +++ b/test/core/templates/verify-change.test.ts @@ -24,13 +24,22 @@ describe('verify-change templates', () => { } }); - it('uses schema-aware apply task fields instead of a hardcoded tasks artifact id', () => { + it('prefers schema-aware apply task fields without assuming a tasks artifact id', () => { for (const [label, body] of bodies) { expect(body, label).toContain('top-level `tasks` and `progress`'); expect(body, label).toContain("schema's `apply.tracks` configuration"); - expect(body, label).toContain('do not look for a `contextFiles.tasks` artifact id'); - expect(body, label).toContain('If `tasks` is empty, even when `progress.total` is nonzero'); - expect(body, label).not.toContain('`contextFiles.tasks` exists'); + expect(body, label).toContain('do not assume every schema uses a `tasks` artifact id'); + expect(body, label).toContain('If neither source provides task descriptions'); + } + }); + + it('preserves task checks when apply metadata cannot represent glob artifacts', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('If `tasks` is empty but `contextFiles.tasks` contains paths'); + expect(body, label).toContain('read every resolved file and count its checkboxes'); + expect(body, label).toContain('Label these as artifact-derived totals'); + expect(body, label).toContain('do not report the zero apply totals as completion'); + expect(body, label).toContain('or the fallback files contain incomplete checkboxes'); } }); @@ -58,6 +67,25 @@ describe('verify-change templates', () => { } }); + it('preserves optional artifacts and the existing archive workflow', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('Verification is advisory'); + expect(body, label).toContain('`skip_specs: true`'); + expect(body, label).toContain('schemas without task tracking'); + expect(body, label).toContain('Do not require or invent optional or intentionally omitted artifacts'); + expect(body, label).toContain('`Not verified` describes a limit of this report, not a new archive prerequisite'); + expect(body, label).toContain('Archive retains its own checks and user-confirmation behavior'); + } + }); + + it('preserves task-only verification without dropping checks supported by other artifacts', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('If only task evidence is available, verify task completion only'); + expect(body, label).toContain('including **Code Pattern Consistency**, as not verified'); + expect(body, label).toContain('With other supporting artifacts, **Code Pattern Consistency** still runs'); + } + }); + it('covers warning and suggestion outcomes without claiming all checks passed', () => { for (const [label, body] of bodies) { expect(body, label).toContain('If no CRITICAL issues, one or more warnings, and no checks were skipped'); From 459af39c25f4ea17d16a6b5fc1685704a352afab Mon Sep 17 00:00:00 2001 From: Clay Good Date: Mon, 31 Aug 2026 10:16:10 -0500 Subject: [PATCH 06/14] fix(apply): resolve tracked task globs by schema path --- .../verify-report-skipped-dimensions.md | 2 +- openspec/specs/opsx-verify-skill/spec.md | 11 +++----- skills/openspec-verify-change/SKILL.md | 13 +++++----- src/commands/workflow/instructions.ts | 13 +++++----- src/core/templates/workflows/verify-change.ts | 26 +++++++++---------- .../commands/apply-instructions-tasks.test.ts | 20 +++++++------- .../templates/skill-templates-parity.test.ts | 6 ++--- test/core/templates/verify-change.test.ts | 14 +++------- 8 files changed, 46 insertions(+), 59 deletions(-) diff --git a/.changeset/verify-report-skipped-dimensions.md b/.changeset/verify-report-skipped-dimensions.md index d553c1a155..540bf6ca36 100644 --- a/.changeset/verify-report-skipped-dimensions.md +++ b/.changeset/verify-report-skipped-dimensions.md @@ -2,4 +2,4 @@ "@fission-ai/openspec": patch --- -Stop `/opsx:verify` from reporting skipped checks as passing. Task completion now uses the schema-aware `tasks` and `progress` fields returned by apply instructions, while absent spec or design inputs are mapped to every check they prevent. When apply metadata has no task entries, verification still reads available task artifact files, including glob outputs. Verification stays advisory and does not require optional or intentionally omitted artifacts. The scorecard identifies each skipped check, and the final assessment does not claim archive readiness when any check did not run. +Stop `/opsx:verify` from reporting skipped checks as passing. Task completion now uses the schema-aware `tasks` and `progress` fields returned by apply instructions, while absent spec or design inputs are mapped to every check they prevent. Apply instructions aggregate every file matched by the configured task path or glob, regardless of the tracked artifact ID. Verification stays advisory and does not require optional or intentionally omitted artifacts. The scorecard identifies each skipped check, and the final assessment does not claim archive readiness when any check did not run. diff --git a/openspec/specs/opsx-verify-skill/spec.md b/openspec/specs/opsx-verify-skill/spec.md index b18202a334..ebae7b772c 100644 --- a/openspec/specs/opsx-verify-skill/spec.md +++ b/openspec/specs/opsx-verify-skill/spec.md @@ -19,7 +19,7 @@ The system SHALL provide an `/opsx:verify` skill that validates implementation a - **AND** announces which change was selected and how to override #### Scenario: Change has no task descriptions -- **WHEN** neither the structured task list nor the resolved task artifact fallback provides usable task descriptions, even if task progress reports nonzero totals +- **WHEN** the structured task list provides no usable task descriptions, even if task progress reports nonzero totals - **THEN** the agent reports Task Completion as not verified with the reason - **AND** continues checks supported by the remaining artifacts - **AND** does not require tasks when the schema does not track them @@ -29,17 +29,12 @@ The agent SHALL verify that all required work has been completed. #### Scenario: Task completion check - **WHEN** verifying completeness -- **THEN** the agent prefers the top-level `tasks` and `progress` from apply instructions, resolved from the active schema's `apply.tracks` configuration +- **THEN** the agent uses the top-level `tasks` and `progress` from apply instructions +- **AND** apply instructions aggregate every concrete file matched by the active schema's `apply.tracks`, regardless of the tracked artifact's ID - **AND** reports complete and total task counts from `progress` - **AND** reports completion status with specific incomplete tasks listed - **AND** reports remaining checkboxes without descriptions when `progress.remaining` exceeds the listed incomplete tasks -#### Scenario: Resolved task artifact fallback -- **WHEN** structured apply tasks are empty but resolved files exist in `contextFiles.tasks` -- **THEN** read every resolved task artifact and derive task descriptions and checkbox totals from those files -- **AND** identify the totals as artifact-derived rather than presenting zero apply progress as task completion -- **AND** report Task Completion as not verified if the files cannot be read or provide no usable task descriptions - #### Scenario: Spec coverage check - **WHEN** verifying completeness - **AND** delta specs exist in `openspec/changes//specs/` diff --git a/skills/openspec-verify-change/SKILL.md b/skills/openspec-verify-change/SKILL.md index 560a2064b9..8b7b2a772f 100644 --- a/skills/openspec-verify-change/SKILL.md +++ b/skills/openspec-verify-change/SKILL.md @@ -45,7 +45,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design openspec instructions apply --change "" --json ``` - This returns the change directory, `contextFiles` (artifact ID -> array of concrete file paths), and top-level `tasks` and `progress` resolved from the schema's `apply.tracks` configuration. Read all available artifacts from `contextFiles`. + This returns the change directory, `contextFiles` (artifact ID -> array of concrete file paths), and top-level `tasks` and `progress` aggregated from every concrete file matched by the schema's `apply.tracks` configuration. Read all available artifacts from `contextFiles`. Treat apply `state` and `instruction` as context, not a verification verdict. Do not implement tasks or archive the change during verification. @@ -67,12 +67,11 @@ Verify that an implementation matches the change artifacts (specs, tasks, design 5. **Verify Completeness** **Task Completion**: - - Prefer the top-level `tasks` and `progress` fields; do not assume every schema uses a `tasks` artifact id. - - If `tasks` is empty but `contextFiles.tasks` contains paths, read every resolved file and count its checkboxes, including nested and blank checkboxes. This preserves verification for task globs that apply metadata does not represent. Label these as artifact-derived totals; do not report the zero apply totals as completion. If any file cannot be read, report the partial evidence and mark **Task Completion** as not verified. - - If neither source provides task descriptions, mark **Task Completion** as not verified and record the reason from the available evidence, including apply `state` and `instruction`. Nonzero totals alone do not establish evaluable tasks. - - Report complete vs total tasks from `progress`, or the artifact-derived totals when using the fallback. - - If `progress.remaining` is greater than 0 or the fallback files contain incomplete checkboxes: - - Add CRITICAL issue for each incomplete task from the selected source. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. + - Use the top-level `tasks` and `progress` fields. They already aggregate every concrete file matched by `apply.tracks`, regardless of the tracked artifact's ID; do not infer tracking from a `contextFiles` key. + - If `tasks` is empty, mark **Task Completion** as not verified and record the reason from apply `state` and `instruction`. Nonzero totals alone do not establish evaluable task descriptions. + - Report complete vs total tasks from `progress`. + - If `progress.remaining` is greater than 0: + - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: diff --git a/src/commands/workflow/instructions.ts b/src/commands/workflow/instructions.ts index 1ae6fac7c0..dfd91e23ea 100644 --- a/src/commands/workflow/instructions.ts +++ b/src/commands/workflow/instructions.ts @@ -12,7 +12,6 @@ import { loadChangeContext, generateInstructions, resolveSchema, - resolveArtifactOutputPath, resolveArtifactOutputs, type ArtifactInstructions, } from '../../core/artifact-graph/index.js'; @@ -412,15 +411,17 @@ export async function generateApplyInstructions( } } - // Parse tasks if tracking file exists + // Parse every concrete file matched by apply.tracks. A tracking path may be + // a glob owned by an artifact with any ID, so treating it as one literal + // path loses task evidence for valid custom schemas. let parsedTasks: ParsedTask[] = []; let tracksFileExists = false; if (tracksFile) { - const tracksPath = resolveArtifactOutputPath(changeDir, tracksFile); - tracksFileExists = fs.existsSync(tracksPath); - if (tracksFileExists) { + const tracksPaths = resolveArtifactOutputs(changeDir, tracksFile); + tracksFileExists = tracksPaths.length > 0; + for (const tracksPath of tracksPaths) { const tasksContent = await fs.promises.readFile(tracksPath, 'utf-8'); - parsedTasks = parseTaskLines(tasksContent); + parsedTasks.push(...parseTaskLines(tasksContent)); } } const tasks = toTaskItems(parsedTasks); diff --git a/src/core/templates/workflows/verify-change.ts b/src/core/templates/workflows/verify-change.ts index 5650197ec1..da6660e0f6 100644 --- a/src/core/templates/workflows/verify-change.ts +++ b/src/core/templates/workflows/verify-change.ts @@ -47,7 +47,7 @@ ${STORE_SELECTION_GUIDANCE} openspec instructions apply --change "" --json \`\`\` - This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` resolved from the schema's \`apply.tracks\` configuration. Read all available artifacts from \`contextFiles\`. + This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` aggregated from every concrete file matched by the schema's \`apply.tracks\` configuration. Read all available artifacts from \`contextFiles\`. Treat apply \`state\` and \`instruction\` as context, not a verification verdict. Do not implement tasks or archive the change during verification. @@ -69,12 +69,11 @@ ${STORE_SELECTION_GUIDANCE} 5. **Verify Completeness** **Task Completion**: - - Prefer the top-level \`tasks\` and \`progress\` fields; do not assume every schema uses a \`tasks\` artifact id. - - If \`tasks\` is empty but \`contextFiles.tasks\` contains paths, read every resolved file and count its checkboxes, including nested and blank checkboxes. This preserves verification for task globs that apply metadata does not represent. Label these as artifact-derived totals; do not report the zero apply totals as completion. If any file cannot be read, report the partial evidence and mark **Task Completion** as not verified. - - If neither source provides task descriptions, mark **Task Completion** as not verified and record the reason from the available evidence, including apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable tasks. - - Report complete vs total tasks from \`progress\`, or the artifact-derived totals when using the fallback. - - If \`progress.remaining\` is greater than 0 or the fallback files contain incomplete checkboxes: - - Add CRITICAL issue for each incomplete task from the selected source. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. + - Use the top-level \`tasks\` and \`progress\` fields. They already aggregate every concrete file matched by \`apply.tracks\`, regardless of the tracked artifact's ID; do not infer tracking from a \`contextFiles\` key. + - If \`tasks\` is empty, mark **Task Completion** as not verified and record the reason from apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable task descriptions. + - Report complete vs total tasks from \`progress\`. + - If \`progress.remaining\` is greater than 0: + - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: @@ -231,7 +230,7 @@ ${STORE_SELECTION_GUIDANCE} openspec instructions apply --change "" --json \`\`\` - This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` resolved from the schema's \`apply.tracks\` configuration. Read all available artifacts from \`contextFiles\`. + This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` aggregated from every concrete file matched by the schema's \`apply.tracks\` configuration. Read all available artifacts from \`contextFiles\`. Treat apply \`state\` and \`instruction\` as context, not a verification verdict. Do not implement tasks or archive the change during verification. @@ -253,12 +252,11 @@ ${STORE_SELECTION_GUIDANCE} 5. **Verify Completeness** **Task Completion**: - - Prefer the top-level \`tasks\` and \`progress\` fields; do not assume every schema uses a \`tasks\` artifact id. - - If \`tasks\` is empty but \`contextFiles.tasks\` contains paths, read every resolved file and count its checkboxes, including nested and blank checkboxes. This preserves verification for task globs that apply metadata does not represent. Label these as artifact-derived totals; do not report the zero apply totals as completion. If any file cannot be read, report the partial evidence and mark **Task Completion** as not verified. - - If neither source provides task descriptions, mark **Task Completion** as not verified and record the reason from the available evidence, including apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable tasks. - - Report complete vs total tasks from \`progress\`, or the artifact-derived totals when using the fallback. - - If \`progress.remaining\` is greater than 0 or the fallback files contain incomplete checkboxes: - - Add CRITICAL issue for each incomplete task from the selected source. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. + - Use the top-level \`tasks\` and \`progress\` fields. They already aggregate every concrete file matched by \`apply.tracks\`, regardless of the tracked artifact's ID; do not infer tracking from a \`contextFiles\` key. + - If \`tasks\` is empty, mark **Task Completion** as not verified and record the reason from apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable task descriptions. + - Report complete vs total tasks from \`progress\`. + - If \`progress.remaining\` is greater than 0: + - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: diff --git a/test/commands/apply-instructions-tasks.test.ts b/test/commands/apply-instructions-tasks.test.ts index 49df1c4a27..e61868d56d 100644 --- a/test/commands/apply-instructions-tasks.test.ts +++ b/test/commands/apply-instructions-tasks.test.ts @@ -79,7 +79,7 @@ ${tracked ? ' tracks: checklist.md\n' : ''}` expect(instructions.instruction).toContain(kind === 'missing' ? 'Missing artifacts: tasks' : 'contains no tasks'); }); - it('exposes resolved task artifacts when apply cannot parse a tracking glob', async () => { + it('aggregates a tracking glob owned by an artifact not named tasks', async () => { const schemaDir = path.join(tempDir, 'openspec', 'schemas', 'glob-tasks'); fs.mkdirSync(schemaDir, { recursive: true }); fs.writeFileSync( @@ -87,13 +87,13 @@ ${tracked ? ' tracks: checklist.md\n' : ''}` `name: glob-tasks version: 1 artifacts: - - id: tasks + - id: implementation generates: "**/tasks.md" description: Implementation checklists template: tasks.md requires: [] apply: - requires: [tasks] + requires: [implementation] tracks: "**/tasks.md" ` ); @@ -112,15 +112,17 @@ apply: tempDir ); - expect(instructions.contextFiles.tasks).toEqual([ + expect(instructions.contextFiles.implementation).toEqual([ fs.realpathSync.native(backendTasks), fs.realpathSync.native(frontendTasks), ]); - // Apply reads tracks as one literal path; verify can still inspect the - // resolved artifacts instead of losing the unfinished task in those files. - expect(instructions.tasks).toEqual([]); - expect(instructions.progress).toEqual({ total: 0, complete: 0, remaining: 0 }); - expect(instructions.state).toBe('blocked'); + expect(instructions.tasks).toEqual([ + { id: '1', description: 'Finished backend task', done: true }, + { id: '2', description: 'Finished frontend task', done: true }, + { id: '3', description: 'Pending frontend task', done: false }, + ]); + expect(instructions.progress).toEqual({ total: 3, complete: 2, remaining: 1 }); + expect(instructions.state).toBe('ready'); expect(listProgress).toEqual({ total: 3, completed: 2 }); }); diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index c76dbd9384..44c0877430 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -53,11 +53,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '56bfada1a5f35a127791b70de9d428a75b5aedd1584d6c9803a1ecb1fd1b4a23', getBulkArchiveChangeSkillTemplate: '93875998cade5322d95b43299fba794bc1da754e917dd63a770406386a6d295d', getOpsxSyncCommandTemplate: '0d2427efb79986e8fff3f96bd075a739c80d45eb29159fae717e950030da8202', - getVerifyChangeSkillTemplate: 'bc090496c01d64a8f48dc84340aa324ffebb20daa5f998f48e9980193088ce9b', + getVerifyChangeSkillTemplate: '6999f85c7bf05d7dd5a9dce13a3cde5afae50e7a64bd30a25e230a705a05d30b', getOpsxArchiveCommandTemplate: '9f973c819b11620985b03322945f0e0a92a02a2ef455b94e74482f5e6292ac5d', getOpsxOnboardCommandTemplate: 'ee99aa99252c602720fbb8c63fb3ac438a5bd4e952fd961ddf1ae956cbfc2c8f', getOpsxBulkArchiveCommandTemplate: '9fa8cdebe2f5667ebfc37bdc023396762c59d5b038c771dac2d8fd2c19e2627b', - getOpsxVerifyCommandTemplate: '84dbb7d95ce08a4da72fa23b940a8ce05d4a2da9b56b169be97dd493f19dd1d3', + getOpsxVerifyCommandTemplate: 'd8bf0d0b58c12082ce763fa7cb04d91f4023dd178f9eaf974aa0956942e56599', getOpsxProposeSkillTemplate: '24623c066f97e34b957d448d1f9a9e8b8a13da3dfce45d45671f6226a2534848', getOpsxProposeCommandTemplate: 'e67ba591efb0fecacb2229d06dfa84af18b825fab8a7b01377279e4f09a06ce4', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -74,7 +74,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': 'd933d8856584d6c1253de91e652e7aee9e85c77ad4d3531f6476f79d84e6e5e8', 'openspec-archive-change': '7c65053d674ba4e1e20e2bf73ba7e5a7f94baef2eaa9b33cee48d4cadea51b7a', 'openspec-bulk-archive-change': '2039b9ecf6e64339dffe0e16272507a386d9fe326f419ff758315aa736fdd96c', - 'openspec-verify-change': 'a11d6b6d2ad1194999a946d781117850af3c79c4c3f34a8ca4a29ea72abada35', + 'openspec-verify-change': '86e1436eea812140b3cbbba0333a5faae6ea9b28a4818fca8808e9e1b135f23a', 'openspec-onboard': 'f6f59476acaf5e4d65dbb180da4cef62432612f3cecf207d471a951295e2003a', 'openspec-propose': '25d08ed4f031770cea219604167d76bca9f3e89fe0c2f545263674482c6f13f0', 'openspec-update-change': '586547406aca94422dfeb3ffedce6c01049429b743f57ce829baa79ebc714d51', diff --git a/test/core/templates/verify-change.test.ts b/test/core/templates/verify-change.test.ts index ead1cf8ce1..db41a1ea34 100644 --- a/test/core/templates/verify-change.test.ts +++ b/test/core/templates/verify-change.test.ts @@ -28,20 +28,12 @@ describe('verify-change templates', () => { for (const [label, body] of bodies) { expect(body, label).toContain('top-level `tasks` and `progress`'); expect(body, label).toContain("schema's `apply.tracks` configuration"); - expect(body, label).toContain('do not assume every schema uses a `tasks` artifact id'); - expect(body, label).toContain('If neither source provides task descriptions'); + expect(body, label).toContain('aggregated from every concrete file matched'); + expect(body, label).toContain("regardless of the tracked artifact's ID"); + expect(body, label).toContain('do not infer tracking from a `contextFiles` key'); } }); - it('preserves task checks when apply metadata cannot represent glob artifacts', () => { - for (const [label, body] of bodies) { - expect(body, label).toContain('If `tasks` is empty but `contextFiles.tasks` contains paths'); - expect(body, label).toContain('read every resolved file and count its checkboxes'); - expect(body, label).toContain('Label these as artifact-derived totals'); - expect(body, label).toContain('do not report the zero apply totals as completion'); - expect(body, label).toContain('or the fallback files contain incomplete checkboxes'); - } - }); it('does not lose incomplete checkboxes omitted from the task list', () => { for (const [label, body] of bodies) { From 4a6494c5795f3d39736de0e6c37c9833e3beaa41 Mon Sep 17 00:00:00 2001 From: Clay Good Date: Wed, 2 Sep 2026 10:33:38 -0500 Subject: [PATCH 07/14] fix(verify): preserve unavailable task evidence --- openspec/specs/opsx-verify-skill/spec.md | 7 ++ skills/openspec-verify-change/SKILL.md | 5 +- src/commands/workflow/instructions.ts | 32 ++++++- src/commands/workflow/shared.ts | 4 + src/core/templates/workflows/verify-change.ts | 10 ++- .../commands/apply-instructions-tasks.test.ts | 88 +++++++++++++++---- .../templates/skill-templates-parity.test.ts | 6 +- test/core/templates/verify-change.test.ts | 8 ++ 8 files changed, 129 insertions(+), 31 deletions(-) diff --git a/openspec/specs/opsx-verify-skill/spec.md b/openspec/specs/opsx-verify-skill/spec.md index ebae7b772c..d4790254a3 100644 --- a/openspec/specs/opsx-verify-skill/spec.md +++ b/openspec/specs/opsx-verify-skill/spec.md @@ -35,6 +35,13 @@ The agent SHALL verify that all required work has been completed. - **AND** reports completion status with specific incomplete tasks listed - **AND** reports remaining checkboxes without descriptions when `progress.remaining` exceeds the listed incomplete tasks +#### Scenario: Tracking evidence becomes unavailable +- **WHEN** one or more files matched by `apply.tracks` cannot be read after resolution +- **THEN** apply instructions include every unavailable path and reason +- **AND** preserve tasks and progress from readable tracking files +- **AND** do not report `all_done` +- **AND** the agent marks Task Completion as not verified from partial evidence + #### Scenario: Spec coverage check - **WHEN** verifying completeness - **AND** delta specs exist in `openspec/changes//specs/` diff --git a/skills/openspec-verify-change/SKILL.md b/skills/openspec-verify-change/SKILL.md index 8b7b2a772f..8c921fb3d8 100644 --- a/skills/openspec-verify-change/SKILL.md +++ b/skills/openspec-verify-change/SKILL.md @@ -45,7 +45,7 @@ Verify that an implementation matches the change artifacts (specs, tasks, design openspec instructions apply --change "" --json ``` - This returns the change directory, `contextFiles` (artifact ID -> array of concrete file paths), and top-level `tasks` and `progress` aggregated from every concrete file matched by the schema's `apply.tracks` configuration. Read all available artifacts from `contextFiles`. + This returns the change directory, `contextFiles` (artifact ID -> array of concrete file paths), and top-level `tasks` and `progress` aggregated from every concrete file matched by the schema's `apply.tracks` configuration that could be read. Read all available artifacts from `contextFiles`. Treat apply `state` and `instruction` as context, not a verification verdict. Do not implement tasks or archive the change during verification. @@ -67,7 +67,8 @@ Verify that an implementation matches the change artifacts (specs, tasks, design 5. **Verify Completeness** **Task Completion**: - - Use the top-level `tasks` and `progress` fields. They already aggregate every concrete file matched by `apply.tracks`, regardless of the tracked artifact's ID; do not infer tracking from a `contextFiles` key. + - Use the top-level `tasks` and `progress` fields. They already aggregate every readable concrete file matched by `apply.tracks`, regardless of the tracked artifact's ID; do not infer tracking from a `contextFiles` key. + - If `unavailableTrackingFiles` is nonempty, mark **Task Completion** as not verified and include every unavailable path and reason. Continue using any readable task evidence, but do not infer completion from the partial `tasks` and `progress` fields. - If `tasks` is empty, mark **Task Completion** as not verified and record the reason from apply `state` and `instruction`. Nonzero totals alone do not establish evaluable task descriptions. - Report complete vs total tasks from `progress`. - If `progress.remaining` is greater than 0: diff --git a/src/commands/workflow/instructions.ts b/src/commands/workflow/instructions.ts index dfd91e23ea..6dc5d66ff3 100644 --- a/src/commands/workflow/instructions.ts +++ b/src/commands/workflow/instructions.ts @@ -415,13 +415,23 @@ export async function generateApplyInstructions( // a glob owned by an artifact with any ID, so treating it as one literal // path loses task evidence for valid custom schemas. let parsedTasks: ParsedTask[] = []; + const unavailableTrackingFiles: Array<{ path: string; reason: string }> = []; let tracksFileExists = false; if (tracksFile) { const tracksPaths = resolveArtifactOutputs(changeDir, tracksFile); tracksFileExists = tracksPaths.length > 0; for (const tracksPath of tracksPaths) { - const tasksContent = await fs.promises.readFile(tracksPath, 'utf-8'); - parsedTasks.push(...parseTaskLines(tasksContent)); + try { + const tasksContent = await fs.promises.readFile(tracksPath, 'utf-8'); + parsedTasks.push(...parseTaskLines(tasksContent)); + } catch (error) { + const code = (error as NodeJS.ErrnoException)?.code; + const message = error instanceof Error ? error.message : String(error); + unavailableTrackingFiles.push({ + path: tracksPath, + reason: code && !message.includes(code) ? `${code}: ${message}` : message, + }); + } } } const tasks = toTaskItems(parsedTasks); @@ -444,13 +454,21 @@ export async function generateApplyInstructions( const tracksFilename = path.basename(tracksFile); state = 'blocked'; instruction = `The ${tracksFilename} file is missing and must be created.\nUse openspec-continue-change to generate the tracking file.`; + } else if (tracksFile && unavailableTrackingFiles.length > 0 && tasks.length === 0) { + state = 'blocked'; + instruction = 'No readable task descriptions are available.'; } else if (tracksFile && tracksFileExists && tasks.length === 0) { // Tracking file exists but lists nothing an agent can work on: either no // checkboxes at all, or only checkboxes with no text after them. const tracksFilename = path.basename(tracksFile); state = 'blocked'; instruction = `The ${tracksFilename} file exists but contains no tasks to work on.\nAdd tasks to ${tracksFilename} or regenerate it with openspec-continue-change.`; - } else if (tracksFile && remaining === 0 && total > 0) { + } else if ( + tracksFile && + unavailableTrackingFiles.length === 0 && + remaining === 0 && + total > 0 + ) { state = 'all_done'; instruction = 'All tasks are complete! This change is ready to be archived.\nConsider running tests and reviewing the changes before archiving.'; } else if (!tracksFile) { @@ -462,6 +480,13 @@ export async function generateApplyInstructions( instruction = schemaInstruction?.trim() ?? 'Read context files, work through pending tasks, mark complete as you go.\nPause if you hit blockers or need clarification.'; } + if (unavailableTrackingFiles.length > 0) { + const unavailableDetails = unavailableTrackingFiles + .map((file) => `- ${file.path}: ${file.reason}`) + .join('\n'); + instruction += `\nTask completion is not verified because tracking evidence was unavailable:\n${unavailableDetails}`; + } + return { changeName, changeDir, @@ -469,6 +494,7 @@ export async function generateApplyInstructions( contextFiles, progress: { total, complete, remaining }, tasks, + ...(unavailableTrackingFiles.length > 0 ? { unavailableTrackingFiles } : {}), state, missingArtifacts: missingArtifacts.length > 0 ? missingArtifacts : undefined, instruction, diff --git a/src/commands/workflow/shared.ts b/src/commands/workflow/shared.ts index 2840e004ed..c6f0dd094f 100644 --- a/src/commands/workflow/shared.ts +++ b/src/commands/workflow/shared.ts @@ -41,6 +41,10 @@ export interface ApplyInstructions { remaining: number; }; tasks: TaskItem[]; + unavailableTrackingFiles?: Array<{ + path: string; + reason: string; + }>; state: 'blocked' | 'all_done' | 'ready'; missingArtifacts?: string[]; instruction: string; diff --git a/src/core/templates/workflows/verify-change.ts b/src/core/templates/workflows/verify-change.ts index da6660e0f6..9d4757604e 100644 --- a/src/core/templates/workflows/verify-change.ts +++ b/src/core/templates/workflows/verify-change.ts @@ -47,7 +47,7 @@ ${STORE_SELECTION_GUIDANCE} openspec instructions apply --change "" --json \`\`\` - This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` aggregated from every concrete file matched by the schema's \`apply.tracks\` configuration. Read all available artifacts from \`contextFiles\`. + This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` aggregated from every concrete file matched by the schema's \`apply.tracks\` configuration that could be read. Read all available artifacts from \`contextFiles\`. Treat apply \`state\` and \`instruction\` as context, not a verification verdict. Do not implement tasks or archive the change during verification. @@ -69,7 +69,8 @@ ${STORE_SELECTION_GUIDANCE} 5. **Verify Completeness** **Task Completion**: - - Use the top-level \`tasks\` and \`progress\` fields. They already aggregate every concrete file matched by \`apply.tracks\`, regardless of the tracked artifact's ID; do not infer tracking from a \`contextFiles\` key. + - Use the top-level \`tasks\` and \`progress\` fields. They already aggregate every readable concrete file matched by \`apply.tracks\`, regardless of the tracked artifact's ID; do not infer tracking from a \`contextFiles\` key. + - If \`unavailableTrackingFiles\` is nonempty, mark **Task Completion** as not verified and include every unavailable path and reason. Continue using any readable task evidence, but do not infer completion from the partial \`tasks\` and \`progress\` fields. - If \`tasks\` is empty, mark **Task Completion** as not verified and record the reason from apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable task descriptions. - Report complete vs total tasks from \`progress\`. - If \`progress.remaining\` is greater than 0: @@ -230,7 +231,7 @@ ${STORE_SELECTION_GUIDANCE} openspec instructions apply --change "" --json \`\`\` - This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` aggregated from every concrete file matched by the schema's \`apply.tracks\` configuration. Read all available artifacts from \`contextFiles\`. + This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` aggregated from every concrete file matched by the schema's \`apply.tracks\` configuration that could be read. Read all available artifacts from \`contextFiles\`. Treat apply \`state\` and \`instruction\` as context, not a verification verdict. Do not implement tasks or archive the change during verification. @@ -252,7 +253,8 @@ ${STORE_SELECTION_GUIDANCE} 5. **Verify Completeness** **Task Completion**: - - Use the top-level \`tasks\` and \`progress\` fields. They already aggregate every concrete file matched by \`apply.tracks\`, regardless of the tracked artifact's ID; do not infer tracking from a \`contextFiles\` key. + - Use the top-level \`tasks\` and \`progress\` fields. They already aggregate every readable concrete file matched by \`apply.tracks\`, regardless of the tracked artifact's ID; do not infer tracking from a \`contextFiles\` key. + - If \`unavailableTrackingFiles\` is nonempty, mark **Task Completion** as not verified and include every unavailable path and reason. Continue using any readable task evidence, but do not infer completion from the partial \`tasks\` and \`progress\` fields. - If \`tasks\` is empty, mark **Task Completion** as not verified and record the reason from apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable task descriptions. - Report complete vs total tasks from \`progress\`. - If \`progress.remaining\` is greater than 0: diff --git a/test/commands/apply-instructions-tasks.test.ts b/test/commands/apply-instructions-tasks.test.ts index e61868d56d..68b8147527 100644 --- a/test/commands/apply-instructions-tasks.test.ts +++ b/test/commands/apply-instructions-tasks.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import * as fs from 'node:fs'; import * as path from 'node:path'; import * as os from 'node:os'; @@ -27,6 +27,7 @@ describe('generateApplyInstructions task list', () => { }); afterEach(() => { + vi.restoreAllMocks(); fs.rmSync(tempDir, { recursive: true, force: true }); }); @@ -34,6 +35,27 @@ describe('generateApplyInstructions task list', () => { fs.writeFileSync(path.join(changeDir, 'tasks.md'), content); } + function writeGlobTasksSchema(): void { + const schemaDir = path.join(tempDir, 'openspec', 'schemas', 'glob-tasks'); + fs.mkdirSync(schemaDir, { recursive: true }); + fs.writeFileSync( + path.join(schemaDir, 'schema.yaml'), + `name: glob-tasks +version: 1 +artifacts: + - id: implementation + generates: "**/tasks.md" + description: Implementation checklists + template: tasks.md + requires: [] +apply: + requires: [implementation] + tracks: "**/tasks.md" +` + ); + fs.writeFileSync(path.join(changeDir, '.openspec.yaml'), 'schema: glob-tasks\n'); + } + it.each([true, false])('resolves custom tracking configuration (enabled: %s)', async (tracked) => { const schemaDir = path.join(tempDir, 'openspec', 'schemas', 'custom'); fs.mkdirSync(schemaDir, { recursive: true }); @@ -80,24 +102,7 @@ ${tracked ? ' tracks: checklist.md\n' : ''}` }); it('aggregates a tracking glob owned by an artifact not named tasks', async () => { - const schemaDir = path.join(tempDir, 'openspec', 'schemas', 'glob-tasks'); - fs.mkdirSync(schemaDir, { recursive: true }); - fs.writeFileSync( - path.join(schemaDir, 'schema.yaml'), - `name: glob-tasks -version: 1 -artifacts: - - id: implementation - generates: "**/tasks.md" - description: Implementation checklists - template: tasks.md - requires: [] -apply: - requires: [implementation] - tracks: "**/tasks.md" -` - ); - fs.writeFileSync(path.join(changeDir, '.openspec.yaml'), 'schema: glob-tasks\n'); + writeGlobTasksSchema(); const backendTasks = path.join(changeDir, 'backend', 'tasks.md'); const frontendTasks = path.join(changeDir, 'frontend', 'tasks.md'); fs.mkdirSync(path.dirname(backendTasks), { recursive: true }); @@ -126,6 +131,51 @@ apply: expect(listProgress).toEqual({ total: 3, completed: 2 }); }); + it('retains partial task evidence when a tracked file is unreadable', async () => { + writeGlobTasksSchema(); + const backendTasks = path.join(changeDir, 'backend', 'tasks.md'); + const frontendTasks = path.join(changeDir, 'frontend', 'tasks.md'); + fs.mkdirSync(path.dirname(backendTasks), { recursive: true }); + fs.mkdirSync(path.dirname(frontendTasks), { recursive: true }); + fs.writeFileSync(backendTasks, '- [x] Finished backend task\n'); + fs.writeFileSync(frontendTasks, '- [x] Finished frontend task\n'); + vi.spyOn(fs.promises, 'readFile').mockRejectedValueOnce( + Object.assign(new Error('permission denied'), { code: 'EACCES' }) + ); + + const instructions = await generateApplyInstructions(tempDir, 'my-change'); + + expect(instructions.tasks).toEqual([ + { id: '1', description: 'Finished frontend task', done: true }, + ]); + expect(instructions.progress).toEqual({ total: 1, complete: 1, remaining: 0 }); + expect(instructions.unavailableTrackingFiles).toEqual([ + { path: fs.realpathSync.native(backendTasks), reason: 'EACCES: permission denied' }, + ]); + expect(instructions.state).toBe('ready'); + expect(instructions.instruction).toContain('Task completion is not verified'); + expect(instructions.instruction).toContain(fs.realpathSync.native(backendTasks)); + }); + + it('reports tracking evidence that disappears after glob resolution', async () => { + writeTasks('- [x] Finished task\n'); + const tasksPath = fs.realpathSync.native(path.join(changeDir, 'tasks.md')); + vi.spyOn(fs.promises, 'readFile').mockRejectedValueOnce( + Object.assign(new Error('no such file or directory'), { code: 'ENOENT' }) + ); + + const instructions = await generateApplyInstructions(tempDir, 'my-change'); + + expect(instructions.tasks).toEqual([]); + expect(instructions.progress).toEqual({ total: 0, complete: 0, remaining: 0 }); + expect(instructions.unavailableTrackingFiles).toEqual([ + { path: tasksPath, reason: 'ENOENT: no such file or directory' }, + ]); + expect(instructions.state).toBe('blocked'); + expect(instructions.instruction).toContain('Task completion is not verified'); + expect(instructions.instruction).toContain(tasksPath); + }); + it('returns existing spec and design paths even when their files contain no evidence', async () => { writeTasks('- [x] Finished task\n'); const spec = path.join(changeDir, 'specs', 'demo', 'spec.md'); diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index 44c0877430..81ab67a1b3 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -53,11 +53,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '56bfada1a5f35a127791b70de9d428a75b5aedd1584d6c9803a1ecb1fd1b4a23', getBulkArchiveChangeSkillTemplate: '93875998cade5322d95b43299fba794bc1da754e917dd63a770406386a6d295d', getOpsxSyncCommandTemplate: '0d2427efb79986e8fff3f96bd075a739c80d45eb29159fae717e950030da8202', - getVerifyChangeSkillTemplate: '6999f85c7bf05d7dd5a9dce13a3cde5afae50e7a64bd30a25e230a705a05d30b', + getVerifyChangeSkillTemplate: '3014e7439c074df105c2be12a26a8a6a0e3cbe31b61e72d78c7f4b987d2a7cd4', getOpsxArchiveCommandTemplate: '9f973c819b11620985b03322945f0e0a92a02a2ef455b94e74482f5e6292ac5d', getOpsxOnboardCommandTemplate: 'ee99aa99252c602720fbb8c63fb3ac438a5bd4e952fd961ddf1ae956cbfc2c8f', getOpsxBulkArchiveCommandTemplate: '9fa8cdebe2f5667ebfc37bdc023396762c59d5b038c771dac2d8fd2c19e2627b', - getOpsxVerifyCommandTemplate: 'd8bf0d0b58c12082ce763fa7cb04d91f4023dd178f9eaf974aa0956942e56599', + getOpsxVerifyCommandTemplate: 'cb72180fb07529805fc36443ad14bf410cdbcd7397ece249112fd896e8a75f2d', getOpsxProposeSkillTemplate: '24623c066f97e34b957d448d1f9a9e8b8a13da3dfce45d45671f6226a2534848', getOpsxProposeCommandTemplate: 'e67ba591efb0fecacb2229d06dfa84af18b825fab8a7b01377279e4f09a06ce4', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -74,7 +74,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': 'd933d8856584d6c1253de91e652e7aee9e85c77ad4d3531f6476f79d84e6e5e8', 'openspec-archive-change': '7c65053d674ba4e1e20e2bf73ba7e5a7f94baef2eaa9b33cee48d4cadea51b7a', 'openspec-bulk-archive-change': '2039b9ecf6e64339dffe0e16272507a386d9fe326f419ff758315aa736fdd96c', - 'openspec-verify-change': '86e1436eea812140b3cbbba0333a5faae6ea9b28a4818fca8808e9e1b135f23a', + 'openspec-verify-change': '1703951f6cfcd7ae7e7e581f6bdd59fa08b8b87be3071040550f3bead3a190ba', 'openspec-onboard': 'f6f59476acaf5e4d65dbb180da4cef62432612f3cecf207d471a951295e2003a', 'openspec-propose': '25d08ed4f031770cea219604167d76bca9f3e89fe0c2f545263674482c6f13f0', 'openspec-update-change': '586547406aca94422dfeb3ffedce6c01049429b743f57ce829baa79ebc714d51', diff --git a/test/core/templates/verify-change.test.ts b/test/core/templates/verify-change.test.ts index db41a1ea34..5c434949c3 100644 --- a/test/core/templates/verify-change.test.ts +++ b/test/core/templates/verify-change.test.ts @@ -34,6 +34,14 @@ describe('verify-change templates', () => { } }); + it('marks partial tracking evidence as not verified', () => { + for (const [label, body] of bodies) { + expect(body, label).toContain('If `unavailableTrackingFiles` is nonempty'); + expect(body, label).toContain('include every unavailable path and reason'); + expect(body, label).toContain('do not infer completion from the partial `tasks` and `progress` fields'); + } + }); + it('does not lose incomplete checkboxes omitted from the task list', () => { for (const [label, body] of bodies) { From 9b08d0d5144d2e900c54873aff7b9288d99bffee Mon Sep 17 00:00:00 2001 From: Clay Good Date: Mon, 21 Sep 2026 16:38:46 -0500 Subject: [PATCH 08/14] fix(verify): distinguish untracked tasks from missing evidence --- openspec/specs/opsx-verify-skill/spec.md | 22 ++++++++----- skills/openspec-verify-change/SKILL.md | 17 ++++++---- src/commands/workflow/instructions.ts | 1 + src/commands/workflow/shared.ts | 1 + src/core/templates/workflows/verify-change.ts | 33 ++++++++++++------- .../commands/apply-instructions-tasks.test.ts | 1 + .../templates/skill-templates-parity.test.ts | 6 ++-- test/core/templates/verify-change.test.ts | 6 +++- 8 files changed, 57 insertions(+), 30 deletions(-) diff --git a/openspec/specs/opsx-verify-skill/spec.md b/openspec/specs/opsx-verify-skill/spec.md index d4790254a3..740b5002c8 100644 --- a/openspec/specs/opsx-verify-skill/spec.md +++ b/openspec/specs/opsx-verify-skill/spec.md @@ -19,10 +19,15 @@ The system SHALL provide an `/opsx:verify` skill that validates implementation a - **AND** announces which change was selected and how to override #### Scenario: Change has no task descriptions -- **WHEN** the structured task list provides no usable task descriptions, even if task progress reports nonzero totals +- **WHEN** the schema configures task tracking but the structured task list provides no usable task descriptions, even if task progress reports nonzero totals - **THEN** the agent reports Task Completion as not verified with the reason - **AND** continues checks supported by the remaining artifacts -- **AND** does not require tasks when the schema does not track them + +#### Scenario: Schema has no task tracking +- **WHEN** the schema does not configure `apply.tracks` +- **THEN** apply instructions report `taskTrackingConfigured: false` +- **AND** the agent reports Task Completion as not applicable, not as skipped or failed +- **AND** continues the checks that apply to the schema ### Requirement: Completeness Verification The agent SHALL verify that all required work has been completed. @@ -53,7 +58,7 @@ The agent SHALL verify that all required work has been completed. - **WHEN** all tasks are marked complete - **THEN** report "Tasks: N/N complete" - **AND** mark Task Completion as passed only when task descriptions are available -- **AND** mark the completeness dimension as passed only when all its checks ran and passed +- **AND** mark the completeness dimension as passed only when all applicable checks ran and passed #### Scenario: Incomplete tasks found - **WHEN** some tasks are incomplete @@ -162,7 +167,7 @@ The agent SHALL produce a structured, prioritized report. - **AND** avoid vague suggestions like "consider reviewing" #### Scenario: All checks pass -- **WHEN** every check ran and no issues were found across all dimensions +- **WHEN** every applicable check ran and no issues were found across all dimensions - **THEN** display: ```text All checks passed. Ready for archive. @@ -178,7 +183,7 @@ The agent SHALL produce a structured, prioritized report. - **AND** name every skipped check and its reason, if any #### Scenario: Only warnings -- **WHEN** every check ran and no CRITICAL issues but warnings exist +- **WHEN** every applicable check ran and no CRITICAL issues but warnings exist - **THEN** display: ```text No critical issues. Y warning(s) to consider. @@ -186,7 +191,7 @@ The agent SHALL produce a structured, prioritized report. ``` #### Scenario: Only suggestions -- **WHEN** every check ran and only suggestions exist +- **WHEN** every applicable check ran and only suggestions exist - **THEN** report "No critical issues or warnings. Z suggestion(s) to consider. Ready for archive (with noted improvements)." #### Scenario: Checks skipped @@ -227,7 +232,8 @@ The agent SHALL gracefully handle changes with varying artifact completeness. #### Scenario: Intentional artifact omissions - **WHEN** a check has no supporting artifacts because the schema omits task tracking or optional artifacts, or the change declares `skip_specs: true` -- **THEN** report why the corresponding checks were not verified +- **THEN** report the corresponding checks as not applicable and explain why +- **AND** exclude not-applicable checks from skipped-check counts and readiness assessment - **AND** do not require or create optional or intentionally skipped artifacts to obtain a passing report -- **AND** treat verification as advisory: not verified describes a limit of the report, not a new archive gate +- **AND** treat verification as advisory: not verified describes missing evidence for an applicable check, not a new archive gate - **AND** leave archive checks and user-confirmation behavior unchanged diff --git a/skills/openspec-verify-change/SKILL.md b/skills/openspec-verify-change/SKILL.md index 0294854240..d5f9d293f8 100644 --- a/skills/openspec-verify-change/SKILL.md +++ b/skills/openspec-verify-change/SKILL.md @@ -56,7 +56,7 @@ In both branches, never create the root as a side effect: do not run `openspec i openspec instructions apply --change "" --json ``` - This returns the change directory, `contextFiles` (artifact ID -> array of concrete file paths), and top-level `tasks` and `progress` aggregated from every concrete file matched by the schema's `apply.tracks` configuration that could be read. Read all available artifacts from `contextFiles`. + This returns the change directory, `contextFiles` (artifact ID -> array of concrete file paths), `taskTrackingConfigured`, and top-level `tasks` and `progress` aggregated from every concrete file matched by the schema's `apply.tracks` configuration that could be read. Read all available artifacts from `contextFiles`. Treat apply `state` and `instruction` as context, not a verification verdict. Do not implement tasks or archive the change during verification. @@ -71,23 +71,27 @@ In both branches, never create the root as a side effect: do not run `openspec i Verification is advisory. Respect intentional omissions such as `skip_specs: true`, optional design documents, and schemas without task tracking. Do not require or invent optional or intentionally omitted artifacts to obtain a clean report. `Not verified` describes a limit of this report, not a new archive prerequisite. Archive retains its own checks and user-confirmation behavior. - If only task evidence is available, verify task completion only and mark the remaining checks, including **Code Pattern Consistency**, as not verified with the reason "Only task evidence available". + Mark checks the schema does not define, or artifacts the status reports as intentionally skipped, as **Not applicable**. Exclude them from skipped-check counts and the archive-readiness assessment. Reserve **Not verified** for applicable checks whose evidence is missing or unusable. + + If only task evidence is available for applicable checks, verify task completion only and mark the remaining applicable checks, including **Code Pattern Consistency**, as not verified with the reason "Only task evidence available". If artifacts cannot be read or contain no usable requirements, scenarios, or design decisions, mark the affected checks as not verified with the specific reason. Continue checks supported by the remaining evidence, but a partially checked input set is not a fully verified check. Missing requirements affect Spec Coverage and Requirement Implementation Mapping; missing scenarios affect Scenario Coverage; missing design decisions affect Design Adherence. 5. **Verify Completeness** **Task Completion**: - - Use the top-level `tasks` and `progress` fields. They already aggregate every readable concrete file matched by `apply.tracks`, regardless of the tracked artifact's ID; do not infer tracking from a `contextFiles` key. + - If `taskTrackingConfigured` is false, report **Task Completion** as not applicable. Do not treat empty `tasks` as missing evidence. + - Otherwise, use the top-level `tasks` and `progress` fields. They already aggregate every readable concrete file matched by `apply.tracks`, regardless of the tracked artifact's ID; do not infer tracking from a `contextFiles` key. - If `unavailableTrackingFiles` is nonempty, mark **Task Completion** as not verified and include every unavailable path and reason. Continue using any readable task evidence, but do not infer completion from the partial `tasks` and `progress` fields. - - If `tasks` is empty, mark **Task Completion** as not verified and record the reason from apply `state` and `instruction`. Nonzero totals alone do not establish evaluable task descriptions. + - If `taskTrackingConfigured` is true and `tasks` is empty, mark **Task Completion** as not verified and record the reason from apply `state` and `instruction`. Nonzero totals alone do not establish evaluable task descriptions. - Report complete vs total tasks from `progress`. - If `progress.remaining` is greater than 0: - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: - - `contextFiles` is keyed by artifact id, and artifact ids come from the active schema. If `contextFiles.specs` is absent or empty, mark **Spec Coverage**, **Requirement Implementation Mapping**, and **Scenario Coverage** as not verified; do not treat any of them as clean. + - If status marks the spec artifact skipped by `skip_specs: true`, or the schema defines no spec artifact, report the spec-dependent checks as not applicable. + - Otherwise, `contextFiles` is keyed by artifact id, and artifact ids come from the active schema. If `contextFiles.specs` is absent or empty, mark **Spec Coverage**, **Requirement Implementation Mapping**, and **Scenario Coverage** as not verified; do not treat any of them as clean. - If delta specs exist in `contextFiles.specs`: - Extract all requirements (marked with "### Requirement:") - For each requirement: @@ -119,13 +123,14 @@ In both branches, never create the root as a side effect: do not run `openspec i 7. **Verify Coherence** **Design Adherence**: + - If the schema defines no design artifact, report **Design Adherence** as not applicable. - If `contextFiles.design` exists: - Extract key decisions (look for sections like "Decision:", "Approach:", "Architecture:") - Verify implementation follows those decisions - If contradiction detected: - Add WARNING: "Design decision not followed: " - Recommendation: "Update implementation or revise design.md to match reality" - - If `contextFiles.design` is absent or empty: mark **Design Adherence** as not verified. With other supporting artifacts, **Code Pattern Consistency** still runs; the task-only case remains limited to task completion. + - Otherwise, if `contextFiles.design` is absent or empty: mark **Design Adherence** as not verified. With other supporting artifacts, **Code Pattern Consistency** still runs; the task-only case remains limited to task completion. **Code Pattern Consistency**: - If implementation changes cannot be identified, mark **Code Pattern Consistency** as not verified and explain the missing evidence. diff --git a/src/commands/workflow/instructions.ts b/src/commands/workflow/instructions.ts index 0fe311d929..aed6418338 100644 --- a/src/commands/workflow/instructions.ts +++ b/src/commands/workflow/instructions.ts @@ -676,6 +676,7 @@ export async function generateApplyInstructions( contextFiles, progress: { total, complete, remaining }, tasks, + taskTrackingConfigured: tracksFile !== null, ...(unavailableTrackingFiles.length > 0 ? { unavailableTrackingFiles } : {}), state, missingArtifacts: missingArtifacts.length > 0 ? missingArtifacts : undefined, diff --git a/src/commands/workflow/shared.ts b/src/commands/workflow/shared.ts index e4b374bf61..e3e93bc73a 100644 --- a/src/commands/workflow/shared.ts +++ b/src/commands/workflow/shared.ts @@ -45,6 +45,7 @@ export interface ApplyInstructions { remaining: number; }; tasks: TaskItem[]; + taskTrackingConfigured: boolean; unavailableTrackingFiles?: Array<{ path: string; reason: string; diff --git a/src/core/templates/workflows/verify-change.ts b/src/core/templates/workflows/verify-change.ts index 9c71aef45c..92c4b0cc2b 100644 --- a/src/core/templates/workflows/verify-change.ts +++ b/src/core/templates/workflows/verify-change.ts @@ -50,7 +50,7 @@ ${PROJECT_ROOT_GUARD} openspec instructions apply --change "" --json \`\`\` - This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` aggregated from every concrete file matched by the schema's \`apply.tracks\` configuration that could be read. Read all available artifacts from \`contextFiles\`. + This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), \`taskTrackingConfigured\`, and top-level \`tasks\` and \`progress\` aggregated from every concrete file matched by the schema's \`apply.tracks\` configuration that could be read. Read all available artifacts from \`contextFiles\`. Treat apply \`state\` and \`instruction\` as context, not a verification verdict. Do not implement tasks or archive the change during verification. @@ -65,23 +65,27 @@ ${PROJECT_ROOT_GUARD} Verification is advisory. Respect intentional omissions such as \`skip_specs: true\`, optional design documents, and schemas without task tracking. Do not require or invent optional or intentionally omitted artifacts to obtain a clean report. \`Not verified\` describes a limit of this report, not a new archive prerequisite. Archive retains its own checks and user-confirmation behavior. - If only task evidence is available, verify task completion only and mark the remaining checks, including **Code Pattern Consistency**, as not verified with the reason "Only task evidence available". + Mark checks the schema does not define, or artifacts the status reports as intentionally skipped, as **Not applicable**. Exclude them from skipped-check counts and the archive-readiness assessment. Reserve **Not verified** for applicable checks whose evidence is missing or unusable. + + If only task evidence is available for applicable checks, verify task completion only and mark the remaining applicable checks, including **Code Pattern Consistency**, as not verified with the reason "Only task evidence available". If artifacts cannot be read or contain no usable requirements, scenarios, or design decisions, mark the affected checks as not verified with the specific reason. Continue checks supported by the remaining evidence, but a partially checked input set is not a fully verified check. Missing requirements affect Spec Coverage and Requirement Implementation Mapping; missing scenarios affect Scenario Coverage; missing design decisions affect Design Adherence. 5. **Verify Completeness** **Task Completion**: - - Use the top-level \`tasks\` and \`progress\` fields. They already aggregate every readable concrete file matched by \`apply.tracks\`, regardless of the tracked artifact's ID; do not infer tracking from a \`contextFiles\` key. + - If \`taskTrackingConfigured\` is false, report **Task Completion** as not applicable. Do not treat empty \`tasks\` as missing evidence. + - Otherwise, use the top-level \`tasks\` and \`progress\` fields. They already aggregate every readable concrete file matched by \`apply.tracks\`, regardless of the tracked artifact's ID; do not infer tracking from a \`contextFiles\` key. - If \`unavailableTrackingFiles\` is nonempty, mark **Task Completion** as not verified and include every unavailable path and reason. Continue using any readable task evidence, but do not infer completion from the partial \`tasks\` and \`progress\` fields. - - If \`tasks\` is empty, mark **Task Completion** as not verified and record the reason from apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable task descriptions. + - If \`taskTrackingConfigured\` is true and \`tasks\` is empty, mark **Task Completion** as not verified and record the reason from apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable task descriptions. - Report complete vs total tasks from \`progress\`. - If \`progress.remaining\` is greater than 0: - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: - - \`contextFiles\` is keyed by artifact id, and artifact ids come from the active schema. If \`contextFiles.specs\` is absent or empty, mark **Spec Coverage**, **Requirement Implementation Mapping**, and **Scenario Coverage** as not verified; do not treat any of them as clean. + - If status marks the spec artifact skipped by \`skip_specs: true\`, or the schema defines no spec artifact, report the spec-dependent checks as not applicable. + - Otherwise, \`contextFiles\` is keyed by artifact id, and artifact ids come from the active schema. If \`contextFiles.specs\` is absent or empty, mark **Spec Coverage**, **Requirement Implementation Mapping**, and **Scenario Coverage** as not verified; do not treat any of them as clean. - If delta specs exist in \`contextFiles.specs\`: - Extract all requirements (marked with "### Requirement:") - For each requirement: @@ -113,13 +117,14 @@ ${PROJECT_ROOT_GUARD} 7. **Verify Coherence** **Design Adherence**: + - If the schema defines no design artifact, report **Design Adherence** as not applicable. - If \`contextFiles.design\` exists: - Extract key decisions (look for sections like "Decision:", "Approach:", "Architecture:") - Verify implementation follows those decisions - If contradiction detected: - Add WARNING: "Design decision not followed: " - Recommendation: "Update implementation or revise design.md to match reality" - - If \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. With other supporting artifacts, **Code Pattern Consistency** still runs; the task-only case remains limited to task completion. + - Otherwise, if \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. With other supporting artifacts, **Code Pattern Consistency** still runs; the task-only case remains limited to task completion. **Code Pattern Consistency**: - If implementation changes cannot be identified, mark **Code Pattern Consistency** as not verified and explain the missing evidence. @@ -236,7 +241,7 @@ ${PROJECT_ROOT_GUARD} openspec instructions apply --change "" --json \`\`\` - This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), and top-level \`tasks\` and \`progress\` aggregated from every concrete file matched by the schema's \`apply.tracks\` configuration that could be read. Read all available artifacts from \`contextFiles\`. + This returns the change directory, \`contextFiles\` (artifact ID -> array of concrete file paths), \`taskTrackingConfigured\`, and top-level \`tasks\` and \`progress\` aggregated from every concrete file matched by the schema's \`apply.tracks\` configuration that could be read. Read all available artifacts from \`contextFiles\`. Treat apply \`state\` and \`instruction\` as context, not a verification verdict. Do not implement tasks or archive the change during verification. @@ -250,24 +255,27 @@ ${PROJECT_ROOT_GUARD} Each dimension can have CRITICAL, WARNING, or SUGGESTION issues. Verification is advisory. Respect intentional omissions such as \`skip_specs: true\`, optional design documents, and schemas without task tracking. Do not require or invent optional or intentionally omitted artifacts to obtain a clean report. \`Not verified\` describes a limit of this report, not a new archive prerequisite. Archive retains its own checks and user-confirmation behavior. + Mark checks the schema does not define, or artifacts the status reports as intentionally skipped, as **Not applicable**. Exclude them from skipped-check counts and the archive-readiness assessment. Reserve **Not verified** for applicable checks whose evidence is missing or unusable. - If only task evidence is available, verify task completion only and mark the remaining checks, including **Code Pattern Consistency**, as not verified with the reason "Only task evidence available". + If only task evidence is available for applicable checks, verify task completion only and mark the remaining applicable checks, including **Code Pattern Consistency**, as not verified with the reason "Only task evidence available". If artifacts cannot be read or contain no usable requirements, scenarios, or design decisions, mark the affected checks as not verified with the specific reason. Continue checks supported by the remaining evidence, but a partially checked input set is not a fully verified check. Missing requirements affect Spec Coverage and Requirement Implementation Mapping; missing scenarios affect Scenario Coverage; missing design decisions affect Design Adherence. 5. **Verify Completeness** **Task Completion**: - - Use the top-level \`tasks\` and \`progress\` fields. They already aggregate every readable concrete file matched by \`apply.tracks\`, regardless of the tracked artifact's ID; do not infer tracking from a \`contextFiles\` key. + - If \`taskTrackingConfigured\` is false, report **Task Completion** as not applicable. Do not treat empty \`tasks\` as missing evidence. + - Otherwise, use the top-level \`tasks\` and \`progress\` fields. They already aggregate every readable concrete file matched by \`apply.tracks\`, regardless of the tracked artifact's ID; do not infer tracking from a \`contextFiles\` key. - If \`unavailableTrackingFiles\` is nonempty, mark **Task Completion** as not verified and include every unavailable path and reason. Continue using any readable task evidence, but do not infer completion from the partial \`tasks\` and \`progress\` fields. - - If \`tasks\` is empty, mark **Task Completion** as not verified and record the reason from apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable task descriptions. + - If \`taskTrackingConfigured\` is true and \`tasks\` is empty, mark **Task Completion** as not verified and record the reason from apply \`state\` and \`instruction\`. Nonzero totals alone do not establish evaluable task descriptions. - Report complete vs total tasks from \`progress\`. - If \`progress.remaining\` is greater than 0: - Add CRITICAL issue for each listed incomplete task. If the remaining count exceeds the listed incomplete tasks, also report the incomplete checkboxes without descriptions and recommend adding descriptions and completing them. Do not infer completion from the listed tasks alone. - Recommendation: "Complete task: " or "Mark as done if already implemented" **Spec Coverage**: - - \`contextFiles\` is keyed by artifact id, and artifact ids come from the active schema. If \`contextFiles.specs\` is absent or empty, mark **Spec Coverage**, **Requirement Implementation Mapping**, and **Scenario Coverage** as not verified; do not treat any of them as clean. + - If status marks the spec artifact skipped by \`skip_specs: true\`, or the schema defines no spec artifact, report the spec-dependent checks as not applicable. + - Otherwise, \`contextFiles\` is keyed by artifact id, and artifact ids come from the active schema. If \`contextFiles.specs\` is absent or empty, mark **Spec Coverage**, **Requirement Implementation Mapping**, and **Scenario Coverage** as not verified; do not treat any of them as clean. - If delta specs exist in \`contextFiles.specs\`: - Extract all requirements (marked with "### Requirement:") - For each requirement: @@ -299,13 +307,14 @@ ${PROJECT_ROOT_GUARD} 7. **Verify Coherence** **Design Adherence**: + - If the schema defines no design artifact, report **Design Adherence** as not applicable. - If \`contextFiles.design\` exists: - Extract key decisions (look for sections like "Decision:", "Approach:", "Architecture:") - Verify implementation follows those decisions - If contradiction detected: - Add WARNING: "Design decision not followed: " - Recommendation: "Update implementation or revise design.md to match reality" - - If \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. With other supporting artifacts, **Code Pattern Consistency** still runs; the task-only case remains limited to task completion. + - Otherwise, if \`contextFiles.design\` is absent or empty: mark **Design Adherence** as not verified. With other supporting artifacts, **Code Pattern Consistency** still runs; the task-only case remains limited to task completion. **Code Pattern Consistency**: - If implementation changes cannot be identified, mark **Code Pattern Consistency** as not verified and explain the missing evidence. diff --git a/test/commands/apply-instructions-tasks.test.ts b/test/commands/apply-instructions-tasks.test.ts index 608a54f469..f11e76aefb 100644 --- a/test/commands/apply-instructions-tasks.test.ts +++ b/test/commands/apply-instructions-tasks.test.ts @@ -80,6 +80,7 @@ ${tracked ? ' tracks: checklist.md\n' : ''}` const instructions = await generateApplyInstructions(tempDir, 'my-change'); expect(instructions.contextFiles).toEqual({ implementation: [fs.realpathSync.native(checklist)] }); + expect(instructions.taskTrackingConfigured).toBe(tracked); expect(instructions.tasks).toEqual(tracked ? [ { id: '1', description: 'Finished task', done: true }, { id: '2', description: 'Pending task', done: false }, diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index a98599e99e..cd85dbce00 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -91,11 +91,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '8447a2489240bf0c27f863065d61453dd0264842d1dabafe27b577d6bff96eb3', getBulkArchiveChangeSkillTemplate: 'f17399959921ff98c7798e4591c8888825b7c9a83b0a90f09d98c7e0984ab793', getOpsxSyncCommandTemplate: '60550b7bb9829421656d6324a9e4c951bc912f48f88882d1a07ce7f78397a5e7', - getVerifyChangeSkillTemplate: '25f891539ebe2b7c71b06bfd776f1d5c75f356e4a7870f1669de4d8441a8d430', + getVerifyChangeSkillTemplate: '40d955c703bd319c3d7ecbf8c6da3a04d7c85746587844ecc60656a89a2f47b3', getOpsxArchiveCommandTemplate: '980109e5f8362610872c70fe0a0f1d48d3d2692275b2b17e2f4c91c3de89c2fd', getOpsxOnboardCommandTemplate: '9cad751f7b938eea039b0ba207247776269c81bec5923eb335bee468f515f244', getOpsxBulkArchiveCommandTemplate: '3db03eadb764abd74c8c180656c3f64a8b9a4971056c91624d38df3209d7b446', - getOpsxVerifyCommandTemplate: 'f494a01f6fc04267ab3deb1d9b69f48e76a1b71a2d21d80992b5ce97bc3898cc', + getOpsxVerifyCommandTemplate: '9c8219820ee83a72d180a94c7504457a5fa54670386be266f8247cccd5c68fce', getOpsxProposeSkillTemplate: '1aa2f2eb9c8cbc4dcab9d777bf8832b92ca04f9ef91d0494f1224a566aefdfe8', getOpsxProposeCommandTemplate: '3b7090ce5e79e879ab9b5bdaf4ff2b52e3c02211f71188838772d36ac337f96c', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -112,7 +112,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': '3909936a236a21a9a6d5bf495f90b396b3b68fc9220d7b2c1894668653beb2e4', 'openspec-archive-change': '305a21a9c76a925055f3bdbaac504f208660ef6948d78f73928de166250609bf', 'openspec-bulk-archive-change': '4bd638a50111d2ee3a667752a2355ed513f770695b137b93fc28848ca7bf60d2', - 'openspec-verify-change': '1a90b49fe3bf5dde58f03b5d23be1e2bdc0116d7f467540e4c9ef6e9a4df5b73', + 'openspec-verify-change': '1a9493b623da4f399ceac0be105c7a7248db421e83eb4c8b983ba3692c4a7453', 'openspec-onboard': 'd4c5f3e24c19c8e389950544ea0d1844027753def14748c9684210ae4c6cd5e5', 'openspec-propose': '66e3395adf9f2d93a09e8ef1d20e4efb010e5e8d4811f2d42a9316e4d1ca5a8b', 'openspec-update-change': '19163b8c1b40ccdc0840019aa8005877a90a3a1cd9f7aadb87f76ccce1342f19', diff --git a/test/core/templates/verify-change.test.ts b/test/core/templates/verify-change.test.ts index 5c434949c3..05fa002949 100644 --- a/test/core/templates/verify-change.test.ts +++ b/test/core/templates/verify-change.test.ts @@ -73,6 +73,10 @@ describe('verify-change templates', () => { expect(body, label).toContain('`skip_specs: true`'); expect(body, label).toContain('schemas without task tracking'); expect(body, label).toContain('Do not require or invent optional or intentionally omitted artifacts'); + expect(body, label).toContain('Mark checks the schema does not define, or artifacts the status reports as intentionally skipped, as **Not applicable**'); + expect(body, label).toContain('Exclude them from skipped-check counts and the archive-readiness assessment'); + expect(body, label).toContain('If `taskTrackingConfigured` is false, report **Task Completion** as not applicable'); + expect(body, label).toContain('If `taskTrackingConfigured` is true and `tasks` is empty, mark **Task Completion** as not verified'); expect(body, label).toContain('`Not verified` describes a limit of this report, not a new archive prerequisite'); expect(body, label).toContain('Archive retains its own checks and user-confirmation behavior'); } @@ -80,7 +84,7 @@ describe('verify-change templates', () => { it('preserves task-only verification without dropping checks supported by other artifacts', () => { for (const [label, body] of bodies) { - expect(body, label).toContain('If only task evidence is available, verify task completion only'); + expect(body, label).toContain('If only task evidence is available for applicable checks, verify task completion only'); expect(body, label).toContain('including **Code Pattern Consistency**, as not verified'); expect(body, label).toContain('With other supporting artifacts, **Code Pattern Consistency** still runs'); } From c3bda9e876a2878b7964612bc94ad5b4e13f1b17 Mon Sep 17 00:00:00 2001 From: Clay Good Date: Mon, 21 Sep 2026 18:06:44 -0500 Subject: [PATCH 09/14] docs(apply): document tracked globs and JSON evidence --- docs-lab/reference/cli.md | 4 +++- docs-lab/reference/schemas/schema-yaml.md | 16 ++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs-lab/reference/cli.md b/docs-lab/reference/cli.md index b09393675e..b319c0a2fc 100644 --- a/docs-lab/reference/cli.md +++ b/docs-lab/reference/cli.md @@ -1288,7 +1288,9 @@ With `--json`, each form returns one object. The artifact form starts: ... ``` -and continues with `outputPath`, `existingOutputPaths`, the full `instruction` and `template` strings, `dependencies`, `unlocks`, and `root`. The `apply` form carries `contextFiles`, `progress`, `tasks`, `state` (`blocked`, `ready`, `all_done`), and `instruction`. +and continues with `outputPath`, `existingOutputPaths`, the full `instruction` and `template` strings, `dependencies`, `unlocks`, and `root`. The `apply` form carries `contextFiles`, `progress`, `tasks`, `taskTrackingConfigured`, `state` (`blocked`, `ready`, `all_done`), and `instruction`. + +`taskTrackingConfigured` is always a boolean: `true` when the schema sets a non-null [`apply.tracks`](schemas/schema-yaml.md#tracks), even if no file matches, and `false` otherwise. If a matched tracking file cannot be read, `unavailableTrackingFiles` contains its absolute `path` and error `reason`. This field is omitted when every matched file is readable. Readable files still contribute to `tasks` and `progress`, but `state` cannot be `all_done` until every matched file is read. **Exit codes** diff --git a/docs-lab/reference/schemas/schema-yaml.md b/docs-lab/reference/schemas/schema-yaml.md index d8f47cd2ca..4c394264f4 100644 --- a/docs-lab/reference/schemas/schema-yaml.md +++ b/docs-lab/reference/schemas/schema-yaml.md @@ -119,7 +119,7 @@ OpenSpec rejects absolute paths and paths containing a `..` segment. | Field | Contract | |---|---| | `requires` | **Required.** A non-empty list of artifacts that must exist before apply instructions become ready. | -| `tracks` | An optional relative path to a Markdown task file in the change folder. Default: `null`. | +| `tracks` | An optional relative path or glob for Markdown task files in the change folder. Default: `null`. | | `instruction` | Optional guidance sent to the agent when apply is ready. OpenSpec uses built-in guidance by default. | Artifact `requires` controls planning order. `apply.requires` controls when apply instructions become ready. @@ -132,7 +132,9 @@ The path starts from the change folder. For a change named `add-auth`, `tracks: openspec/changes/add-auth/tasks.md ``` -Apply stays blocked if that file is missing or contains no checkbox with task text. OpenSpec counts these checkbox forms: +A glob such as `tracks: "**/tasks.md"` reads every matching file, such as `backend/tasks.md` and `frontend/tasks.md`. OpenSpec combines their tasks and progress. Use the same value for an artifact's `generates` field so status and list track the same files. + +Apply stays blocked if no file matches or the matched files contain no checkbox with task text. OpenSpec counts these checkbox forms: ```markdown - [ ] Pending task @@ -145,11 +147,13 @@ Apply stays blocked if that file is missing or contains no checkbox with task te Any Markdown list marker works: `-`, `*`, `+`, or a number of up to nine digits followed by `.` or `)`. Leading spaces are allowed. The [tasks.md section of the spec-driven page](spec-driven/index.md#tasksmd) defines the stricter format produced by the default schema. -The tracked file drives the apply state: +The tracked files drive the apply state: + +- **`blocked`**: no file matches, or no readable file has a checkbox with task text. +- **`ready`**: at least one task is pending, or a matched file could not be read while another provides tasks. +- **`all_done`**: every tracked task is checked and every matched file was read. -- **`blocked`**: the file is missing, or no checkbox has task text. -- **`ready`**: at least one tracked task is pending. -- **`all_done`**: every tracked task is checked. +If a matched file cannot be read, apply keeps the tasks and progress from readable files but does not mark the change `all_done`. [Apply JSON output](../cli.md#openspec-instructions) identifies each unavailable file and the reason. OpenSpec rejects absolute paths and paths containing a `..` segment. From 314a136a5f53346ced3bd9fad0fd2f7d7ff6c6be Mon Sep 17 00:00:00 2001 From: Clay Good Date: Tue, 22 Sep 2026 13:37:12 -0500 Subject: [PATCH 10/14] test(parity): regenerate hashes after merging #1940 Co-Authored-By: Claude Opus 5 --- test/core/templates/skill-templates-parity.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index cd85dbce00..5ecfcfc8b0 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -80,20 +80,20 @@ const EXPECTED_FUNCTION_HASHES: Record = { getNewChangeSkillTemplate: '0e5035b7b42198afc430206a1dbc9579096650ef0813d85e837d5a6cd0b98a85', getContinueChangeSkillTemplate: '550dc22bc8e0921b1ca5cef867379c4f370c5f1902c420bf9fa3bbfa75cea933', getApplyChangeSkillTemplate: '04ae407c97b5f9cb0cc15199fe877ccc7cd1eff78bfe10ad70c16a112b10a661', - getFfChangeSkillTemplate: '6fb5492e78b9ceec068949080ec9f2e0d2a8baff75a2fe33d07ad33ffe542b65', + getFfChangeSkillTemplate: 'd091600476a815ba99f69b446bcd46af5bf73d1c2810215a0c6196937d019cf6', getSyncSpecsSkillTemplate: 'bc80fe9b07eaa289e5eb8a3ce65eb7df722a16d864e37283c678220712e4f230', - getOnboardSkillTemplate: '7d92756ffc0b30053838716005610daf3f65c3fa011f3f4d29b6488f303f9cfb', + getOnboardSkillTemplate: '8f4bb13c097c7c2dfca714c3f51765039008d87f844e62186e066bebd7637376', getOpsxExploreCommandTemplate: 'f6cf22825643281d653355745623a6c1a4566db46cc2f262d2282243c6d8169a', getOpsxNewCommandTemplate: '6d504fef1e0d4ced7c423f4cc9d9d2cee11b1a6224edf685e06a3f0757e0ebff', getOpsxContinueCommandTemplate: 'ace5c9cc239c12b57dc86fd9a1c02a6ca467cb8e1245127340c07ab1b9d37c11', getOpsxApplyCommandTemplate: 'd70cecce3b7d1dd4dbd5fd1fc2bccb538f5e61f5b43d520e4beca896e3f9e6b3', - getOpsxFfCommandTemplate: '04cb49b0bf3ebe364b45268a283564ee4fd50b78b01ec1d3f975bcae68179d2d', + getOpsxFfCommandTemplate: '743a7304c7efc84aa87f556154c034e1e0e561c276c51870a30ada58f33eb9af', getArchiveChangeSkillTemplate: '8447a2489240bf0c27f863065d61453dd0264842d1dabafe27b577d6bff96eb3', getBulkArchiveChangeSkillTemplate: 'f17399959921ff98c7798e4591c8888825b7c9a83b0a90f09d98c7e0984ab793', getOpsxSyncCommandTemplate: '60550b7bb9829421656d6324a9e4c951bc912f48f88882d1a07ce7f78397a5e7', getVerifyChangeSkillTemplate: '40d955c703bd319c3d7ecbf8c6da3a04d7c85746587844ecc60656a89a2f47b3', getOpsxArchiveCommandTemplate: '980109e5f8362610872c70fe0a0f1d48d3d2692275b2b17e2f4c91c3de89c2fd', - getOpsxOnboardCommandTemplate: '9cad751f7b938eea039b0ba207247776269c81bec5923eb335bee468f515f244', + getOpsxOnboardCommandTemplate: '35332b79e943daefd4118513f03dc48926267af60348f781b8d92bbe5086a986', getOpsxBulkArchiveCommandTemplate: '3db03eadb764abd74c8c180656c3f64a8b9a4971056c91624d38df3209d7b446', getOpsxVerifyCommandTemplate: '9c8219820ee83a72d180a94c7504457a5fa54670386be266f8247cccd5c68fce', getOpsxProposeSkillTemplate: '1aa2f2eb9c8cbc4dcab9d777bf8832b92ca04f9ef91d0494f1224a566aefdfe8', @@ -108,12 +108,12 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-new-change': '27e09d43785953827efc9a98bb9d6cf06db48fe6abe7e1c049409fe5b5061323', 'openspec-continue-change': '182f015de6a1a114c79a6106c0565fd71f368d629641d0ad088de54bd871b52f', 'openspec-apply-change': 'f3e92c229fab8d77df9f0a77dcb117cf46279b53a208d53aed89bfe0bab2ac09', - 'openspec-ff-change': '8ffad1b1a2deea5f097eb7294fb8b9474d5dfb1c31ee2fd3311d9a9d78259323', + 'openspec-ff-change': 'a7ab656d46f04d45dff0c8888df4a126a2e62288b7336f7445bce4d1715055f5', 'openspec-sync-specs': '3909936a236a21a9a6d5bf495f90b396b3b68fc9220d7b2c1894668653beb2e4', 'openspec-archive-change': '305a21a9c76a925055f3bdbaac504f208660ef6948d78f73928de166250609bf', 'openspec-bulk-archive-change': '4bd638a50111d2ee3a667752a2355ed513f770695b137b93fc28848ca7bf60d2', 'openspec-verify-change': '1a9493b623da4f399ceac0be105c7a7248db421e83eb4c8b983ba3692c4a7453', - 'openspec-onboard': 'd4c5f3e24c19c8e389950544ea0d1844027753def14748c9684210ae4c6cd5e5', + 'openspec-onboard': '526bb7f9b8ceb8670b600ce33b0a62fe268393b7e778e120c796316da3a3cc3d', 'openspec-propose': '66e3395adf9f2d93a09e8ef1d20e4efb010e5e8d4811f2d42a9316e4d1ca5a8b', 'openspec-update-change': '19163b8c1b40ccdc0840019aa8005877a90a3a1cd9f7aadb87f76ccce1342f19', }; From b8c5ea5463db1edf58af6839dd7ae1ae8c0e80e3 Mon Sep 17 00:00:00 2001 From: Clay Good Date: Tue, 22 Sep 2026 14:52:46 -0500 Subject: [PATCH 11/14] test(parity): restore the #1837 regression tests dropped in the merge The earlier conflict resolution took our whole side of the parity file, which discarded the two threshold tests main gained in #1940. Take main's file verbatim and regenerate the hashes instead. Co-Authored-By: Claude Opus 5 --- .../templates/skill-templates-parity.test.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index 5ecfcfc8b0..2b01dcc108 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -156,6 +156,51 @@ function hash(value: string): string { } describe('skill templates split parity', () => { + it('uses one clarification threshold in fast-forward guidance (#1837)', () => { + const variants: Array<[string, string]> = [ + ['ff skill', getFfChangeSkillTemplate().instructions], + ['ff command', getOpsxFfCommandTemplate().content], + ]; + + for (const [variant, content] of variants) { + expect(content, variant).toContain( + '**If an artifact requires user input** (critically unclear context)' + ); + expect(content, variant).not.toContain( + '**If an artifact requires user input** (unclear context)' + ); + } + }); + + it('approves onboarding tasks before saving or offering implementation (#1837)', () => { + const variants: Array<[string, string]> = [ + ['onboard skill', getOnboardSkillTemplate().instructions], + ['onboard command', getOpsxOnboardCommandTemplate().content], + ]; + + for (const [variant, content] of variants) { + expect(content, variant).toContain('Does this task breakdown look right?'); + expect(content, variant).not.toContain( + 'Each checkbox becomes a unit of work in the apply phase. Ready to implement?' + ); + expect(content, variant).toContain( + '**PAUSE** - Wait for user approval/feedback.\n\n' + + 'After approval, save to the `resolvedOutputPath` from `openspec instructions tasks --change "" --json`.' + ); + expect(content, variant).toContain('> "Tasks are saved. Ready to implement?"'); + expect(content, variant).toContain( + '**PAUSE** - Wait for user to confirm before implementation.' + ); + + const saveAt = content.indexOf('After approval, save to the `resolvedOutputPath`'); + const implementationChoiceAt = content.indexOf('> "Tasks are saved. Ready to implement?"'); + const implementationAt = content.indexOf('## Phase 9: Apply (Implementation)'); + expect(saveAt, variant).toBeGreaterThanOrEqual(0); + expect(implementationChoiceAt, variant).toBeGreaterThan(saveAt); + expect(implementationAt, variant).toBeGreaterThan(implementationChoiceAt); + } + }); + it('preserves all template function payloads exactly', () => { const functionFactories: Record unknown> = { getExploreSkillTemplate, From 9867ab2a3e1c4e67a2d26995734f6b3ef0a9cce1 Mon Sep 17 00:00:00 2001 From: Clay Good Date: Wed, 23 Sep 2026 10:38:37 -0500 Subject: [PATCH 12/14] test(parity): regenerate hashes after merging #1955 Co-Authored-By: Claude Opus 5 --- test/core/templates/skill-templates-parity.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index 5bbcb847c3..506c65420e 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -91,11 +91,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '8447a2489240bf0c27f863065d61453dd0264842d1dabafe27b577d6bff96eb3', getBulkArchiveChangeSkillTemplate: 'f17399959921ff98c7798e4591c8888825b7c9a83b0a90f09d98c7e0984ab793', getOpsxSyncCommandTemplate: '60550b7bb9829421656d6324a9e4c951bc912f48f88882d1a07ce7f78397a5e7', - getVerifyChangeSkillTemplate: '2e069a277dac23818b13bb50b66e806ab405bc3b7f535400e1ebf81b84153699', + getVerifyChangeSkillTemplate: '40d955c703bd319c3d7ecbf8c6da3a04d7c85746587844ecc60656a89a2f47b3', getOpsxArchiveCommandTemplate: '980109e5f8362610872c70fe0a0f1d48d3d2692275b2b17e2f4c91c3de89c2fd', getOpsxOnboardCommandTemplate: '0cf66e164c0e14c916c6d1ebb5d80ded07d7fb8e55d4eb34eba43e8ca9c28558', getOpsxBulkArchiveCommandTemplate: '3db03eadb764abd74c8c180656c3f64a8b9a4971056c91624d38df3209d7b446', - getOpsxVerifyCommandTemplate: '938f52f20fb9a3b811ea47314baac1034cd550e8ab363ae878ccba4b6329348f', + getOpsxVerifyCommandTemplate: '9c8219820ee83a72d180a94c7504457a5fa54670386be266f8247cccd5c68fce', getOpsxProposeSkillTemplate: '1aa2f2eb9c8cbc4dcab9d777bf8832b92ca04f9ef91d0494f1224a566aefdfe8', getOpsxProposeCommandTemplate: '3b7090ce5e79e879ab9b5bdaf4ff2b52e3c02211f71188838772d36ac337f96c', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -112,7 +112,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': '3909936a236a21a9a6d5bf495f90b396b3b68fc9220d7b2c1894668653beb2e4', 'openspec-archive-change': '305a21a9c76a925055f3bdbaac504f208660ef6948d78f73928de166250609bf', 'openspec-bulk-archive-change': '4bd638a50111d2ee3a667752a2355ed513f770695b137b93fc28848ca7bf60d2', - 'openspec-verify-change': 'ad8a3098bd27d852721687c47a12db7107ed8b8dfc7f071406bb19961652e7ee', + 'openspec-verify-change': '1a9493b623da4f399ceac0be105c7a7248db421e83eb4c8b983ba3692c4a7453', 'openspec-onboard': '6993eff867d97d485e080078f9dfb80e968e242f3b17a924eeb077715fd548fa', 'openspec-propose': '66e3395adf9f2d93a09e8ef1d20e4efb010e5e8d4811f2d42a9316e4d1ca5a8b', 'openspec-update-change': '19163b8c1b40ccdc0840019aa8005877a90a3a1cd9f7aadb87f76ccce1342f19', From 62dbdf6bfa7a3775fd65a89191f5f59f05299cd4 Mon Sep 17 00:00:00 2001 From: Clay Good Date: Wed, 23 Sep 2026 11:39:32 -0500 Subject: [PATCH 13/14] test(parity): regenerate hashes after merging #1795 and #1926 Co-Authored-By: Claude Opus 5 --- test/core/templates/skill-templates-parity.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index b394b30ca4..1ebd8484f3 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -91,11 +91,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '71715f9d5899498942af03e182e6d1ac2c95952dde967950c2a9161084a53a8b', getBulkArchiveChangeSkillTemplate: '5997f8c5c33f8933b7b3e61a0e122222e27e36e0b713167c6254e0508d1b95bf', getOpsxSyncCommandTemplate: '60550b7bb9829421656d6324a9e4c951bc912f48f88882d1a07ce7f78397a5e7', - getVerifyChangeSkillTemplate: '2e069a277dac23818b13bb50b66e806ab405bc3b7f535400e1ebf81b84153699', + getVerifyChangeSkillTemplate: '40d955c703bd319c3d7ecbf8c6da3a04d7c85746587844ecc60656a89a2f47b3', getOpsxArchiveCommandTemplate: '3d2a330b46043fbb9f220831aa42ebbb62f411e9597b2bb491ad1ac1fa2d0873', getOpsxOnboardCommandTemplate: '0cf66e164c0e14c916c6d1ebb5d80ded07d7fb8e55d4eb34eba43e8ca9c28558', getOpsxBulkArchiveCommandTemplate: '090913b517d62bf6dc093689ab44d1c1c00df22719b6824977c7cc092dbed38f', - getOpsxVerifyCommandTemplate: '938f52f20fb9a3b811ea47314baac1034cd550e8ab363ae878ccba4b6329348f', + getOpsxVerifyCommandTemplate: '9c8219820ee83a72d180a94c7504457a5fa54670386be266f8247cccd5c68fce', getOpsxProposeSkillTemplate: '1aa2f2eb9c8cbc4dcab9d777bf8832b92ca04f9ef91d0494f1224a566aefdfe8', getOpsxProposeCommandTemplate: '3b7090ce5e79e879ab9b5bdaf4ff2b52e3c02211f71188838772d36ac337f96c', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -112,7 +112,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': '3909936a236a21a9a6d5bf495f90b396b3b68fc9220d7b2c1894668653beb2e4', 'openspec-archive-change': 'd01d9eeb06223ee89708b7963e82c5ebc11719c5b2dc62d4abb268ee016fcb7b', 'openspec-bulk-archive-change': '8018999e6444bdfe97b3b95ffdb5451fb06becf2cfb46bd8fd3277157bdde5d7', - 'openspec-verify-change': 'ad8a3098bd27d852721687c47a12db7107ed8b8dfc7f071406bb19961652e7ee', + 'openspec-verify-change': '1a9493b623da4f399ceac0be105c7a7248db421e83eb4c8b983ba3692c4a7453', 'openspec-onboard': '6993eff867d97d485e080078f9dfb80e968e242f3b17a924eeb077715fd548fa', 'openspec-propose': '66e3395adf9f2d93a09e8ef1d20e4efb010e5e8d4811f2d42a9316e4d1ca5a8b', 'openspec-update-change': 'c53a8416550d1c2dd7b2131acddd3498b8eef5c10c86bc2bc21bb1cdecb891aa', From 93d0114c45e71cb7882b1b4ea02fc395c1655311 Mon Sep 17 00:00:00 2001 From: Clay Good Date: Wed, 23 Sep 2026 12:24:17 -0500 Subject: [PATCH 14/14] test(parity): regenerate hashes after merging #1731 Co-Authored-By: Claude Opus 5 --- test/core/templates/skill-templates-parity.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index 4157a5d9d0..03fda1ae79 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -91,11 +91,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: '71715f9d5899498942af03e182e6d1ac2c95952dde967950c2a9161084a53a8b', getBulkArchiveChangeSkillTemplate: '2a6ec08fea0f942158b4abe9c8d1af9622038e0c4dc684e7c73dad2fb8379a54', getOpsxSyncCommandTemplate: '60550b7bb9829421656d6324a9e4c951bc912f48f88882d1a07ce7f78397a5e7', - getVerifyChangeSkillTemplate: '2e069a277dac23818b13bb50b66e806ab405bc3b7f535400e1ebf81b84153699', + getVerifyChangeSkillTemplate: '40d955c703bd319c3d7ecbf8c6da3a04d7c85746587844ecc60656a89a2f47b3', getOpsxArchiveCommandTemplate: '3d2a330b46043fbb9f220831aa42ebbb62f411e9597b2bb491ad1ac1fa2d0873', getOpsxOnboardCommandTemplate: '0cf66e164c0e14c916c6d1ebb5d80ded07d7fb8e55d4eb34eba43e8ca9c28558', getOpsxBulkArchiveCommandTemplate: '4e2e39c4d634074f4a1ed67f076d5c4d0ead8b998f4d75218c33cdc6173719be', - getOpsxVerifyCommandTemplate: '938f52f20fb9a3b811ea47314baac1034cd550e8ab363ae878ccba4b6329348f', + getOpsxVerifyCommandTemplate: '9c8219820ee83a72d180a94c7504457a5fa54670386be266f8247cccd5c68fce', getOpsxProposeSkillTemplate: '1aa2f2eb9c8cbc4dcab9d777bf8832b92ca04f9ef91d0494f1224a566aefdfe8', getOpsxProposeCommandTemplate: '3b7090ce5e79e879ab9b5bdaf4ff2b52e3c02211f71188838772d36ac337f96c', getFeedbackSkillTemplate: 'dabeb5e825b9349abc8156c3e7b8608f27987912a6d9bf47ef29addde6138133', @@ -112,7 +112,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': '3909936a236a21a9a6d5bf495f90b396b3b68fc9220d7b2c1894668653beb2e4', 'openspec-archive-change': 'd01d9eeb06223ee89708b7963e82c5ebc11719c5b2dc62d4abb268ee016fcb7b', 'openspec-bulk-archive-change': '10f050ad5ef77084dc55a202427988b23903ee122f00985238a4eb9354a5dc3c', - 'openspec-verify-change': 'ad8a3098bd27d852721687c47a12db7107ed8b8dfc7f071406bb19961652e7ee', + 'openspec-verify-change': '1a9493b623da4f399ceac0be105c7a7248db421e83eb4c8b983ba3692c4a7453', 'openspec-onboard': '6993eff867d97d485e080078f9dfb80e968e242f3b17a924eeb077715fd548fa', 'openspec-propose': '66e3395adf9f2d93a09e8ef1d20e4efb010e5e8d4811f2d42a9316e4d1ca5a8b', 'openspec-update-change': '5f4ea19aa732b33d87a2120ec393ee34578e70678d97e8c3bb10f988c00cb4d3',