From f8cdb97f769906dfefefffff165e47c95294a21a Mon Sep 17 00:00:00 2001 From: Tym Rabchuk Date: Sun, 5 Apr 2026 18:24:08 -0400 Subject: [PATCH 1/2] Add CI count validation for README/ROADMAP numbers New validate-counts job checks that documented counts (commands, skills, workflows, hooks) match actual file counts. Runs on every push/PR. This session alone had 3 count mismatches (9 vs 10 hooks, 14 vs 15 skills, 12 vs 13 workflows) caught only by manual review. Automates the most common review finding. --- .github/workflows/ci.yml | 95 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e128868..ae9ad5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,3 +38,98 @@ jobs: echo "$unformatted" exit 1 fi + + validate-counts: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Validate README and ROADMAP counts + run: | + ERRORS="" + + # Count actual files + CMD_COUNT=$(ls commands/*.md 2>/dev/null | wc -l | tr -d ' ') + SKILL_COUNT=$(ls -d skills/*/SKILL.md 2>/dev/null | wc -l | tr -d ' ') + WORKFLOW_COUNT=$(ls workflows/*.yml 2>/dev/null | wc -l | tr -d ' ') + HOOK_COUNT=$(cat hooks/hooks.json | python3 -c " + import sys, json + data = json.load(sys.stdin) + scripts = set() + for event in data.get('hooks', {}).values(): + for entry in event: + for h in entry.get('hooks', []): + cmd = h.get('command', '') + scripts.add(cmd.split('/')[-1]) + print(len(scripts)) + ") + + echo "Actual counts: commands=$CMD_COUNT skills=$SKILL_COUNT workflows=$WORKFLOW_COUNT hooks=$HOOK_COUNT" + + # Check README + for file in README.md ROADMAP.md; do + if [ ! -f "$file" ]; then continue; fi + + # Commands + if grep -qE "\*\*[0-9]+ slash commands\*\*" "$file"; then + DOC_CMD=$(grep -oE '\*\*[0-9]+ slash commands\*\*' "$file" | grep -oE '[0-9]+') + if [ "$DOC_CMD" != "$CMD_COUNT" ]; then + ERRORS="$ERRORS\n$file: says $DOC_CMD commands, actual is $CMD_COUNT" + fi + fi + + # Skills + if grep -qE "\*\*[0-9]+ context-activated skills\*\*" "$file"; then + DOC_SKILL=$(grep -oE '\*\*[0-9]+ context-activated skills\*\*' "$file" | grep -oE '[0-9]+') + if [ "$DOC_SKILL" != "$SKILL_COUNT" ]; then + ERRORS="$ERRORS\n$file: says $DOC_SKILL skills, actual is $SKILL_COUNT" + fi + fi + + # Workflows + if grep -qE "\*\*[0-9]+ YAML workflows\*\*" "$file"; then + DOC_WF=$(grep -oE '\*\*[0-9]+ YAML workflows\*\*' "$file" | grep -oE '[0-9]+') + if [ "$DOC_WF" != "$WORKFLOW_COUNT" ]; then + ERRORS="$ERRORS\n$file: says $DOC_WF workflows, actual is $WORKFLOW_COUNT" + fi + fi + + # Hooks + if grep -qE "\*\*[0-9]+ hooks\*\*" "$file"; then + DOC_HOOK=$(grep -oE '\*\*[0-9]+ hooks\*\*' "$file" | grep -oE '[0-9]+') + if [ "$DOC_HOOK" != "$HOOK_COUNT" ]; then + ERRORS="$ERRORS\n$file: says $DOC_HOOK hooks, actual is $HOOK_COUNT" + fi + fi + done + + # Check README repo structure comments + if grep -qE '# [0-9]+ slash commands' README.md; then + TREE_CMD=$(grep -oE '# [0-9]+ slash commands' README.md | grep -oE '[0-9]+') + if [ "$TREE_CMD" != "$CMD_COUNT" ]; then + ERRORS="$ERRORS\nREADME.md tree: says $TREE_CMD commands, actual is $CMD_COUNT" + fi + fi + if grep -qE '# [0-9]+ context-activated skills' README.md; then + TREE_SKILL=$(grep -oE '# [0-9]+ context-activated skills' README.md | grep -oE '[0-9]+') + if [ "$TREE_SKILL" != "$SKILL_COUNT" ]; then + ERRORS="$ERRORS\nREADME.md tree: says $TREE_SKILL skills, actual is $SKILL_COUNT" + fi + fi + if grep -qE '# [0-9]+ YAML workflow' README.md; then + TREE_WF=$(grep -oE '# [0-9]+ YAML workflow' README.md | grep -oE '[0-9]+') + if [ "$TREE_WF" != "$WORKFLOW_COUNT" ]; then + ERRORS="$ERRORS\nREADME.md tree: says $TREE_WF workflows, actual is $WORKFLOW_COUNT" + fi + fi + + if [ -n "$ERRORS" ]; then + echo "" + echo "COUNT MISMATCHES FOUND:" + printf "$ERRORS\n" + echo "" + echo "Fix the counts in README.md and/or ROADMAP.md to match actual file counts." + exit 1 + else + echo "All counts match." + fi From 7f3ec9df1ac83a2f0ce735f6d27d52a1587628bd Mon Sep 17 00:00:00 2001 From: Tym Rabchuk Date: Sun, 5 Apr 2026 18:27:52 -0400 Subject: [PATCH 2/2] Harden CI count validation against silent failures Fixes from PR review (code-reviewer + silent-failure-hunter): - Add set -uo pipefail to catch command failures - Verify directories exist before counting - Validate counts are non-zero (catches missing dirs) - Add || exit 1 on python3 hooks.json parse (catches malformed JSON) - Use grep -m1 + head -1 on all extractions (prevents multi-match bugs) - Use printf '%b' instead of printf "$var" (prevents format injection) - Track CHECKS counter, fail if no patterns found (prevents silent pass when count claims are removed from docs) --- .github/workflows/ci.yml | 64 ++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae9ad5d..b735087 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,15 +46,25 @@ jobs: - name: Validate README and ROADMAP counts run: | + set -uo pipefail ERRORS="" + CHECKS=0 + + # Verify directories exist + for dir in commands skills workflows hooks; do + if [ ! -d "$dir" ]; then + echo "FATAL: $dir/ directory missing" + exit 1 + fi + done # Count actual files - CMD_COUNT=$(ls commands/*.md 2>/dev/null | wc -l | tr -d ' ') - SKILL_COUNT=$(ls -d skills/*/SKILL.md 2>/dev/null | wc -l | tr -d ' ') - WORKFLOW_COUNT=$(ls workflows/*.yml 2>/dev/null | wc -l | tr -d ' ') - HOOK_COUNT=$(cat hooks/hooks.json | python3 -c " + CMD_COUNT=$(set -- commands/*.md; echo $#) + SKILL_COUNT=$(find skills -maxdepth 2 -name 'SKILL.md' | wc -l | tr -d ' ') + WORKFLOW_COUNT=$(set -- workflows/*.yml; echo $#) + HOOK_COUNT=$(python3 -c " import sys, json - data = json.load(sys.stdin) + data = json.load(open('hooks/hooks.json')) scripts = set() for event in data.get('hooks', {}).values(): for entry in event: @@ -62,17 +72,27 @@ jobs: cmd = h.get('command', '') scripts.add(cmd.split('/')[-1]) print(len(scripts)) - ") + ") || { echo "FATAL: failed to parse hooks/hooks.json"; exit 1; } + + # Validate counts are non-zero + for var_name in CMD_COUNT SKILL_COUNT WORKFLOW_COUNT HOOK_COUNT; do + val=$(eval echo \$$var_name) + if [ -z "$val" ] || [ "$val" = "0" ]; then + echo "FATAL: $var_name is empty or zero ($val)" + exit 1 + fi + done echo "Actual counts: commands=$CMD_COUNT skills=$SKILL_COUNT workflows=$WORKFLOW_COUNT hooks=$HOOK_COUNT" - # Check README + # Check README and ROADMAP for file in README.md ROADMAP.md; do if [ ! -f "$file" ]; then continue; fi # Commands if grep -qE "\*\*[0-9]+ slash commands\*\*" "$file"; then - DOC_CMD=$(grep -oE '\*\*[0-9]+ slash commands\*\*' "$file" | grep -oE '[0-9]+') + DOC_CMD=$(grep -m1 -oE '\*\*[0-9]+ slash commands\*\*' "$file" | grep -oE '[0-9]+' | head -1) + CHECKS=$((CHECKS + 1)) if [ "$DOC_CMD" != "$CMD_COUNT" ]; then ERRORS="$ERRORS\n$file: says $DOC_CMD commands, actual is $CMD_COUNT" fi @@ -80,7 +100,8 @@ jobs: # Skills if grep -qE "\*\*[0-9]+ context-activated skills\*\*" "$file"; then - DOC_SKILL=$(grep -oE '\*\*[0-9]+ context-activated skills\*\*' "$file" | grep -oE '[0-9]+') + DOC_SKILL=$(grep -m1 -oE '\*\*[0-9]+ context-activated skills\*\*' "$file" | grep -oE '[0-9]+' | head -1) + CHECKS=$((CHECKS + 1)) if [ "$DOC_SKILL" != "$SKILL_COUNT" ]; then ERRORS="$ERRORS\n$file: says $DOC_SKILL skills, actual is $SKILL_COUNT" fi @@ -88,7 +109,8 @@ jobs: # Workflows if grep -qE "\*\*[0-9]+ YAML workflows\*\*" "$file"; then - DOC_WF=$(grep -oE '\*\*[0-9]+ YAML workflows\*\*' "$file" | grep -oE '[0-9]+') + DOC_WF=$(grep -m1 -oE '\*\*[0-9]+ YAML workflows\*\*' "$file" | grep -oE '[0-9]+' | head -1) + CHECKS=$((CHECKS + 1)) if [ "$DOC_WF" != "$WORKFLOW_COUNT" ]; then ERRORS="$ERRORS\n$file: says $DOC_WF workflows, actual is $WORKFLOW_COUNT" fi @@ -96,7 +118,8 @@ jobs: # Hooks if grep -qE "\*\*[0-9]+ hooks\*\*" "$file"; then - DOC_HOOK=$(grep -oE '\*\*[0-9]+ hooks\*\*' "$file" | grep -oE '[0-9]+') + DOC_HOOK=$(grep -m1 -oE '\*\*[0-9]+ hooks\*\*' "$file" | grep -oE '[0-9]+' | head -1) + CHECKS=$((CHECKS + 1)) if [ "$DOC_HOOK" != "$HOOK_COUNT" ]; then ERRORS="$ERRORS\n$file: says $DOC_HOOK hooks, actual is $HOOK_COUNT" fi @@ -105,31 +128,40 @@ jobs: # Check README repo structure comments if grep -qE '# [0-9]+ slash commands' README.md; then - TREE_CMD=$(grep -oE '# [0-9]+ slash commands' README.md | grep -oE '[0-9]+') + TREE_CMD=$(grep -m1 -oE '# [0-9]+ slash commands' README.md | grep -oE '[0-9]+' | head -1) + CHECKS=$((CHECKS + 1)) if [ "$TREE_CMD" != "$CMD_COUNT" ]; then ERRORS="$ERRORS\nREADME.md tree: says $TREE_CMD commands, actual is $CMD_COUNT" fi fi if grep -qE '# [0-9]+ context-activated skills' README.md; then - TREE_SKILL=$(grep -oE '# [0-9]+ context-activated skills' README.md | grep -oE '[0-9]+') + TREE_SKILL=$(grep -m1 -oE '# [0-9]+ context-activated skills' README.md | grep -oE '[0-9]+' | head -1) + CHECKS=$((CHECKS + 1)) if [ "$TREE_SKILL" != "$SKILL_COUNT" ]; then ERRORS="$ERRORS\nREADME.md tree: says $TREE_SKILL skills, actual is $SKILL_COUNT" fi fi if grep -qE '# [0-9]+ YAML workflow' README.md; then - TREE_WF=$(grep -oE '# [0-9]+ YAML workflow' README.md | grep -oE '[0-9]+') + TREE_WF=$(grep -m1 -oE '# [0-9]+ YAML workflow' README.md | grep -oE '[0-9]+' | head -1) + CHECKS=$((CHECKS + 1)) if [ "$TREE_WF" != "$WORKFLOW_COUNT" ]; then ERRORS="$ERRORS\nREADME.md tree: says $TREE_WF workflows, actual is $WORKFLOW_COUNT" fi fi + # Ensure at least some counts were validated + if [ "$CHECKS" -lt 1 ]; then + echo "FATAL: no count patterns found in README.md or ROADMAP.md — expected at least one" + exit 1 + fi + if [ -n "$ERRORS" ]; then echo "" echo "COUNT MISMATCHES FOUND:" - printf "$ERRORS\n" + printf '%b\n' "$ERRORS" echo "" echo "Fix the counts in README.md and/or ROADMAP.md to match actual file counts." exit 1 else - echo "All counts match." + echo "All counts match ($CHECKS checks passed)." fi