From 9304600a2b73a5977696e216cb8eded991683c28 Mon Sep 17 00:00:00 2001 From: eric-sabe Date: Tue, 7 Jul 2026 23:53:06 -0400 Subject: [PATCH] fix(skillspector): fail loud on output-schema drift, not silent clean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified honey's skillspector lens against SkillSpector 2.3.11 (upstream moved 2.0.0 -> 2.3.11): the `issues[]` contract is unchanged — id/pattern/severity/ category/location — and the lens produces identical well-formed findings on both bash and PowerShell (116 findings; severities are now UPPERCASE, which the lens already lowercases). But the lens normalized `(.issues // [])`, so if a FUTURE version renames or drops that array, honey would yield ZERO findings and report `clean` — a silent false all-clear over an unscanned surface, exactly the risk of a fast-moving, release-less upstream. Add a schema guard: valid JSON that lacks a top-level `issues` key is treated as scan_error (visible) rather than clean. A genuine `issues: []` is still correctly clean. Both platforms; shellcheck clean; PSScriptAnalyzer clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- lenses/skillspector.sh | 13 +++++++++++-- win/lenses/skillspector.ps1 | 10 +++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/lenses/skillspector.sh b/lenses/skillspector.sh index 3e23c6a..67f5abf 100755 --- a/lenses/skillspector.sh +++ b/lenses/skillspector.sh @@ -85,8 +85,17 @@ for skill in "${skills[@]}"; do echo "scan produced no valid JSON for: $skill" >>"$work/errors.log" errors=$((errors+1)); continue fi - # Normalize v2.0.0 `issues[]` into honey's finding shape (schema verified - # against SkillSpector v2.0.0 --format json output). + # Schema guard: honey normalizes the `issues[]` array. If a future SkillSpector + # renames/drops it, `(.issues // [])` would silently yield ZERO findings — a + # false "clean". Treat valid-JSON-without-`issues` as a schema-drift ERROR so + # it surfaces as scan_error (visible), never a silent all-clear. + if ! jq -e 'has("issues")' "$raw" >/dev/null 2>&1; then + echo "SCHEMA DRIFT: SkillSpector output for $skill has no top-level .issues (output format changed? re-verify the lens normalizer)" >>"$work/errors.log" + errors=$((errors+1)); continue + fi + # Normalize `issues[]` into honey's finding shape. Schema verified against + # SkillSpector v2.0.0 and v2.3.11 --format json output (issues[] is the tool's + # documented "core contract": id/pattern/severity/category/location). norm="$(jq --arg skill "$skill" ' (.issues // []) | map({ severity: ((.severity // "unknown") | ascii_downcase), diff --git a/win/lenses/skillspector.ps1 b/win/lenses/skillspector.ps1 index 1f19372..0e2c093 100644 --- a/win/lenses/skillspector.ps1 +++ b/win/lenses/skillspector.ps1 @@ -76,8 +76,16 @@ foreach ($skill in $skills) { $data = $null try { $data = Get-Content -LiteralPath $raw -Raw | ConvertFrom-Json } catch { $data = $null } if ($null -eq $data) { Add-Content -LiteralPath $errLog -Value "no valid JSON for: $skill"; $errors++; continue } + # Schema guard: honey normalizes the `issues[]` array. If a future SkillSpector + # renames/drops it, iterating a missing property yields ZERO findings — a false + # "clean". Treat valid-JSON-without-`issues` as schema-drift ERROR (surfaces as + # scan_error), never a silent all-clear. + if ($data.PSObject.Properties.Name -notcontains 'issues') { + Add-Content -LiteralPath $errLog -Value "SCHEMA DRIFT: SkillSpector output for $skill has no top-level .issues (output format changed? re-verify the lens normalizer)" + $errors++; continue + } - foreach ($issue in @(if ($data.PSObject.Properties.Name -contains 'issues') { $data.issues } else { @() })) { + foreach ($issue in @($data.issues)) { $id = if ($issue.PSObject.Properties.Name -contains 'id' -and $issue.id) { $issue.id } else { '' } $pattern = if ($issue.PSObject.Properties.Name -contains 'pattern' -and $issue.pattern) { $issue.pattern } else { '' } $title = (($id + ' ' + $pattern).Trim())