From 54bd4bf75b93d2cb2bac2be0c5b189242988adb9 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Thu, 27 Aug 2026 01:29:12 +0530 Subject: [PATCH 1/4] fix: accept UTF-8 BOM in SKILL.md frontmatter Schema read utf-8 and anchored frontmatter at ^---, so a Notepad BOM made a valid skill fail HIGH while unicode called the same BOM benign. Read utf-8-sig and allow an optional BOM before the opening fence. Fixes #91 Signed-off-by: mimran-khan --- CHANGELOG.md | 3 ++ .../validators/frontmatter_parser.py | 4 +-- src/skillevaluator/validators/schema.py | 6 ++-- tests/validators/test_frontmatter_parser.py | 19 +++++++++++++ tests/validators/test_schema.py | 28 +++++++++++++++++++ 5 files changed, 55 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2ad6515..273742aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ All notable changes to SkillEvaluator are documented in this file. ### Fixed +- Schema and frontmatter parsing accept a leading UTF-8 BOM, matching the + unicode scanner's "benign BOM" note + ([#91](https://github.com/NVIDIA/SkillEvaluator/issues/91)). - Tier 3 paired pass@k evidence now respects Python's active integer-string conversion limit, preserves nonzero Wilson interval widths and paired-effect directions at large case counts, and documents exact-rational omission diff --git a/src/skillevaluator/validators/frontmatter_parser.py b/src/skillevaluator/validators/frontmatter_parser.py index 9c8f1aa4..a1aeac18 100644 --- a/src/skillevaluator/validators/frontmatter_parser.py +++ b/src/skillevaluator/validators/frontmatter_parser.py @@ -18,7 +18,7 @@ # Regex pattern for extracting frontmatter between --- markers FRONTMATTER_PATTERN = re.compile( - r"^---[^\S\r\n]*\r?\n(.*?)\r?\n---[^\S\r\n]*(?=\r?\n|\Z)(?:\r?\n)?(.*)", + r"^\ufeff?---[^\S\r\n]*\r?\n(.*?)\r?\n---[^\S\r\n]*(?=\r?\n|\Z)(?:\r?\n)?(.*)", re.DOTALL, ) @@ -47,7 +47,7 @@ def parse_frontmatter(file_path: Path) -> tuple[ParsedFrontmatter | None, Valida result = ValidationResult() try: - content = file_path.read_text(encoding="utf-8") + content = file_path.read_text(encoding="utf-8-sig") except Exception as e: result.add_error(f"Failed to read {file_path}: {e}") return None, result diff --git a/src/skillevaluator/validators/schema.py b/src/skillevaluator/validators/schema.py index e85e4721..26d2bbf2 100644 --- a/src/skillevaluator/validators/schema.py +++ b/src/skillevaluator/validators/schema.py @@ -179,7 +179,7 @@ def _validate_frontmatter(self, skill_md: Path) -> ValidationResult: file_path = str(skill_md) try: - content = skill_md.read_text(encoding="utf-8") + content = skill_md.read_text(encoding="utf-8-sig") except Exception as e: result.add_finding( Finding( @@ -439,7 +439,7 @@ def _validate_line_count(self, skill_md: Path) -> ValidationResult: file_path = str(skill_md) try: - line_count = len(skill_md.read_text(encoding="utf-8").splitlines()) + line_count = len(skill_md.read_text(encoding="utf-8-sig").splitlines()) if line_count > MAX_SKILL_MD_LINES: result.add_finding( Finding( @@ -490,7 +490,7 @@ def _validate_body_content(self, skill_md: Path) -> ValidationResult: file_path = str(skill_md) try: - content = skill_md.read_text(encoding="utf-8") + content = skill_md.read_text(encoding="utf-8-sig") except Exception: return result diff --git a/tests/validators/test_frontmatter_parser.py b/tests/validators/test_frontmatter_parser.py index 9baf27b4..6f444454 100644 --- a/tests/validators/test_frontmatter_parser.py +++ b/tests/validators/test_frontmatter_parser.py @@ -43,6 +43,25 @@ def test_first_body_line_indentation_is_preserved(self, tmp_path: Path, newline: assert result.passed assert parsed.content == " [code](false.md)\n" + def test_utf8_bom_is_accepted(self, tmp_path: Path): + """A valid frontmatter file that starts with a UTF-8 BOM still parses (#91).""" + test_file = tmp_path / "bom.md" + body = """--- +title: Bom +description: A file whose only extra is a leading UTF-8 BOM +--- + +# Content +""" + test_file.write_bytes(b"\xef\xbb\xbf" + body.encode("utf-8")) + + parsed, result = parse_frontmatter(test_file) + + assert parsed is not None + assert result.passed + assert parsed.yaml_data["title"] == "Bom" + assert parsed.content.strip() == "# Content" + def test_missing_frontmatter(self, tmp_path: Path): """Test file without frontmatter markers.""" test_file = tmp_path / "test.mdc" diff --git a/tests/validators/test_schema.py b/tests/validators/test_schema.py index 78db3ad6..43474afe 100644 --- a/tests/validators/test_schema.py +++ b/tests/validators/test_schema.py @@ -685,6 +685,34 @@ def test_all_required_sections_present_passes(self, tmp_path: Path): assert result.passed, f"Skill with complete body should pass. Errors: {result.errors}" + def test_utf8_bom_skill_md_passes_schema(self, tmp_path: Path): + """A valid SKILL.md that only adds a UTF-8 BOM must still pass schema (#91).""" + skill_dir = tmp_path / "bom-skill" + skill_dir.mkdir() + body = """--- +name: bom-skill +description: Valid skill whose SKILL.md starts with a UTF-8 BOM +metadata: + author: Bom User +--- + +# BOM Skill + +## Instructions + +1. Open the file in an editor that writes a BOM. + +## Examples + +Example usage. +""" + (skill_dir / "SKILL.md").write_bytes(b"\xef\xbb\xbf" + body.encode("utf-8")) + + result = SchemaValidator().validate(skill_dir) + + assert result.passed, f"BOM-only difference should still pass schema. Errors: {result.errors}" + assert all(f.check_name != "frontmatter_format" for f in result.findings) + def test_canonical_support_dirs_accepted(self, tmp_path: Path): """Canonical public skill support directories must not be flagged. From 990efbb270a1dd2164e2831915e8011a5019bc83 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Fri, 28 Aug 2026 14:01:35 +0530 Subject: [PATCH 2/4] fix: parse UTF-8 BOM frontmatter in the quality gate Schema already accepted a BOM-prefixed SKILL.md, but quality still matched ^--- and skipped XML-tag checks. Read utf-8-sig and reuse the shared FRONTMATTER_PATTERN. Fixes #91 Signed-off-by: mimran-khan --- .../validators/quality_score.py | 7 ++--- tests/validators/test_quality_score.py | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/skillevaluator/validators/quality_score.py b/src/skillevaluator/validators/quality_score.py index c43105f7..45af0b11 100644 --- a/src/skillevaluator/validators/quality_score.py +++ b/src/skillevaluator/validators/quality_score.py @@ -37,6 +37,7 @@ from skillevaluator.models.result import Finding, Severity, ValidationResult from skillevaluator.models.skill import XML_TAG_RE from skillevaluator.validators.base import ValidatorBase +from skillevaluator.validators.frontmatter_parser import FRONTMATTER_PATTERN from skillevaluator.validators.markdown import markdown_link_targets logger = get_logger(__name__) @@ -263,7 +264,7 @@ def _validate_single_skill(self, skill_path: Path) -> ValidationResult: result.metadata["quality_scores"] = qs.to_dict() return result - content = manifest.read_text(encoding="utf-8") + content = manifest.read_text(encoding="utf-8-sig") lines = content.split("\n") frontmatter_data = self._parse_frontmatter(content) @@ -315,7 +316,7 @@ def _validate_single_skill(self, skill_path: Path) -> ValidationResult: @staticmethod def _parse_frontmatter(content: str) -> dict | None: - fm_match = re.match(r"^---\s*\n(.*?)\n---", content, re.DOTALL) + fm_match = FRONTMATTER_PATTERN.match(content) if not fm_match: return None try: @@ -744,7 +745,7 @@ def _check_efficiency( # Token estimates qs.total_tokens = len(content) // 4 - fm_match = re.match(r"^---\s*\n(.*?)\n---", content, re.DOTALL) + fm_match = FRONTMATTER_PATTERN.match(content) if fm_match: qs.frontmatter_tokens = len(fm_match.group(1)) // 4 inst_start = content.find("---", 3) + 3 diff --git a/tests/validators/test_quality_score.py b/tests/validators/test_quality_score.py index 89f62151..23d98c25 100644 --- a/tests/validators/test_quality_score.py +++ b/tests/validators/test_quality_score.py @@ -314,6 +314,33 @@ def test_xml_tags_in_description_remain_quality_error(self, tmp_path): assert any("Description contains XML tags" in finding.message for finding in result.findings) + def test_bom_prefixed_manifest_still_parses_frontmatter(self, tmp_path): + """A UTF-8 BOM must not hide frontmatter from the quality parser.""" + skill_dir = tmp_path / "bom-xml-desc" + skill_dir.mkdir() + body = ( + "---\n" + "name: bom-xml-desc\n" + "description: \"A skill with injected tags\"\n" + "metadata:\n" + " author: Test User \n" + "---\n\n" + "# XML Description\n\n" + "## Instructions\n\n1. Inspect frontmatter quality findings.\n\n" + "## Examples\n\n" + "```text\n" + "Validate the skill.\n" + "```\n" + ) + (skill_dir / "SKILL.md").write_bytes(b"\xef\xbb\xbf" + body.encode("utf-8")) + + result = QualityScoreValidator(min_score=0).validate(skill_dir) + scores = result.metadata["quality_scores"] + + assert scores["metrics"]["has_frontmatter"] is True + assert scores["metrics"]["frontmatter_tokens"] > 0 + assert any("Description contains XML tags" in finding.message for finding in result.findings) + def test_unclosed_xml_tag_in_description_remains_quality_error(self, tmp_path): """Unclosed tag-like descriptions remain covered by XML-tag detection.""" skill_dir = tmp_path / "unclosed-xml-desc" From c4e249d0c79fa75497f837699948da58cd33da99 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Wed, 2 Sep 2026 02:05:39 +0530 Subject: [PATCH 3/4] fix: make security PII scan honor UTF-8 BOM frontmatter Read SKILL manifests with utf-8-sig and treat BOM-prefixed --- fences as valid frontmatter so metadata.author emails are exempt in LF and CRLF files. --- src/skillevaluator/validators/security.py | 13 ++++++-- tests/validators/test_security.py | 39 +++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/skillevaluator/validators/security.py b/src/skillevaluator/validators/security.py index 62f01a3e..e4cdae61 100644 --- a/src/skillevaluator/validators/security.py +++ b/src/skillevaluator/validators/security.py @@ -1590,6 +1590,11 @@ def _passes_luhn(digits: str) -> bool: total += n return total % 10 == 0 + @staticmethod + def _is_frontmatter_delimiter(line: str) -> bool: + """True when a line is a YAML frontmatter fence, including BOM-prefixed openers.""" + return line.strip().removeprefix("\ufeff").strip() == "---" + def _scan_file_for_pii(self, file_path: Path, protected_usernames: set[str] | None = None) -> list[dict]: """Scan a single file for PII patterns, yielding findings with full context. @@ -1601,7 +1606,7 @@ def _scan_file_for_pii(self, file_path: Path, protected_usernames: set[str] | No protected_usernames = self._protected_home_usernames(file_path.parent) try: - content = file_path.read_text(encoding="utf-8", errors="ignore") + content = file_path.read_text(encoding="utf-8-sig", errors="ignore") except Exception as e: logger.warning(f"Could not read {file_path}: {e}") return [] @@ -1630,10 +1635,12 @@ def _scan_file_for_pii(self, file_path: Path, protected_usernames: set[str] | No def _frontmatter_author_emails(self, file_path: Path, lines: list[str]) -> dict[int, str]: """Map valid frontmatter author lines to the public contributor email.""" - if file_path.name not in SKILL_MANIFEST_VARIANTS or not lines or lines[0].strip() != "---": + if file_path.name not in SKILL_MANIFEST_VARIANTS or not lines or not self._is_frontmatter_delimiter(lines[0]): return {} try: - frontmatter_end = next(index for index, line in enumerate(lines[1:], 1) if line.strip() == "---") + frontmatter_end = next( + index for index, line in enumerate(lines[1:], 1) if self._is_frontmatter_delimiter(line) + ) except StopIteration: return {} diff --git a/tests/validators/test_security.py b/tests/validators/test_security.py index 7e6975f5..06af20c2 100644 --- a/tests/validators/test_security.py +++ b/tests/validators/test_security.py @@ -612,6 +612,45 @@ def test_valid_public_author_email_is_exempt_only_in_frontmatter(self, tmp_path: assert len(email_findings) == 1 assert email_findings[0].line_content == "Contact contributor@contributors.invalid for private support." + @pytest.mark.parametrize("newline", ["\n", "\r\n"]) + def test_bom_prefixed_frontmatter_author_email_not_flagged_in_security_scan( + self, tmp_path: Path, newline: str + ): + """UTF-8 BOM must not turn a valid frontmatter author email into a PII finding.""" + skill_dir = tmp_path / "bom-public-author-skill" + skill_dir.mkdir() + skill_md = skill_dir / "SKILL.md" + body = newline.join( + [ + "---", + "name: bom-public-author-skill", + "description: A public skill with contributor metadata and a body contact leak", + "metadata:", + " author: Example Contributor ", + "---", + "", + "# Public Author Skill", + "", + "## Instructions", + "", + "Contact contributor@contributors.invalid for private support.", + "", + "## Examples", + "", + "Run the documented workflow.", + "", + ] + ) + skill_md.write_bytes(b"\xef\xbb\xbf" + body.encode("utf-8")) + + schema_result = SchemaValidator().validate(skill_dir) + pii_result = SecurityValidator(submitter_usernames=[]).validate_pii_only(skill_dir) + + assert not [finding for finding in schema_result.findings if finding.check_name == "author_format"] + email_findings = [finding for finding in pii_result.findings if finding.check_name == "emails"] + assert len(email_findings) == 1 + assert email_findings[0].line_content == "Contact contributor@contributors.invalid for private support." + def test_unrelated_home_roots_not_flagged(self, tmp_path: Path): """Unrelated /home roots stay unflagged without an organization allowlist.""" skill_dir = tmp_path / "shared-home-skill" From 5ce993163ee2d339d065361c2e4bc15ed0843bf6 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Wed, 2 Sep 2026 02:05:56 +0530 Subject: [PATCH 4/4] docs: note security BOM handling in CHANGELOG --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 769e4c94..663d1161 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,8 @@ All notable changes to SkillEvaluator are documented in this file. ### Fixed -- Schema, frontmatter, and quality parsing accept a leading UTF-8 BOM, - matching the unicode scanner's "benign BOM" note +- Schema, frontmatter, quality parsing, and security PII scanning accept a leading + UTF-8 BOM, matching the unicode scanner's "benign BOM" note ([#91](https://github.com/NVIDIA/SkillEvaluator/issues/91)). - Windows personal-path PII now flags `C:\Users\...` usernames that start with `s` (for example `steve`), matching the intended whitespace class rather than