From e1837026c0b4df4d2d80125a2095d61eed4ab959 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Thu, 27 Aug 2026 00:12:29 +0530 Subject: [PATCH] fix: flag Windows PII usernames that start with s The Windows personal-path rule was single-quoted, so the character class treated s as a literal rather than whitespace. Usernames like steve were skipped. Quote it the same way as the macOS rule. Fixes #87 Signed-off-by: mimran-khan --- CHANGELOG.md | 4 ++ src/skillevaluator/config/pii_patterns.yaml | 2 +- tests/validators/test_security.py | 50 +++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2ad6515..3a822625 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to SkillEvaluator are documented in this file. ### Fixed +- Windows personal-path PII now flags `C:\Users\...` usernames that start with + `s` (for example `steve`), matching the intended whitespace class rather than + excluding the letter `s` ([#87](https://github.com/NVIDIA/SkillEvaluator/issues/87)). + - Tier 3 paired pass@k evidence now respects Python's active integer-string conversion limit, preserves nonzero Wilson interval widths and paired-effect directions at large case counts, and documents exact-rational omission diff --git a/src/skillevaluator/config/pii_patterns.yaml b/src/skillevaluator/config/pii_patterns.yaml index bef3d64c..06f85ec0 100644 --- a/src/skillevaluator/config/pii_patterns.yaml +++ b/src/skillevaluator/config/pii_patterns.yaml @@ -33,7 +33,7 @@ personal_paths: - "DELETE /users" - "api/users" - - pattern: 'C:\\Users\\[^\\s]+' + - pattern: "C:\\\\Users\\\\[^\\\\\\s]+" severity: high description: Personal Windows user path suggestion: "Use generic path like C:\\path\\to\\file or %USERPROFILE% variable" diff --git a/tests/validators/test_security.py b/tests/validators/test_security.py index 57172f28..d8930833 100644 --- a/tests/validators/test_security.py +++ b/tests/validators/test_security.py @@ -11,6 +11,7 @@ import pytest +from skillevaluator.config import load_pii_patterns from skillevaluator.reporting import CLIReporter, HTMLReporter, JSONReporter, MarkdownReporter from skillevaluator.utils.tool_runner import ToolResult, Tools from skillevaluator.validators.base import Finding, Severity, ValidationResult @@ -135,6 +136,55 @@ def test_detects_personal_path(self, tmp_path: Path): all_findings = result.errors + result.warnings assert any("path" in f.lower() for f in all_findings), f"Expected path detection. Findings: {all_findings}" + def test_detects_windows_personal_path_for_alice(self, tmp_path: Path): + """Windows C:\\Users\\alice paths are still flagged.""" + skill_dir = tmp_path / "win-alice-skill" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text("""--- +name: win-alice-skill +description: A skill with a Windows personal path for testing detection +--- + +# Path Test + +See C:\\Users\\alice\\Documents\\notes.txt +""") + result = SecurityValidator().validate_pii_only(skill_dir) + all_findings = result.errors + result.warnings + assert any("alice" in f and "C:\\Users\\" in f for f in all_findings), ( + f"Expected Windows path detection for alice. Findings: {all_findings}" + ) + + def test_detects_windows_personal_path_for_steve(self, tmp_path: Path): + """Windows C:\\Users\\steve paths are flagged (usernames starting with s).""" + skill_dir = tmp_path / "win-steve-skill" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text("""--- +name: win-steve-skill +description: A skill with a Windows personal path for testing detection +--- + +# Path Test + +See C:\\Users\\steve\\Documents\\notes.txt +""") + result = SecurityValidator().validate_pii_only(skill_dir) + all_findings = result.errors + result.warnings + assert any("steve" in f and "C:\\Users\\" in f for f in all_findings), ( + f"Expected Windows path detection for steve. Findings: {all_findings}" + ) + + def test_windows_personal_path_pattern_matches_usernames_starting_with_s(self): + """Loaded Windows personal-path regex uses a whitespace class, not letter s.""" + pattern = load_pii_patterns()["personal_paths"][1]["pattern"] + assert r"[^\\\s]" in pattern or r"[^\s]" in pattern + regex = re.compile(pattern, re.IGNORECASE) + assert regex.search(r"C:\Users\steve\Documents\notes.txt") + assert regex.search(r"C:\Users\alice\Documents\notes.txt") + assert regex.search(r"C:\Users\sam\file.txt") + assert regex.search(r"C:\Users\session\file.txt") + assert not regex.search(r"C:\Temp\file.txt") + def test_api_route_not_flagged_as_personal_path(self, tmp_path: Path): """Test that REST API routes like /users/:id are not flagged as personal macOS paths.""" skill_dir = tmp_path / "api-skill"