diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eaaf579..791838e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ All notable changes to SkillEvaluator are documented in this file. ### Fixed +- Windows personal-path PII now flags `C:\Users\...` usernames that start with + `s` (for example `steve`), matching the intended whitespace class rather than + excluding the letter `s` ([#87](https://github.com/NVIDIA/SkillEvaluator/issues/87)). - Quality scoring, script lint, and `create-eval-dataset` now treat `tools/` the same as `scripts/` for executable helpers. - License detection no longer treats a frontmatter `license` identifier as diff --git a/src/skillevaluator/config/pii_patterns.yaml b/src/skillevaluator/config/pii_patterns.yaml index bef3d64c..06f85ec0 100644 --- a/src/skillevaluator/config/pii_patterns.yaml +++ b/src/skillevaluator/config/pii_patterns.yaml @@ -33,7 +33,7 @@ personal_paths: - "DELETE /users" - "api/users" - - pattern: 'C:\\Users\\[^\\s]+' + - pattern: "C:\\\\Users\\\\[^\\\\\\s]+" severity: high description: Personal Windows user path suggestion: "Use generic path like C:\\path\\to\\file or %USERPROFILE% variable" diff --git a/tests/validators/test_security.py b/tests/validators/test_security.py index 8cce7acc..7e6975f5 100644 --- a/tests/validators/test_security.py +++ b/tests/validators/test_security.py @@ -11,6 +11,7 @@ import pytest +from skillevaluator.config import load_pii_patterns from skillevaluator.reporting import CLIReporter, HTMLReporter, JSONReporter, MarkdownReporter from skillevaluator.utils.tool_runner import ToolResult, Tools from skillevaluator.validators.base import Finding, Severity, ValidationResult @@ -135,6 +136,55 @@ def test_detects_personal_path(self, tmp_path: Path): all_findings = result.errors + result.warnings assert any("path" in f.lower() for f in all_findings), f"Expected path detection. Findings: {all_findings}" + def test_detects_windows_personal_path_for_alice(self, tmp_path: Path): + """Windows C:\\Users\\alice paths are still flagged.""" + skill_dir = tmp_path / "win-alice-skill" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text("""--- +name: win-alice-skill +description: A skill with a Windows personal path for testing detection +--- + +# Path Test + +See C:\\Users\\alice\\Documents\\notes.txt +""") + result = SecurityValidator().validate_pii_only(skill_dir) + all_findings = result.errors + result.warnings + assert any("alice" in f and "C:\\Users\\" in f for f in all_findings), ( + f"Expected Windows path detection for alice. Findings: {all_findings}" + ) + + def test_detects_windows_personal_path_for_steve(self, tmp_path: Path): + """Windows C:\\Users\\steve paths are flagged (usernames starting with s).""" + skill_dir = tmp_path / "win-steve-skill" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text("""--- +name: win-steve-skill +description: A skill with a Windows personal path for testing detection +--- + +# Path Test + +See C:\\Users\\steve\\Documents\\notes.txt +""") + result = SecurityValidator().validate_pii_only(skill_dir) + all_findings = result.errors + result.warnings + assert any("steve" in f and "C:\\Users\\" in f for f in all_findings), ( + f"Expected Windows path detection for steve. Findings: {all_findings}" + ) + + def test_windows_personal_path_pattern_matches_usernames_starting_with_s(self): + """Loaded Windows personal-path regex uses a whitespace class, not letter s.""" + pattern = load_pii_patterns()["personal_paths"][1]["pattern"] + assert r"[^\\\s]" in pattern or r"[^\s]" in pattern + regex = re.compile(pattern, re.IGNORECASE) + assert regex.search(r"C:\Users\steve\Documents\notes.txt") + assert regex.search(r"C:\Users\alice\Documents\notes.txt") + assert regex.search(r"C:\Users\sam\file.txt") + assert regex.search(r"C:\Users\session\file.txt") + assert not regex.search(r"C:\Temp\file.txt") + def test_api_route_not_flagged_as_personal_path(self, tmp_path: Path): """Test that REST API routes like /users/:id are not flagged as personal macOS paths.""" skill_dir = tmp_path / "api-skill"