Skip to content
Merged
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ All notable changes to SkillEvaluator are documented in this file.

### Fixed

- Windows personal-path PII now flags `C:\Users\...` usernames that start with
`s` (for example `steve`), matching the intended whitespace class rather than
excluding the letter `s` ([#87](https://github.com/NVIDIA/SkillEvaluator/issues/87)).
- Quality scoring, script lint, and `create-eval-dataset` now treat `tools/`
the same as `scripts/` for executable helpers.
- License detection no longer treats a frontmatter `license` identifier as
Expand Down
2 changes: 1 addition & 1 deletion src/skillevaluator/config/pii_patterns.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ personal_paths:
- "DELETE /users"
- "api/users"

- pattern: 'C:\\Users\\[^\\s]+'
- pattern: "C:\\\\Users\\\\[^\\\\\\s]+"
severity: high
description: Personal Windows user path
suggestion: "Use generic path like C:\\path\\to\\file or %USERPROFILE% variable"
Expand Down
50 changes: 50 additions & 0 deletions tests/validators/test_security.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import pytest

from skillevaluator.config import load_pii_patterns
from skillevaluator.reporting import CLIReporter, HTMLReporter, JSONReporter, MarkdownReporter
from skillevaluator.utils.tool_runner import ToolResult, Tools
from skillevaluator.validators.base import Finding, Severity, ValidationResult
Expand Down Expand Up @@ -135,6 +136,55 @@ def test_detects_personal_path(self, tmp_path: Path):
all_findings = result.errors + result.warnings
assert any("path" in f.lower() for f in all_findings), f"Expected path detection. Findings: {all_findings}"

def test_detects_windows_personal_path_for_alice(self, tmp_path: Path):
"""Windows C:\\Users\\alice paths are still flagged."""
skill_dir = tmp_path / "win-alice-skill"
skill_dir.mkdir()
(skill_dir / "SKILL.md").write_text("""---
name: win-alice-skill
description: A skill with a Windows personal path for testing detection
---

# Path Test

See C:\\Users\\alice\\Documents\\notes.txt
""")
result = SecurityValidator().validate_pii_only(skill_dir)
all_findings = result.errors + result.warnings
assert any("alice" in f and "C:\\Users\\" in f for f in all_findings), (
f"Expected Windows path detection for alice. Findings: {all_findings}"
)

def test_detects_windows_personal_path_for_steve(self, tmp_path: Path):
"""Windows C:\\Users\\steve paths are flagged (usernames starting with s)."""
skill_dir = tmp_path / "win-steve-skill"
skill_dir.mkdir()
(skill_dir / "SKILL.md").write_text("""---
name: win-steve-skill
description: A skill with a Windows personal path for testing detection
---

# Path Test

See C:\\Users\\steve\\Documents\\notes.txt
""")
result = SecurityValidator().validate_pii_only(skill_dir)
all_findings = result.errors + result.warnings
assert any("steve" in f and "C:\\Users\\" in f for f in all_findings), (
f"Expected Windows path detection for steve. Findings: {all_findings}"
)

def test_windows_personal_path_pattern_matches_usernames_starting_with_s(self):
"""Loaded Windows personal-path regex uses a whitespace class, not letter s."""
pattern = load_pii_patterns()["personal_paths"][1]["pattern"]
assert r"[^\\\s]" in pattern or r"[^\s]" in pattern
regex = re.compile(pattern, re.IGNORECASE)
assert regex.search(r"C:\Users\steve\Documents\notes.txt")
assert regex.search(r"C:\Users\alice\Documents\notes.txt")
assert regex.search(r"C:\Users\sam\file.txt")
assert regex.search(r"C:\Users\session\file.txt")
assert not regex.search(r"C:\Temp\file.txt")

def test_api_route_not_flagged_as_personal_path(self, tmp_path: Path):
"""Test that REST API routes like /users/:id are not flagged as personal macOS paths."""
skill_dir = tmp_path / "api-skill"
Expand Down
Loading