diff --git a/docs/07-evaluation-plan.md b/docs/07-evaluation-plan.md index 2fe3157..ee8c701 100644 --- a/docs/07-evaluation-plan.md +++ b/docs/07-evaluation-plan.md @@ -21,6 +21,12 @@ supported precision/recall/F1, contradiction recall, abstention, citation validity, and confusion reporting. This vertical slice supports the public demo but does not mark the broader M2 or M5 milestones complete. +A separate six-case `ssdlc_synthetic_v1` starter set now covers all six phases +with deterministic risk/compliance-gap detection, severity, policy-control, +evidence-locator, and schema metrics. Its manifest explicitly records +`not_expert_reviewed`; it validates the data and scoring contracts but does not +satisfy M4's 20–30 case or two-senior-reviewer requirements. + --- ## 1. Goals and Non-Goals diff --git a/evals/README.md b/evals/README.md index eed0ece..95aca65 100644 --- a/evals/README.md +++ b/evals/README.md @@ -13,9 +13,14 @@ Current scope: precision/recall/F1, contradiction recall, abstention rate, citation validity, and a verdict confusion matrix. This is a focused Design-phase addition; it does not claim completion of the broader M2 or M5 milestones. +- **Six-phase synthetic starter set**: one offline, license-clean case for each + SSDLC phase, with 24 golden risks and compliance gaps total, stable evidence + locators, project-local policy mappings, deterministic one-to-one matching, + and set/severity/policy/evidence scorecards. -Raw datasets are not committed. Use the dataset-specific fetch script to download -public data into ignored local paths. +External raw datasets are not committed. Use each dataset-specific fetch script +to download public data into ignored local paths. Fully synthetic fixtures may +be committed when their manifest and license metadata explicitly permit it. ```bash python evals/datasets/owasp_benchmark/fetch.py @@ -32,3 +37,19 @@ The runner writes: - `evals/reports//scorecard.md` M1 uses deterministic hard-key CWE triage scoring and does not use an LLM judge. + +## Six-phase synthetic starter set + +```bash +python -m evals.runner.run_eval \ + --dataset-id ssdlc_synthetic_v1 \ + --raw-dir evals/datasets/ssdlc_synthetic_v1 \ + --run-id local-six-phase \ + --repeats 1 +``` + +The dataset manifest pins every committed fixture and ground-truth file. Its +`review_status` is deliberately `not_expert_reviewed`: this is a small, +AI-assisted starter set and scorer contract, not completion of M4's 20–30 case, +two-expert annotation protocol. The checked-in deterministic oracle baseline +tests the harness itself and must not be reported as model performance. diff --git a/evals/adapters/ssdlc_synthetic.py b/evals/adapters/ssdlc_synthetic.py new file mode 100644 index 0000000..d546e57 --- /dev/null +++ b/evals/adapters/ssdlc_synthetic.py @@ -0,0 +1,123 @@ +"""Adapter for DocSentinel's six-phase, synthetic SSDLC golden set.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +from evals.models import EvalCase + +DATASET_ID = "ssdlc_synthetic_v1" +CASES_FILE = "cases.jsonl" +MANIFEST_FILE = "manifest.json" +PHASES = { + "requirements", + "design", + "development", + "testing", + "deployment", + "operations", +} + + +def to_cases(raw_dir: Path) -> Iterable[EvalCase]: + """Validate dataset integrity and yield normalized six-phase cases.""" + root = Path(raw_dir).resolve() + manifest = _load_json(root / MANIFEST_FILE) + _validate_manifest(root, manifest) + + seen_ids: set[str] = set() + seen_phases: set[str] = set() + with (root / CASES_FILE).open(encoding="utf-8") as handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + continue + try: + case = EvalCase.model_validate_json(line) + except ValueError as exc: + raise ValueError( + f"Invalid {CASES_FILE} line {line_number}: {exc}" + ) from exc + _validate_case(root, case, seen_ids) + seen_ids.add(case.case_id) + seen_phases.add(case.phase) + yield case + + if seen_phases != PHASES: + missing = sorted(PHASES - seen_phases) + extra = sorted(seen_phases - PHASES) + raise ValueError( + "Synthetic dataset must cover all six phases; " + f"missing={missing}, extra={extra}" + ) + expected_cases = int(manifest.get("n_cases") or 0) + if len(seen_ids) != expected_cases: + raise ValueError( + f"Manifest declares {expected_cases} cases but {len(seen_ids)} were loaded" + ) + + +def _validate_case(root: Path, case: EvalCase, seen_ids: set[str]) -> None: + if case.dataset_id != DATASET_ID: + raise ValueError(f"Unexpected dataset_id for {case.case_id}: {case.dataset_id}") + if case.case_id in seen_ids: + raise ValueError(f"Duplicate case_id: {case.case_id}") + if case.phase not in PHASES: + raise ValueError(f"Unsupported phase for {case.case_id}: {case.phase}") + if case.skill_id != f"ssdlc-{case.phase}": + raise ValueError( + f"Skill/phase mismatch for {case.case_id}: {case.skill_id} / {case.phase}" + ) + if not case.inputs: + raise ValueError(f"Case has no input documents: {case.case_id}") + if not case.ground_truth.risk_items or not case.ground_truth.compliance_gaps: + raise ValueError(f"Case has incomplete finding ground truth: {case.case_id}") + for item in [ + *case.ground_truth.risk_items, + *case.ground_truth.compliance_gaps, + ]: + if not item.id or not item.match_terms or not item.evidence_locators: + raise ValueError( + "Ground truth needs id, match_terms, and evidence_locators: " + f"{case.case_id}" + ) + for item in case.inputs: + path = (root / item.path).resolve() + if root not in path.parents or not path.is_file(): + raise ValueError( + f"Missing or escaping input for {case.case_id}: {item.path}" + ) + + +def _validate_manifest(root: Path, manifest: dict[str, Any]) -> None: + if manifest.get("dataset_id") != DATASET_ID: + raise ValueError( + f"Unexpected manifest dataset_id: {manifest.get('dataset_id')}" + ) + if manifest.get("provenance") != "fully_synthetic": + raise ValueError( + "Synthetic dataset manifest must declare fully_synthetic provenance" + ) + if manifest.get("review_status") != "not_expert_reviewed": + raise ValueError("Manifest must preserve the not_expert_reviewed boundary") + files = manifest.get("files") or {} + if not files: + raise ValueError("Manifest has no file checksums") + for relative, expected in sorted(files.items()): + path = (root / relative).resolve() + if root not in path.parents or not path.is_file(): + raise ValueError( + f"Manifest path missing or escaping dataset root: {relative}" + ) + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected: + raise ValueError( + f"Checksum mismatch for {relative}: {actual} != {expected}" + ) + + +def _load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) diff --git a/evals/configs/matrix.yaml b/evals/configs/matrix.yaml index fa9ec8b..6f3abee 100644 --- a/evals/configs/matrix.yaml +++ b/evals/configs/matrix.yaml @@ -8,10 +8,13 @@ datasets: repeats: 1 phase: testing skill_id: ssdlc-testing + ssdlc_synthetic_v1: + repeats: 1 + phase: full_ssdlc + skill_id: phase-specific models: - id: ollama-local provider: ollama model_id: llama2 temperature: 0.0 - diff --git a/evals/datasets/ssdlc_synthetic_v1/README.md b/evals/datasets/ssdlc_synthetic_v1/README.md new file mode 100644 index 0000000..69ebf4b --- /dev/null +++ b/evals/datasets/ssdlc_synthetic_v1/README.md @@ -0,0 +1,34 @@ +# Six-Phase Synthetic SSDLC Golden Set v1 + +This small offline dataset exercises DocSentinel's requirements, design, +development, testing, deployment, and operations skills. Every organization, +system, artifact, and finding is fictional. The files contain no copied client +material, production secrets, or third-party benchmark records and are covered +by the repository's MIT license. + +Each source statement has a stable locator such as `REQ-02` or `DEP-03`. +`cases.jsonl` records expected risks, severity, project-local policy controls, +and supporting locators. `manifest.json` pins every committed data file by +SHA-256 so an evaluation cannot silently run against modified ground truth. + +Important review boundary: these AI-assisted fixtures are intentionally marked +`not_expert_reviewed`. They are a contestable starter set and a deterministic +scorer/CI contract, not an expert-approved security baseline. The committed +oracle scorecard proves the harness can recover known truth; it is not model +performance. Maintainers should review and revise findings before promoting a +future baseline to `approved`. + +Run an actual pipeline evaluation with: + +```bash +python -m evals.runner.run_eval \ + --dataset-id ssdlc_synthetic_v1 \ + --raw-dir evals/datasets/ssdlc_synthetic_v1 \ + --run-id local-six-phase \ + --repeats 1 +``` + +The scorer is deterministic: exact IDs match first; otherwise at least 60% of +an expected finding's declared terms must appear in the prediction. It then +reports risk and compliance-gap precision/recall/F1 plus severity, policy +mapping, evidence-locator, and schema fidelity on matched findings. diff --git a/evals/datasets/ssdlc_synthetic_v1/cases.jsonl b/evals/datasets/ssdlc_synthetic_v1/cases.jsonl new file mode 100644 index 0000000..0fcef2b --- /dev/null +++ b/evals/datasets/ssdlc_synthetic_v1/cases.jsonl @@ -0,0 +1,6 @@ +{"case_id":"synthetic-requirements-001","dataset_id":"ssdlc_synthetic_v1","phase":"requirements","skill_id":"ssdlc-requirements","inputs":[{"path":"inputs/requirements.md","type":"markdown"}],"ground_truth":{"risk_items":[{"id":"REQ-RISK-MFA","title":"Privileged accounts lack MFA","severity":"high","description":"Password-only administrator authentication increases account-takeover risk.","match_terms":["administrator","password","multi-factor authentication"],"evidence_locators":["REQ-02"]},{"id":"REQ-RISK-RETENTION","title":"Personal data has indefinite retention","severity":"medium","description":"No deletion workflow or retention schedule is defined for tickets and logs.","match_terms":["retained indefinitely","deletion","retention schedule"],"evidence_locators":["REQ-03"]}],"compliance_gaps":[{"id":"REQ-GAP-MFA","framework":"generic-ssdlc","control_or_clause":"GEN-IAM-01","gap_description":"Strong authentication is not required for administrator access.","match_terms":["strong authentication","administrator","multi-factor"],"evidence_locators":["REQ-02"]},{"id":"REQ-GAP-RETENTION","framework":"generic-ssdlc","control_or_clause":"GEN-PRV-02","gap_description":"The requirements omit a data-retention and deletion policy.","match_terms":["data retention","deletion","schedule"],"evidence_locators":["REQ-03"]}]},"meta":{"license":"MIT","provenance":"fully_synthetic","review_status":"not_expert_reviewed","annotators":0,"iaa":null}} +{"case_id":"synthetic-design-001","dataset_id":"ssdlc_synthetic_v1","phase":"design","skill_id":"ssdlc-design","inputs":[{"path":"inputs/design.md","type":"markdown"}],"ground_truth":{"risk_items":[{"id":"DES-RISK-IDENTITY","title":"Client-controlled identity header enables impersonation","severity":"high","description":"The API trusts an unsigned browser-supplied user identifier as account identity.","match_terms":["X-User-Id","account identity","without a signed session"],"evidence_locators":["DES-01"]},{"id":"DES-RISK-TRANSPORT","title":"Sensitive metadata crosses an unencrypted service hop","severity":"high","description":"Customer document metadata is sent to the indexer over plaintext HTTP.","match_terms":["unencrypted HTTP","customer names","indexing service"],"evidence_locators":["DES-02"]}],"compliance_gaps":[{"id":"DES-GAP-IDENTITY","framework":"generic-ssdlc","control_or_clause":"GEN-IAM-01","gap_description":"The design lacks a trusted authentication mechanism for the document API.","match_terms":["authentication","trusted","document API"],"evidence_locators":["DES-01"]},{"id":"DES-GAP-TRANSPORT","framework":"generic-ssdlc","control_or_clause":"GEN-ENC-01","gap_description":"Encryption in transit is missing between the API and indexing service.","match_terms":["encryption in transit","API","indexing service"],"evidence_locators":["DES-02"]}]},"meta":{"license":"MIT","provenance":"fully_synthetic","review_status":"not_expert_reviewed","annotators":0,"iaa":null}} +{"case_id":"synthetic-development-001","dataset_id":"ssdlc_synthetic_v1","phase":"development","skill_id":"ssdlc-development","inputs":[{"path":"inputs/development.md","type":"markdown"}],"ground_truth":{"risk_items":[{"id":"DEV-RISK-SQLI","title":"Raw search input is interpolated into SQL","severity":"critical","description":"Direct query-string interpolation creates SQL injection risk.","match_terms":["SQL","interpolating","query parameter"],"evidence_locators":["DEV-01"]},{"id":"DEV-RISK-CVE","title":"Deployed dependencies lack continuous vulnerability monitoring","severity":"high","description":"Quarterly manual review leaves newly disclosed dependency CVEs undetected.","match_terms":["quarterly","continuous alerting","CVE"],"evidence_locators":["DEV-02"]}],"compliance_gaps":[{"id":"DEV-GAP-SQLI","framework":"generic-ssdlc","control_or_clause":"GEN-SEC-01","gap_description":"The search handler does not validate input or use a parameterized query.","match_terms":["search handler","input","parameterized"],"evidence_locators":["DEV-01"]},{"id":"DEV-GAP-CVE","framework":"generic-ssdlc","control_or_clause":"GEN-SC-03","gap_description":"Continuous dependency vulnerability monitoring is absent.","match_terms":["dependency vulnerability","continuous","monitoring"],"evidence_locators":["DEV-02"]}]},"meta":{"license":"MIT","provenance":"fully_synthetic","review_status":"not_expert_reviewed","annotators":0,"iaa":null}} +{"case_id":"synthetic-testing-001","dataset_id":"ssdlc_synthetic_v1","phase":"testing","skill_id":"ssdlc-testing","inputs":[{"path":"inputs/testing.md","type":"markdown"}],"ground_truth":{"risk_items":[{"id":"TST-RISK-COVERAGE","title":"DAST excludes the privileged administration surface","severity":"high","description":"Unauthenticated scanning leaves the entire admin API outside dynamic testing.","match_terms":["DAST","admin API","excludes"],"evidence_locators":["TST-01"]},{"id":"TST-RISK-BYPASS","title":"Critical authentication bypass is unretested before release","severity":"critical","description":"A known administrator bypass remains open and verification is deferred until after launch.","match_terms":["authentication bypass","critical","retest"],"evidence_locators":["TST-02"]}],"compliance_gaps":[{"id":"TST-GAP-COVERAGE","framework":"generic-ssdlc","control_or_clause":"GEN-VUL-02","gap_description":"Dynamic testing does not cover authenticated administration endpoints.","match_terms":["dynamic testing","administration","authenticated"],"evidence_locators":["TST-01"]},{"id":"TST-GAP-BYPASS","framework":"generic-ssdlc","control_or_clause":"GEN-VUL-01","gap_description":"The critical authentication finding lacks pre-release remediation verification.","match_terms":["critical","authentication finding","verification"],"evidence_locators":["TST-02"]}]},"meta":{"license":"MIT","provenance":"fully_synthetic","review_status":"not_expert_reviewed","annotators":0,"iaa":null}} +{"case_id":"synthetic-deployment-001","dataset_id":"ssdlc_synthetic_v1","phase":"deployment","skill_id":"ssdlc-deployment","inputs":[{"path":"inputs/deployment.md","type":"markdown"}],"ground_truth":{"risk_items":[{"id":"DEP-RISK-IMAGE","title":"Production image is released without vulnerability scanning","severity":"high","description":"Unknown container and infrastructure-image vulnerabilities can reach production.","match_terms":["release image","without","vulnerability scan"],"evidence_locators":["DEP-01"]},{"id":"DEP-RISK-RESTORE","title":"Backups have no demonstrated restore capability","severity":"high","description":"Untested restores and undefined recovery objectives make recovery uncertain.","match_terms":["backups","never restored","recovery"],"evidence_locators":["DEP-02"]}],"compliance_gaps":[{"id":"DEP-GAP-IMAGE","framework":"generic-ssdlc","control_or_clause":"GEN-VUL-03","gap_description":"Container and infrastructure images are not scanned before promotion.","match_terms":["container","images","scanned"],"evidence_locators":["DEP-01"]},{"id":"DEP-GAP-RESTORE","framework":"generic-ssdlc","control_or_clause":"GEN-BCP-01","gap_description":"Backup recovery is not tested and recovery objectives are undefined.","match_terms":["backup","recovery","tested"],"evidence_locators":["DEP-02"]}]},"meta":{"license":"MIT","provenance":"fully_synthetic","review_status":"not_expert_reviewed","annotators":0,"iaa":null}} +{"case_id":"synthetic-operations-001","dataset_id":"ssdlc_synthetic_v1","phase":"operations","skill_id":"ssdlc-operations","inputs":[{"path":"inputs/operations.md","type":"markdown"}],"ground_truth":{"risk_items":[{"id":"OPS-RISK-LOGS","title":"Short-lived local logs cannot support durable incident investigation","severity":"high","description":"Node-local security logs are overwritten after one day and lack tamper resistance.","match_terms":["security logs","overwritten","tamper-resistant"],"evidence_locators":["OPS-01"]},{"id":"OPS-RISK-PATCH","title":"Critical dependency advisories have no remediation SLA","severity":"high","description":"Quarterly backlog review allows critical dependency exposure to persist.","match_terms":["critical dependency","quarter","remediation deadline"],"evidence_locators":["OPS-02"]}],"compliance_gaps":[{"id":"OPS-GAP-LOGS","framework":"generic-ssdlc","control_or_clause":"GEN-LOG-02","gap_description":"Log retention and tamper-resistant centralized storage are insufficient.","match_terms":["log retention","tamper-resistant","centralized"],"evidence_locators":["OPS-01"]},{"id":"OPS-GAP-PATCH","framework":"generic-ssdlc","control_or_clause":"GEN-SC-03","gap_description":"Critical dependency vulnerabilities have no defined remediation deadline.","match_terms":["critical dependency","remediation","deadline"],"evidence_locators":["OPS-02"]}]},"meta":{"license":"MIT","provenance":"fully_synthetic","review_status":"not_expert_reviewed","annotators":0,"iaa":null}} diff --git a/evals/datasets/ssdlc_synthetic_v1/inputs/deployment.md b/evals/datasets/ssdlc_synthetic_v1/inputs/deployment.md new file mode 100644 index 0000000..561d395 --- /dev/null +++ b/evals/datasets/ssdlc_synthetic_v1/inputs/deployment.md @@ -0,0 +1,13 @@ +# Deployment Review: ExampleCo Reporting Service + +[DEP-01] The release image is promoted to production without a container or +infrastructure image vulnerability scan. + +[DEP-02] Database backups are created nightly, but the team has never restored +one and has not measured recovery time or recovery-point objectives. + +[DEP-03] Production credentials are injected from a managed secret store and +are absent from the image and deployment manifest. + +[DEP-04] The service runs as a non-root user with a read-only root filesystem +and drops all Linux capabilities not required at runtime. diff --git a/evals/datasets/ssdlc_synthetic_v1/inputs/design.md b/evals/datasets/ssdlc_synthetic_v1/inputs/design.md new file mode 100644 index 0000000..5f0c3a0 --- /dev/null +++ b/evals/datasets/ssdlc_synthetic_v1/inputs/design.md @@ -0,0 +1,14 @@ +# Design Review: ExampleCo Document Service + +[DES-01] A browser sends `X-User-Id` directly to the document API. The API uses +that value as the account identity without a signed session or trusted gateway. + +[DES-02] The API sends document metadata to an internal indexing service over +unencrypted HTTP on the cluster network. Metadata includes customer names and +document titles. + +[DES-03] Object storage denies public access, and each document is encrypted at +rest with a managed key whose rotation is enabled. + +[DES-04] Authorization checks bind each document ID to the authenticated tenant +before object storage is queried. diff --git a/evals/datasets/ssdlc_synthetic_v1/inputs/development.md b/evals/datasets/ssdlc_synthetic_v1/inputs/development.md new file mode 100644 index 0000000..3c0ea34 --- /dev/null +++ b/evals/datasets/ssdlc_synthetic_v1/inputs/development.md @@ -0,0 +1,14 @@ +# Development Review: ExampleCo Search API + +[DEV-01] The search handler builds a SQL statement by interpolating the raw +`query` request parameter into the statement text before execution. + +[DEV-02] Dependency versions are pinned, but vulnerability review is a manual +quarterly spreadsheet exercise. There is no continuous alerting for newly +disclosed CVEs affecting versions already deployed. + +[DEV-03] The repository blocks commits containing known credential formats and +uses short-lived workload identity for CI publishing. + +[DEV-04] Unit tests cover authorization failures and output encoding for search +results. diff --git a/evals/datasets/ssdlc_synthetic_v1/inputs/operations.md b/evals/datasets/ssdlc_synthetic_v1/inputs/operations.md new file mode 100644 index 0000000..3acda11 --- /dev/null +++ b/evals/datasets/ssdlc_synthetic_v1/inputs/operations.md @@ -0,0 +1,13 @@ +# Operations Review: ExampleCo Customer Platform + +[OPS-01] Security logs remain only on each application node and are overwritten +after 24 hours. They are not forwarded to tamper-resistant centralized storage. + +[OPS-02] Critical dependency advisories are collected, but the backlog is +reviewed only once per quarter and no remediation deadline is defined. + +[OPS-03] The on-call rotation is staffed continuously and the incident runbook +contains tested escalation contacts and customer-notification templates. + +[OPS-04] Administrative sessions are revoked when an employee leaves the +organization. diff --git a/evals/datasets/ssdlc_synthetic_v1/inputs/requirements.md b/evals/datasets/ssdlc_synthetic_v1/inputs/requirements.md new file mode 100644 index 0000000..c6bcbc8 --- /dev/null +++ b/evals/datasets/ssdlc_synthetic_v1/inputs/requirements.md @@ -0,0 +1,13 @@ +# Requirements Review: ExampleCo Support Portal + +[REQ-01] The portal stores customer names, email addresses, and support-ticket +text. The product is intended for customers in several regions. + +[REQ-02] Administrators authenticate with a password. Multi-factor +authentication is explicitly out of scope for the first release. + +[REQ-03] Ticket records and access logs are retained indefinitely. The +requirements do not define a deletion request workflow or retention schedule. + +[REQ-04] Ordinary customers can view only tickets belonging to their own +account, and every access-control decision is logged. diff --git a/evals/datasets/ssdlc_synthetic_v1/inputs/testing.md b/evals/datasets/ssdlc_synthetic_v1/inputs/testing.md new file mode 100644 index 0000000..c3a2554 --- /dev/null +++ b/evals/datasets/ssdlc_synthetic_v1/inputs/testing.md @@ -0,0 +1,13 @@ +# Testing Review: ExampleCo Administration API + +[TST-01] The DAST plan covers public customer endpoints but explicitly excludes +the `/admin` API because the scanner cannot authenticate to that route. + +[TST-02] A penetration test found an administrator authentication bypass rated +critical. The release plan defers the fix and any retest until after launch. + +[TST-03] SAST runs on every pull request and blocks newly introduced high or +critical findings. + +[TST-04] Test evidence includes the scanner version, configuration, timestamp, +and immutable report checksum. diff --git a/evals/datasets/ssdlc_synthetic_v1/manifest.json b/evals/datasets/ssdlc_synthetic_v1/manifest.json new file mode 100644 index 0000000..abbd50b --- /dev/null +++ b/evals/datasets/ssdlc_synthetic_v1/manifest.json @@ -0,0 +1,29 @@ +{ + "dataset_id": "ssdlc_synthetic_v1", + "version": "1.0.0", + "license": "MIT (same as the containing repository)", + "provenance": "fully_synthetic", + "authoring": "AI-assisted synthetic fixtures with deterministic validation", + "review_status": "not_expert_reviewed", + "contains_real_personal_data": false, + "contains_third_party_content": false, + "n_cases": 6, + "phases": [ + "requirements", + "design", + "development", + "testing", + "deployment", + "operations" + ], + "files": { + "README.md": "2b587a684e47c266fc9ff676e98dd06c0a93658ff772eec9480e621f4d9eebc1", + "cases.jsonl": "f3a81e28128a7dbba58a094a5cdd54d496e90a43b98ec0102686db1caa612cef", + "inputs/deployment.md": "dd30bd9bf38ebd3645a92f7bb245333f4504c9b64b990c172380d72ec0636e33", + "inputs/design.md": "48855d7fc5164f068df65de7f7fb1a4cbe6f88c24835703533303c17457a1686", + "inputs/development.md": "9777a29e273904461a6fd32f92bae135202693d53babecb4c8b53e53fd8d2fc2", + "inputs/operations.md": "c5348319bbe56b3d42e82d5bd8943edc1c74760baae05be1c34ea60e309af02b", + "inputs/requirements.md": "bd279ff8a023bc8c79ea21f8d0b7c752167d91e1a0b488285765369ab780c58b", + "inputs/testing.md": "03fbe427e74ab59dd54e6fec6523236197ed940a6200a84a03aa816c312478ab" + } +} diff --git a/evals/models.py b/evals/models.py index 5c0b074..3062889 100644 --- a/evals/models.py +++ b/evals/models.py @@ -18,6 +18,7 @@ "full_ssdlc", ] TruthLabel = Literal["true_positive", "false_positive"] +Severity = Literal["low", "medium", "high", "critical"] class EvalInput(BaseModel): @@ -36,12 +37,34 @@ class VulnerabilityTruth(BaseModel): test_name: str | None = None +class RiskTruth(BaseModel): + """Expected report risk with deterministic matching and evidence anchors.""" + + id: str | None = None + title: str | None = None + severity: Severity + description: str + match_terms: list[str] = Field(default_factory=list) + evidence_locators: list[str] = Field(default_factory=list) + + +class ComplianceGapTruth(BaseModel): + """Expected policy gap aligned to one project-local control identifier.""" + + id: str | None = None + framework: str + control_or_clause: str + gap_description: str + match_terms: list[str] = Field(default_factory=list) + evidence_locators: list[str] = Field(default_factory=list) + + class EvalGroundTruth(BaseModel): """Ground-truth item sets, aligned to the assessment report schema.""" threats: list[dict[str, Any]] = Field(default_factory=list) - risk_items: list[dict[str, Any]] = Field(default_factory=list) - compliance_gaps: list[dict[str, Any]] = Field(default_factory=list) + risk_items: list[RiskTruth] = Field(default_factory=list) + compliance_gaps: list[ComplianceGapTruth] = Field(default_factory=list) vulnerabilities: list[VulnerabilityTruth] = Field(default_factory=list) @@ -75,4 +98,3 @@ class RunConfig(BaseModel): phase: Phase = "testing" skill_id: str = "ssdlc-testing" limit: int | None = Field(default=None, ge=1) - diff --git a/evals/registry.yaml b/evals/registry.yaml index 67f91e3..2171756 100644 --- a/evals/registry.yaml +++ b/evals/registry.yaml @@ -14,3 +14,19 @@ datasets: contamination_risk: high n_cases: 2740 raw_data_policy: fetch_only_do_not_commit + scorer: triage + ssdlc_synthetic_v1: + id: ssdlc_synthetic_v1 + name: DocSentinel Six-Phase Synthetic SSDLC Golden Set v1 + source_url: https://github.com/arthurpanhku/DocSentinel/tree/main/evals/datasets/ssdlc_synthetic_v1 + license: MIT (same as the containing repository) + phase: full_ssdlc + skill_id: phase-specific + adapter: evals.adapters.ssdlc_synthetic:to_cases + scorer: ssdlc_golden + checksum_algorithm: sha256-per-file-manifest + checksum: see evals/datasets/ssdlc_synthetic_v1/manifest.json + contamination_risk: low + review_status: not_expert_reviewed + n_cases: 6 + raw_data_policy: committed_fully_synthetic diff --git a/evals/reports/.gitignore b/evals/reports/.gitignore index 9f56b6f..020968f 100644 --- a/evals/reports/.gitignore +++ b/evals/reports/.gitignore @@ -1,4 +1,4 @@ * !.gitignore !baselines/ - +!baselines/** diff --git a/evals/reports/baselines/ssdlc-synthetic-oracle-v1.json b/evals/reports/baselines/ssdlc-synthetic-oracle-v1.json new file mode 100644 index 0000000..b6b127f --- /dev/null +++ b/evals/reports/baselines/ssdlc-synthetic-oracle-v1.json @@ -0,0 +1,43 @@ +{ + "approved": false, + "baseline_kind": "deterministic_oracle_scorer_contract", + "dataset_id": "ssdlc_synthetic_v1", + "description": "Perfect synthetic predictions used only to regression-test adapter, runner, matching, and scorecard contracts. This is not model performance and the golden findings are not expert reviewed.", + "metrics": { + "compliance_gap_detection": { + "expected": 12, + "f1": 1.0, + "false_negative": 0, + "false_positive": 0, + "precision": 1.0, + "predicted": 12, + "recall": 1.0, + "true_positive": 12 + }, + "evidence_locator_accuracy_on_matched": 1.0, + "overall_detection": { + "expected": 24, + "f1": 1.0, + "false_negative": 0, + "false_positive": 0, + "precision": 1.0, + "predicted": 24, + "recall": 1.0, + "true_positive": 24 + }, + "policy_mapping_accuracy_on_matched": 1.0, + "risk_detection": { + "expected": 12, + "f1": 1.0, + "false_negative": 0, + "false_positive": 0, + "precision": 1.0, + "predicted": 12, + "recall": 1.0, + "true_positive": 12 + }, + "schema_validity": 1.0, + "severity_accuracy_on_matched": 1.0 + }, + "review_status": "not_expert_reviewed" +} diff --git a/evals/runner/run_eval.py b/evals/runner/run_eval.py index 4455e36..254afa0 100644 --- a/evals/runner/run_eval.py +++ b/evals/runner/run_eval.py @@ -21,6 +21,18 @@ from app.services.assessment_service import AssessmentRunner, AssessmentService from evals.models import EvalCase, RunConfig from evals.runner.parse import parse_inputs +from evals.scoring.scorers.set_detection import ( + GoldenRecord, +) +from evals.scoring.scorers.set_detection import ( + record_from_report as golden_record_from_report, +) +from evals.scoring.scorers.set_detection import ( + score_records as score_golden_records, +) +from evals.scoring.scorers.set_detection import ( + serialize_record as serialize_golden_record, +) from evals.scoring.scorers.triage import ( TriageRecord, record_from_report, @@ -78,12 +90,15 @@ async def run_cases( report_root: Path | None = None, assessment_runner: AssessmentRunner | None = None, ) -> dict[str, Any]: - """Run cases, score SAST triage, and write scorecard files.""" + """Run cases, apply the registry-selected scorer, and write scorecards.""" selected = list(cases) if cfg.limit is not None: selected = selected[: cfg.limit] - records: list[TriageRecord] = [] + dataset_meta = _dataset_meta(cfg.dataset_id) + scorer = str(dataset_meta.get("scorer") or "triage") + triage_records: list[TriageRecord] = [] + golden_records: list[GoldenRecord] = [] for case in selected: for repeat in range(cfg.repeats): report = await run_case( @@ -92,14 +107,27 @@ async def run_cases( input_root=input_root, assessment_runner=assessment_runner, ) - records.append(record_from_report(case, report, repeat)) - - scorecard = build_scorecard( - records, - cfg, - n_cases=len(selected), - dataset_meta=_dataset_meta(cfg.dataset_id), - ) + if scorer == "triage": + triage_records.append(record_from_report(case, report, repeat)) + elif scorer == "ssdlc_golden": + golden_records.append(golden_record_from_report(case, report, repeat)) + else: + raise ValueError(f"Unsupported scorer for {cfg.dataset_id}: {scorer}") + + if scorer == "triage": + scorecard = build_scorecard( + triage_records, + cfg, + n_cases=len(selected), + dataset_meta=dataset_meta, + ) + else: + scorecard = build_golden_scorecard( + golden_records, + cfg, + n_cases=len(selected), + dataset_meta=dataset_meta, + ) write_scorecard( scorecard, (report_root or EVALS_DIR / "reports") / cfg.run_id, @@ -107,6 +135,58 @@ async def run_cases( return scorecard +def build_golden_scorecard( + records: list[GoldenRecord], + cfg: RunConfig, + *, + n_cases: int, + dataset_meta: dict[str, Any], +) -> dict[str, Any]: + """Build a six-phase set-detection and fidelity scorecard.""" + phase_skill: dict[tuple[str, str], list[GoldenRecord]] = {} + for record in records: + phase_skill.setdefault((record.phase, record.skill_id), []).append(record) + return { + "schema_version": "eval-scorecard-v2", + "run_id": cfg.run_id, + "created_at": datetime.now(UTC).isoformat(), + "dataset": { + "id": cfg.dataset_id, + "source_url": dataset_meta.get("source_url"), + "license": dataset_meta.get("license"), + "checksum_algorithm": dataset_meta.get("checksum_algorithm"), + "checksum": dataset_meta.get("checksum"), + "contamination_risk": dataset_meta.get("contamination_risk"), + "review_status": dataset_meta.get("review_status"), + }, + "model": { + "provider": cfg.provider, + "model_id": cfg.model_id, + "temperature": cfg.temperature, + }, + "run_config": { + "repeats": cfg.repeats, + "timeout_seconds": cfg.timeout_seconds, + "collaborative": cfg.collaborative, + "phase": cfg.phase, + "skill_id": cfg.skill_id, + "limit": cfg.limit, + }, + "n_cases": n_cases, + "n_records": len(records), + "metrics": score_golden_records(records), + "by_phase_skill": [ + { + "phase": phase, + "skill_id": skill_id, + "metrics": score_golden_records(group), + } + for (phase, skill_id), group in sorted(phase_skill.items()) + ], + "cases": [serialize_golden_record(record) for record in records], + } + + def build_scorecard( records: list[TriageRecord], cfg: RunConfig, @@ -182,6 +262,8 @@ def write_scorecard(scorecard: dict[str, Any], output_dir: Path) -> None: def render_scorecard_markdown(scorecard: dict[str, Any]) -> str: + if scorecard["schema_version"] == "eval-scorecard-v2": + return render_golden_scorecard_markdown(scorecard) metrics = scorecard["metrics"] lines = [ f"# Eval Scorecard: {scorecard['run_id']}", @@ -236,6 +318,77 @@ def render_scorecard_markdown(scorecard: dict[str, Any]) -> str: return "\n".join(lines) +def render_golden_scorecard_markdown(scorecard: dict[str, Any]) -> str: + """Render six-phase finding and fidelity metrics without overstating review.""" + metrics = scorecard["metrics"] + lines = [ + f"# Eval Scorecard: {scorecard['run_id']}", + "", + f"- Dataset: `{scorecard['dataset']['id']}`", + f"- Model: `{scorecard['model']['provider']}:{scorecard['model']['model_id']}`", + f"- Repeats: `{scorecard['run_config']['repeats']}`", + f"- Contamination risk: `{scorecard['dataset']['contamination_risk']}`", + f"- Review status: `{scorecard['dataset']['review_status']}`", + "", + "> This synthetic dataset has not been expert reviewed. Scores are", + "> provisional until maintainers approve the golden findings.", + "", + "## Overall Metrics", + "", + "| Surface | Precision | Recall | F1 | FP | FN |", + "| :--- | ---: | ---: | ---: | ---: | ---: |", + ] + for label, key in ( + ("Risk findings", "risk_detection"), + ("Compliance gaps", "compliance_gap_detection"), + ("All findings", "overall_detection"), + ): + group = metrics[key] + lines.append( + f"| {label} | {group['precision']:.4f} | {group['recall']:.4f} | " + f"{group['f1']:.4f} | {group['false_positive']} | " + f"{group['false_negative']} |" + ) + lines.extend( + [ + "", + "| Fidelity check | Value |", + "| :--- | ---: |", + "| Severity accuracy (matched risks) | " + f"{metrics['severity_accuracy_on_matched']:.4f} |", + "| Policy mapping accuracy (matched gaps) | " + f"{metrics['policy_mapping_accuracy_on_matched']:.4f} |", + "| Evidence locator accuracy (matched findings) | " + f"{metrics['evidence_locator_accuracy_on_matched']:.4f} |", + f"| Schema validity | {metrics['schema_validity']:.4f} |", + "", + "## By Skill / Phase", + "", + "| Phase | Skill | Finding F1 | Severity | Policy | Evidence |", + "| :--- | :--- | ---: | ---: | ---: | ---: |", + ] + ) + for item in scorecard["by_phase_skill"]: + group = item["metrics"] + lines.append( + f"| {item['phase']} | {item['skill_id']} | " + f"{group['overall_detection']['f1']:.4f} | " + f"{group['severity_accuracy_on_matched']:.4f} | " + f"{group['policy_mapping_accuracy_on_matched']:.4f} | " + f"{group['evidence_locator_accuracy_on_matched']:.4f} |" + ) + lines.extend( + [ + "", + "Matching is deterministic and one-to-one. Exact finding IDs win; " + "otherwise at least 60% of the case's declared match terms must appear. " + "No embedding model or LLM judge is used.", + "", + ] + ) + return "\n".join(lines) + + def load_adapter(dataset_id: str): dataset = _dataset_meta(dataset_id) adapter_path = dataset.get("adapter") diff --git a/evals/scoring/scorers/set_detection.py b/evals/scoring/scorers/set_detection.py index 790fd9d..dcb41dc 100644 --- a/evals/scoring/scorers/set_detection.py +++ b/evals/scoring/scorers/set_detection.py @@ -1,2 +1,288 @@ -"""Future set-detection scorer for risks, threats, and compliance gaps.""" +"""Deterministic set scoring for synthetic SSDLC risks and policy gaps.""" +from __future__ import annotations + +import re +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, TypeVar + +from app.models.assessment import AssessmentReport +from evals.models import EvalCase + +PredictedItem = TypeVar("PredictedItem") +ExpectedItem = TypeVar("ExpectedItem") + + +@dataclass(frozen=True) +class GoldenRecord: + case_id: str + phase: str + skill_id: str + repeat: int + risk_expected: int + risk_predicted: int + risk_matched: int + gap_expected: int + gap_predicted: int + gap_matched: int + severity_total: int + severity_correct: int + policy_total: int + policy_correct: int + evidence_total: int + evidence_correct: int + matches: tuple[dict[str, Any], ...] + + +def record_from_report( + case: EvalCase, + report: AssessmentReport, + repeat: int, +) -> GoldenRecord: + """Match one structured report against a synthetic case's golden findings.""" + risk_matches = _match_items( + report.risk_items, + case.ground_truth.risk_items, + predicted_id=lambda item: item.id, + predicted_text=lambda item: f"{item.title} {item.description or ''}", + expected_id=lambda item: item.id or "", + expected_terms=lambda item: item.match_terms, + ) + gap_matches = _match_items( + report.compliance_gaps, + case.ground_truth.compliance_gaps, + predicted_id=lambda item: item.id, + predicted_text=lambda item: ( + f"{item.framework or ''} {item.control_or_clause} {item.gap_description}" + ), + expected_id=lambda item: item.id or "", + expected_terms=lambda item: item.match_terms, + ) + citations = {source.id: source for source in report.sources} + details: list[dict[str, Any]] = [] + severity_correct = 0 + evidence_correct = 0 + for predicted_index, expected_index, score in risk_matches: + predicted = report.risk_items[predicted_index] + expected = case.ground_truth.risk_items[expected_index] + severity_match = predicted.severity == expected.severity + evidence_match = _evidence_matches( + predicted.source_ref, + predicted.citation_ids, + expected.evidence_locators, + citations, + ) + severity_correct += int(severity_match) + evidence_correct += int(evidence_match) + details.append( + { + "kind": "risk", + "expected_id": expected.id, + "predicted_id": predicted.id, + "match_score": score, + "severity_correct": severity_match, + "evidence_correct": evidence_match, + } + ) + + policy_correct = 0 + for predicted_index, expected_index, score in gap_matches: + predicted = report.compliance_gaps[predicted_index] + expected = case.ground_truth.compliance_gaps[expected_index] + policy_match = _normalize(predicted.framework or "") == _normalize( + expected.framework + ) and _normalize(predicted.control_or_clause) == _normalize( + expected.control_or_clause + ) + evidence_match = _evidence_matches( + None, + predicted.citation_ids, + expected.evidence_locators, + citations, + ) + policy_correct += int(policy_match) + evidence_correct += int(evidence_match) + details.append( + { + "kind": "compliance_gap", + "expected_id": expected.id, + "predicted_id": predicted.id, + "match_score": score, + "policy_correct": policy_match, + "evidence_correct": evidence_match, + } + ) + + return GoldenRecord( + case_id=case.case_id, + phase=case.phase, + skill_id=case.skill_id, + repeat=repeat, + risk_expected=len(case.ground_truth.risk_items), + risk_predicted=len(report.risk_items), + risk_matched=len(risk_matches), + gap_expected=len(case.ground_truth.compliance_gaps), + gap_predicted=len(report.compliance_gaps), + gap_matched=len(gap_matches), + severity_total=len(risk_matches), + severity_correct=severity_correct, + policy_total=len(gap_matches), + policy_correct=policy_correct, + evidence_total=len(risk_matches) + len(gap_matches), + evidence_correct=evidence_correct, + matches=tuple(details), + ) + + +def score_records(records: list[GoldenRecord]) -> dict[str, Any]: + """Compute micro-averaged finding and fidelity metrics.""" + risk = _detection_metrics( + sum(record.risk_matched for record in records), + sum(record.risk_predicted for record in records), + sum(record.risk_expected for record in records), + ) + gaps = _detection_metrics( + sum(record.gap_matched for record in records), + sum(record.gap_predicted for record in records), + sum(record.gap_expected for record in records), + ) + overall = _detection_metrics( + risk["true_positive"] + gaps["true_positive"], + risk["predicted"] + gaps["predicted"], + risk["expected"] + gaps["expected"], + ) + severity_total = sum(record.severity_total for record in records) + policy_total = sum(record.policy_total for record in records) + evidence_total = sum(record.evidence_total for record in records) + return { + "risk_detection": risk, + "compliance_gap_detection": gaps, + "overall_detection": overall, + "severity_accuracy_on_matched": _divide( + sum(record.severity_correct for record in records), severity_total + ), + "policy_mapping_accuracy_on_matched": _divide( + sum(record.policy_correct for record in records), policy_total + ), + "evidence_locator_accuracy_on_matched": _divide( + sum(record.evidence_correct for record in records), evidence_total + ), + "schema_validity": 1.0 if records else 0.0, + } + + +def serialize_record(record: GoldenRecord) -> dict[str, Any]: + return { + "case_id": record.case_id, + "phase": record.phase, + "skill_id": record.skill_id, + "repeat": record.repeat, + "risk": { + "expected": record.risk_expected, + "predicted": record.risk_predicted, + "matched": record.risk_matched, + }, + "compliance_gaps": { + "expected": record.gap_expected, + "predicted": record.gap_predicted, + "matched": record.gap_matched, + }, + "matches": list(record.matches), + } + + +def _match_items( + predicted: list[PredictedItem], + expected: list[ExpectedItem], + *, + predicted_id: Callable[[PredictedItem], str], + predicted_text: Callable[[PredictedItem], str], + expected_id: Callable[[ExpectedItem], str], + expected_terms: Callable[[ExpectedItem], list[str]], +) -> list[tuple[int, int, float]]: + candidates: list[tuple[float, int, int]] = [] + for predicted_index, predicted_item in enumerate(predicted): + for expected_index, expected_item in enumerate(expected): + score = _match_score( + predicted_id(predicted_item), + predicted_text(predicted_item), + expected_id(expected_item), + expected_terms(expected_item), + ) + if score >= 0.6: + candidates.append((score, predicted_index, expected_index)) + candidates.sort(key=lambda item: (-item[0], item[1], item[2])) + used_predicted: set[int] = set() + used_expected: set[int] = set() + matches: list[tuple[int, int, float]] = [] + for score, predicted_index, expected_index in candidates: + if predicted_index in used_predicted or expected_index in used_expected: + continue + used_predicted.add(predicted_index) + used_expected.add(expected_index) + matches.append((predicted_index, expected_index, round(score, 6))) + return matches + + +def _match_score( + predicted_id: str, + predicted_text: str, + expected_id: str, + expected_terms: list[str], +) -> float: + if expected_id and _normalize(predicted_id) == _normalize(expected_id): + return 1.0 + text = _normalize(predicted_text) + terms = [_normalize(term) for term in expected_terms if _normalize(term)] + if not terms: + return 0.0 + return sum(term in text for term in terms) / len(terms) + + +def _evidence_matches( + source_ref: str | None, + citation_ids: list[str], + expected_locators: list[str], + citations: dict[str, Any], +) -> bool: + observed = {source_ref or "", *citation_ids} + for citation_id in citation_ids: + citation = citations.get(citation_id) + if citation is None: + continue + observed.update( + { + citation.id, + citation.locator or "", + citation.paragraph_id or "", + } + ) + normalized_observed = {_normalize(value) for value in observed if value} + return bool( + normalized_observed + & {_normalize(locator) for locator in expected_locators if locator} + ) + + +def _detection_metrics(matched: int, predicted: int, expected: int) -> dict[str, Any]: + precision = _divide(matched, predicted) + recall = _divide(matched, expected) + return { + "expected": expected, + "predicted": predicted, + "true_positive": matched, + "false_positive": predicted - matched, + "false_negative": expected - matched, + "precision": precision, + "recall": recall, + "f1": _divide(2 * precision * recall, precision + recall), + } + + +def _normalize(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "", str(value).lower()) + + +def _divide(numerator: float, denominator: float) -> float: + return numerator / denominator if denominator else 0.0 diff --git a/evals/tests/test_set_detection_scorer.py b/evals/tests/test_set_detection_scorer.py new file mode 100644 index 0000000..bc96f3b --- /dev/null +++ b/evals/tests/test_set_detection_scorer.py @@ -0,0 +1,122 @@ +from app.models.assessment import ( + AssessmentReport, + ComplianceGap, + RiskItem, + SourceCitation, +) +from evals.models import ( + ComplianceGapTruth, + EvalCase, + EvalGroundTruth, + EvalInput, + RiskTruth, +) +from evals.scoring.scorers.set_detection import record_from_report, score_records + + +def _case() -> EvalCase: + return EvalCase( + case_id="case-1", + dataset_id="ssdlc_synthetic_v1", + phase="design", + skill_id="ssdlc-design", + inputs=[EvalInput(path="design.md", type="markdown")], + ground_truth=EvalGroundTruth( + risk_items=[ + RiskTruth( + id="R1", + title="Unsigned identity", + severity="high", + description="The client controls identity.", + match_terms=["client", "identity"], + evidence_locators=["DES-01"], + ) + ], + compliance_gaps=[ + ComplianceGapTruth( + id="G1", + framework="generic-ssdlc", + control_or_clause="GEN-IAM-01", + gap_description="Authentication is missing.", + match_terms=["authentication", "missing"], + evidence_locators=["DES-01"], + ) + ], + ), + ) + + +def test_scorer_reports_detection_and_fidelity_failures_separately(): + report = AssessmentReport( + task_id="task", + phase="design", + status="completed", + summary="test", + risk_items=[ + RiskItem( + id="R1", + title="Unsigned identity", + severity="medium", + description="The client controls identity.", + citation_ids=["C1"], + ), + RiskItem( + id="EXTRA", + title="Unsupported extra risk", + severity="low", + ), + ], + compliance_gaps=[ + ComplianceGap( + id="G1", + framework="other-framework", + control_or_clause="OTHER-1", + gap_description="Authentication is missing.", + citation_ids=["C1"], + ) + ], + sources=[ + SourceCitation( + id="C1", + file="design.md", + excerpt="unrelated evidence", + locator="DES-99", + source_kind="current_document", + ) + ], + ) + + metrics = score_records([record_from_report(_case(), report, 0)]) + + assert metrics["risk_detection"]["precision"] == 0.5 + assert metrics["risk_detection"]["recall"] == 1.0 + assert metrics["compliance_gap_detection"]["f1"] == 1.0 + assert metrics["severity_accuracy_on_matched"] == 0.0 + assert metrics["policy_mapping_accuracy_on_matched"] == 0.0 + assert metrics["evidence_locator_accuracy_on_matched"] == 0.0 + assert metrics["schema_validity"] == 1.0 + + +def test_term_matching_is_one_to_one_when_prediction_ids_differ(): + report = AssessmentReport( + task_id="task", + phase="design", + status="completed", + summary="test", + risk_items=[ + RiskItem( + id="generated-risk", + title="Client identity is trusted", + severity="high", + description="The client supplies identity without authentication.", + source_ref="DES-01", + ) + ], + compliance_gaps=[], + ) + + record = record_from_report(_case(), report, 0) + + assert record.risk_matched == 1 + assert record.risk_predicted == 1 + assert record.risk_expected == 1 diff --git a/evals/tests/test_ssdlc_synthetic_adapter.py b/evals/tests/test_ssdlc_synthetic_adapter.py new file mode 100644 index 0000000..2350a2f --- /dev/null +++ b/evals/tests/test_ssdlc_synthetic_adapter.py @@ -0,0 +1,61 @@ +import json +import re +import shutil +from pathlib import Path + +import pytest + +from evals.adapters.ssdlc_synthetic import to_cases + +DATASET = Path(__file__).parents[1] / "datasets" / "ssdlc_synthetic_v1" + + +def test_adapter_loads_exactly_one_case_for_each_ssdlc_phase(): + cases = list(to_cases(DATASET)) + + assert [case.phase for case in cases] == [ + "requirements", + "design", + "development", + "testing", + "deployment", + "operations", + ] + assert all(len(case.ground_truth.risk_items) == 2 for case in cases) + assert all(len(case.ground_truth.compliance_gaps) == 2 for case in cases) + assert all(case.meta["review_status"] == "not_expert_reviewed" for case in cases) + + +def test_adapter_rejects_modified_ground_truth(tmp_path): + copied = tmp_path / "dataset" + shutil.copytree(DATASET, copied) + with (copied / "cases.jsonl").open("a", encoding="utf-8") as handle: + handle.write("\n") + + with pytest.raises(ValueError, match="Checksum mismatch for cases.jsonl"): + list(to_cases(copied)) + + +def test_manifest_declares_only_synthetic_non_expert_reviewed_data(): + manifest = json.loads((DATASET / "manifest.json").read_text(encoding="utf-8")) + + assert manifest["provenance"] == "fully_synthetic" + assert manifest["review_status"] == "not_expert_reviewed" + assert manifest["contains_real_personal_data"] is False + assert manifest["contains_third_party_content"] is False + + +def test_committed_synthetic_inputs_have_no_obvious_secret_or_contact_markers(): + text = "\n".join( + path.read_text(encoding="utf-8") + for path in sorted((DATASET / "inputs").glob("*.md")) + ) + + assert "-----BEGIN" not in text + assert not re.search(r"\bAKIA[0-9A-Z]{16}\b", text) + assert not re.search(r"(?i)\b(?:api[_-]?key|password)\s*[:=]\s*['\"][^'\"]+", text) + assert not re.search( + r"\b[A-Z0-9._%+-]+@(?![^\s]*\.test\b)[A-Z0-9.-]+\.[A-Z]{2,}\b", + text, + re.I, + ) diff --git a/evals/tests/test_synthetic_runner.py b/evals/tests/test_synthetic_runner.py new file mode 100644 index 0000000..45c9336 --- /dev/null +++ b/evals/tests/test_synthetic_runner.py @@ -0,0 +1,120 @@ +import json +from pathlib import Path +from uuid import UUID + +import pytest + +from app.models.assessment import ( + AssessmentReport, + ComplianceGap, + RiskItem, + SourceCitation, +) +from app.models.parser import ParsedDocument +from evals.adapters.ssdlc_synthetic import to_cases +from evals.models import RunConfig +from evals.runner.run_eval import run_cases + +DATASET = Path(__file__).parents[1] / "datasets" / "ssdlc_synthetic_v1" +BASELINE = ( + Path(__file__).parents[1] + / "reports" + / "baselines" + / "ssdlc-synthetic-oracle-v1.json" +) +CASES = list(to_cases(DATASET)) +CASES_BY_FILENAME = {Path(case.inputs[0].path).name: case for case in CASES} + + +async def _oracle_runner( + task_id: UUID, + parsed_documents: list[ParsedDocument], + **_: object, +) -> AssessmentReport: + filename = parsed_documents[0].metadata.filename + case = CASES_BY_FILENAME[filename] + sources: list[SourceCitation] = [] + risks: list[RiskItem] = [] + gaps: list[ComplianceGap] = [] + for truth in case.ground_truth.risk_items: + citation_id = f"source-{truth.id}" + sources.append( + SourceCitation( + id=citation_id, + file=filename, + excerpt=f"Synthetic evidence for {truth.id}", + locator=truth.evidence_locators[0], + source_kind="current_document", + ) + ) + risks.append( + RiskItem( + id=str(truth.id), + title=str(truth.title), + severity=truth.severity, + description=truth.description, + citation_ids=[citation_id], + ) + ) + for truth in case.ground_truth.compliance_gaps: + citation_id = f"source-{truth.id}" + sources.append( + SourceCitation( + id=citation_id, + file=filename, + excerpt=f"Synthetic evidence for {truth.id}", + locator=truth.evidence_locators[0], + source_kind="current_document", + ) + ) + gaps.append( + ComplianceGap( + id=str(truth.id), + framework=truth.framework, + control_or_clause=truth.control_or_clause, + gap_description=truth.gap_description, + citation_ids=[citation_id], + ) + ) + return AssessmentReport( + task_id=str(task_id), + phase=case.phase, + status="completed", + summary="deterministic oracle report; not model performance", + risk_items=risks, + compliance_gaps=gaps, + sources=sources, + ) + + +@pytest.mark.asyncio +async def test_six_phase_runner_reproduces_unapproved_oracle_baseline(tmp_path): + cfg = RunConfig( + run_id="six-phase-oracle", + dataset_id="ssdlc_synthetic_v1", + repeats=1, + provider="fixture", + model_id="deterministic-oracle-not-a-model", + phase="full_ssdlc", + skill_id="ssdlc-testing", + ) + + scorecard = await run_cases( + CASES, + cfg, + input_root=DATASET, + report_root=tmp_path, + assessment_runner=_oracle_runner, + ) + baseline = json.loads(BASELINE.read_text(encoding="utf-8")) + + assert baseline["approved"] is False + assert baseline["baseline_kind"] == "deterministic_oracle_scorer_contract" + assert scorecard["metrics"] == baseline["metrics"] + assert len(scorecard["by_phase_skill"]) == 6 + assert scorecard["dataset"]["review_status"] == "not_expert_reviewed" + markdown = (tmp_path / "six-phase-oracle" / "scorecard.md").read_text( + encoding="utf-8" + ) + assert "has not been expert reviewed" in markdown + assert "not model performance" not in markdown.lower()