diff --git a/.github/secret_scanning.yml b/.github/secret_scanning.yml index 79a6431..3d38582 100644 --- a/.github/secret_scanning.yml +++ b/.github/secret_scanning.yml @@ -2,5 +2,4 @@ # Keep forge secret scanning on real leaks. paths-ignore: - "tests/**" - - "src/traust_engine/data/secure-code-audit/gitleaks-rules/**" - "src/traust_engine/data/secure-code-audit/opengrep-rules/**" diff --git a/.ratchet.json b/.ratchet.json index 41396e1..110de62 100644 --- a/.ratchet.json +++ b/.ratchet.json @@ -2,49 +2,18 @@ "src/traust_engine/_ops_base.py": { "contracts": 1 }, - "src/traust_engine/_util/safe_exec.py": { - "contracts": 1 - }, "src/traust_engine/adapters/_contract_bridge.py": { "contracts": 2 }, "src/traust_engine/adapters/checkov.py": { "contracts": 1 }, - "src/traust_engine/adapters/gitleaks.py": { - "contracts": 2 - }, - "src/traust_engine/adapters/govulncheck.py": { - "contracts": 1 - }, - "src/traust_engine/adapters/grype.py": { - "contracts": 1 - }, - "src/traust_engine/adapters/joern.py": { - "contracts": 1 - }, "src/traust_engine/adapters/opengrep.py": { "contracts": 2 }, "src/traust_engine/adapters/ops.py": { "contracts": 1 }, - "src/traust_engine/adapters/osv.py": { - "contracts": 1 - }, - "src/traust_engine/adapters/yara.py": { - "contracts": 1 - }, - "src/traust_engine/compliance/dashboard.py": { - "contracts": 2, - "database": 1 - }, - "src/traust_engine/compliance/ops.py": { - "contracts": 1 - }, - "src/traust_engine/compliance/scope.py": { - "contracts": 1 - }, "src/traust_engine/corpus/resolver.py": { "contracts": 2, "state-file": 2 @@ -77,35 +46,12 @@ "src/traust_engine/portfolio/graph.py": { "database": 1 }, - "src/traust_engine/registry/feeds.py": { - "contracts": 1 - }, "src/traust_engine/registry/models.py": { "contracts": 1 }, "src/traust_engine/registry/ops.py": { "contracts": 1 }, - "src/traust_engine/registry/products.py": { - "contracts": 1 - }, - "src/traust_engine/reporting/lint.py": { - "contracts": 1 - }, - "src/traust_engine/reporting/render.py": { - "contracts": 1 - }, - "src/traust_engine/reporting/sarif.py": { - "contracts": 1, - "state-file": 4 - }, - "src/traust_engine/reporting/threat_rating.py": { - "contracts": 1 - }, - "src/traust_engine/reporting/validate.py": { - "contracts": 3, - "state-file": 3 - }, "src/traust_engine/sweep/engine.py": { "contracts": 1 }, @@ -114,8 +60,5 @@ }, "src/traust_engine/sweep/rule_lane.py": { "contracts": 1 - }, - "src/traust_engine/toolchain/ops.py": { - "contracts": 1 } } diff --git a/CODEOWNERS b/CODEOWNERS index 70830f1..282113b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -10,11 +10,7 @@ # Disposition ledger and corpus resolution /src/traust_engine/corpus/ @traust-security/traust-admins -# Finding identity (traust-ledger) and report integrity validation -/src/traust_engine/_util/finding_identity.py @traust-security/traust-admins -/src/traust_engine/_util/safe_exec.py @traust-security/traust-admins +# Metrics journal /src/traust_engine/metrics/history.py @traust-security/traust-admins -/src/traust_engine/reporting/validate.py @traust-security/traust-admins -/tests/test_validate_report.py @traust-security/traust-admins * @traust-security/traust-admins diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 55c00e0..5efef1d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,7 +65,7 @@ Direct downstream: `traust`. ```bash make test # unit tests (no external binaries needed) -uv run pytest -q # full suite (integration needs gitleaks/opengrep) +uv run pytest -q # full suite (integration needs opengrep) ``` ## Architecture diff --git a/README.md b/README.md index 99ee713..4dffb5a 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,28 @@ # traust-engine -Processing library for Traust: scanner adapters, report tooling and deterministic -workflows. Finding identity, disposition and persistence belong to traust-core, -not the engine or app CLI. Import as `traust_engine`. +The continuous-ops service for Traust, on traust-core: `traust_engine.ops` runs a +pack's assets through traust-core services. Finding identity, disposition, +persistence, tools and reporting belong to traust-core. Import as `traust_engine`. ## Architecture | Module | Role | |---|---| -| `traust_engine.validation` | Scope enforcement and soundness gates | -| `traust_engine.sweep.calibration` | Rule-pack calibration library API | -| `traust_engine.adapters` | Scanner adapters (gitleaks, opengrep, …) | -| `traust_engine.reporting` | Report validation, SARIF, render | | `traust_engine.ops` | Continuous ops: `Pack` (the pack's `assets.yaml`, checked against contracts `config/v1/pack`), the tick, admission and the reaper (`EngineLoop.run_once`), all through traust-core services | -Finding identity, disposition and persisted finding reads belong to -`traust_core.v1.services.findings`; the old engine cumulative replay is retired. -Scanner adapters keep tool identity in `metadata.tool`, not finding `origin`. -Native scanner fingerprints remain native scanner data, never invented Traust -identity stamps. Supplied finding classifier/dependency metadata survives the -typed bridge, and unknown severity or undeclared top-level fields are refused. -Final artifact admission goes through core's `AdapterResultArtifact` and -`ArtifactPublisher`; sparse scanner claims are not padded into audit findings. - -Native scanner results use the adapter-specific observation contract. -Unreported CWEs/description/remediation stay unreported; shared supplied-value -constraints remain enforced. Authored report findings keep their strict schema. -`tests/test_native_adapter_artifact.py` runs the real binary directly through -`AdapterResultArtifact.from_document` without hand-completing fields. -The real bundled token, DSN and encrypted-Vault publication regression also -verifies that storing observations does not create canonical findings/verdicts. - -Rule packs ship under `data/secure-code-audit/` (`opengrep-rules/`, -`gitleaks-rules/`). This library ships **no configuration files and never loads +Kept for traust until Shaun's retire-or-port decisions land (each package is +tagged `PENDING(decision: …)`); deleted when traust retires or ports it: + +| Module | Kept for | +|---|---| +| `traust_engine.engine` | `HarnessEngine` facade: `.metrics`, `.models`, `.adapters`, `.sweep`, `.portfolio` | +| `traust_engine.sweep` | sweep stages: collect, draft, sweep, emit, benchmark, rule lane, mining, calibration | +| `traust_engine.portfolio` | portfolio graph layers | +| `traust_engine.metrics`, `traust_engine.registry` | metrics journal and spend pricing | +| `traust_engine.adapters` | opengrep (sweep stages) and checkov (portfolio graph) | +| `traust_engine.corpus` | the report population the sweep stages read | + +The opengrep rule pack ships under `data/secure-code-audit/opengrep-rules/`. This library ships **no configuration files and never loads config itself — it is a pure consumer.** A caller builds the context once (`traust_contracts.load_context()`, done for you by `HarnessEngine.load(config_home=…)`) and the engine receives it *injected*: the @@ -107,13 +97,10 @@ holds every config file; edit them there (the AWS/SSH model). | `locations.yaml` → `progress_tracker` | Metrics journal + dashboard output root | | `locations.yaml` → `feeds_cache` | Feed cache directory | | `locations.yaml` → `portfolio_graph` | Portfolio graph DB (defaults under `analysis_results`) | -| `locations.yaml` → `gitleaks_config` | Gitleaks TOML (overrides the bundled default) | | `locations.yaml` → `opengrep_rules` | Opengrep rule pack dir (overrides the bundled default) | -| `locations.yaml` → `product_definitions` | Product/ownership registry endpoint (optional) | -| `locations.yaml` → `sarif_tool_uri` | SARIF `informationUri` (optional; omitted when unset) | Secrets and OS/tool standards stay environment-driven (`GITHUB_TOKEN`, signing key -material, `XDG_*`, `JAVA_OPTS`, `SAFE_EXEC_DISABLED`). +material, `XDG_*`, `JAVA_OPTS`). No config ships in the wheel. Templates for every operational file live in the harness repo as `config/*.example.*`; `install_traust` copies them into @@ -135,7 +122,7 @@ Manual equivalents: uv run ruff check . uv run ruff format --check . uv run pytest -m "not integration" -q # unit tests (no external binaries) -uv run pytest -q # full suite; integration skips when gitleaks/opengrep absent +uv run pytest -q # full suite; integration skips when opengrep is absent uv build ``` diff --git a/pyproject.toml b/pyproject.toml index 4376f3b..ace4049 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,12 +6,10 @@ build-backend = "hatchling.build" name = "traust-engine" version = "0.39.3" license = "Apache-2.0" -description = "Processing core for the AI security harness: scanners, data management, analysis, format handlers" +description = "Continuous-ops service for the AI security harness, on traust-core" requires-python = ">=3.11" dependencies = [ "traust-core", - "jsonschema>=4.0", - "referencing", "pyyaml", ] @@ -50,9 +48,8 @@ exclude = ["/.ci-venv", "/.venv"] [tool.pytest.ini_options] testpaths = ["tests"] markers = [ - "integration: tests requiring external binaries (gitleaks, opengrep) on PATH", + "integration: tests requiring external binaries (opengrep) on PATH", "requires_git: test creates git repos in tmp dirs (skipped when git init is blocked)", - "unbound_safe_exec: skip autouse profile bind; assert fallback or explicit profile_map", ] [tool.ruff] @@ -66,6 +63,4 @@ ignore = ["RUF001", "RUF002", "RUF003"] [tool.ruff.lint.per-file-ignores] "tests/*" = ["S101", "E741", "E402", "E501", "RUF012"] -"src/traust_engine/reporting/lint.py" = ["E501"] "src/traust_engine/metrics/history.py" = ["E501"] -"src/traust_engine/reporting/validate.py" = ["E402"] diff --git a/src/traust_engine/__init__.py b/src/traust_engine/__init__.py index 6be6516..5c8b76e 100644 --- a/src/traust_engine/__init__.py +++ b/src/traust_engine/__init__.py @@ -1,18 +1,17 @@ -"""Harness Engine — processing core for the AI security harness. +"""Harness Engine — the continuous-ops service on traust-core. Submodules: ops — continuous ops: pack spec, controller (tick), admission, reaper - corpus — population management (discovery, identity, dedup) - metrics — hash-chained metrics recording + spend tracking - portfolio — source-code dependency graph analysis + +Kept for traust until its PENDING decisions (sweep stages, portfolio graph +layers, metrics journal) retire or port them: + engine — the ``HarnessEngine`` facade (metrics, models, adapters, sweep, portfolio) sweep — class-generalization scan engine - impact — language-aware impact analysis - adapters — external scanner wrappers (opengrep, osv, gitleaks, ...) - compliance — compliance scope + boundary resolution - reporting — output format handlers (validate, render, SARIF) - registry — model + product resolution - escaping — public text-safety helpers (HTML, JSON-in-script, md cells, CSV) - _util — leaf utilities (paths, safe_exec, elf, redact) + portfolio — source-code dependency graph analysis + metrics — hash-chained metrics recording + spend tracking + registry — model resolution and spend pricing + adapters — opengrep (sweep stages) and checkov (portfolio graph) + corpus — population resolver the sweep stages read """ from traust_engine.assets import harness_version as _harness_version diff --git a/src/traust_engine/_ops_base.py b/src/traust_engine/_ops_base.py index c82271d..e41a83e 100644 --- a/src/traust_engine/_ops_base.py +++ b/src/traust_engine/_ops_base.py @@ -3,6 +3,9 @@ Holds the injected context and the fail-loud location resolvers every namespace needs. The resolution logic itself lives in :mod:`traust_engine.locations`; this just binds it to ``self._ctx.locations`` so no namespace re-derives it. + +PENDING(decision: sweep stages | portfolio graph layers | metrics journal) — kept for traust; +deleted when traust retires or ports it. """ from __future__ import annotations @@ -17,15 +20,6 @@ class ContextOps: def __init__(self, ctx: HarnessContext) -> None: self._ctx = ctx - self._safe_exec_profile_map: dict | None = None - - def safe_exec_profile_map(self) -> dict: - """Resolved safe_exec profiles for this context (one per cached *Ops instance).""" - if self._safe_exec_profile_map is None: - from traust_engine._util import safe_exec - - self._safe_exec_profile_map = safe_exec.profiles_from_section(self._ctx.safe_exec) - return self._safe_exec_profile_map def _optional_config_path(self, filename: str, loaded: object | None) -> Path | None: """Path under config_home when an optional section was loaded at entry.""" diff --git a/src/traust_engine/_util/__init__.py b/src/traust_engine/_util/__init__.py deleted file mode 100644 index 1b6c532..0000000 --- a/src/traust_engine/_util/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Leaf utilities — no domain logic, no upward imports.""" diff --git a/src/traust_engine/_util/elf.py b/src/traust_engine/_util/elf.py deleted file mode 100644 index 532d275..0000000 --- a/src/traust_engine/_util/elf.py +++ /dev/null @@ -1,271 +0,0 @@ -"""General-purpose ELF binary analyzer — no crypto domain knowledge.""" - -from __future__ import annotations - -import mmap -import re -import struct -from dataclasses import dataclass -from pathlib import Path - -_PT_LOAD = 1 -_PT_DYNAMIC = 2 -_DT_NULL = 0 -_DT_NEEDED = 1 -_DT_STRTAB = 5 -_DT_STRSZ = 10 - -_MACHINE_NAMES: dict[int, str] = { - 3: "i386", - 62: "x86_64", - 40: "arm", - 183: "aarch64", - 243: "riscv64", - 8: "mips", - 10: "mips64", -} - - -@dataclass -class ElfMetadata: - arch: str - elf_class: int - endian: str - - -@dataclass -class StringMatch: - pattern_id: str - matched: str - offset: int - - -@dataclass -class LinkedLibrary: - name: str - - -class ElfAnalyzer: - """Parse ELF headers and dynamic sections without external tools.""" - - def __init__(self, path: str | Path) -> None: - self.path = Path(path) - self._data: bytes | None = None - - def _load(self) -> bytes: - if self._data is None: - self._data = self.path.read_bytes() - return self._data - - def _parse_header(self, data: bytes) -> tuple[int, str, int, int] | None: - if len(data) < 64 or data[:4] != b"\x7fELF": - return None - ei_class = data[4] - if data[5] == 1: - endian = "<" - elif data[5] == 2: - endian = ">" - else: - return None - if ei_class == 2 or ei_class == 1: - e_machine = struct.unpack_from(endian + "H", data, 18)[0] - else: - return None - return ei_class, endian, e_machine, ei_class - - def metadata(self) -> ElfMetadata | None: - data = self._load() - parsed = self._parse_header(data) - if parsed is None: - return None - ei_class, endian, e_machine, _ = parsed - arch = _MACHINE_NAMES.get(e_machine, f"unknown({e_machine})") - return ElfMetadata( - arch=arch, - elf_class=32 if ei_class == 1 else 64, - endian="little" if endian == "<" else "big", - ) - - def _load_segments(self, data: bytes) -> tuple[list[tuple[int, int, int]], int, int]: - """Return (PT_LOAD segments, dynamic_offset, dynamic_size).""" - parsed = self._parse_header(data) - if parsed is None: - return [], 0, 0 - ei_class, endian, _, _ = parsed - load_segments: list[tuple[int, int, int]] = [] - dynamic_offset = 0 - dynamic_size = 0 - - try: - if ei_class == 2: - e_phoff = struct.unpack_from(endian + "Q", data, 32)[0] - e_phentsize = struct.unpack_from(endian + "H", data, 54)[0] - e_phnum = struct.unpack_from(endian + "H", data, 56)[0] - for i in range(e_phnum): - off = e_phoff + i * e_phentsize - if off + 56 > len(data): - break - p_type, _flags, p_offset, p_vaddr, _paddr, p_filesz, _memsz, _align = ( - struct.unpack_from(endian + "IIQQQQQQ", data, off) - ) - if p_type == _PT_LOAD: - load_segments.append((p_offset, p_vaddr, p_filesz)) - elif p_type == _PT_DYNAMIC: - dynamic_offset = p_offset - dynamic_size = p_filesz - elif ei_class == 1: - e_phoff = struct.unpack_from(endian + "I", data, 28)[0] - e_phentsize = struct.unpack_from(endian + "H", data, 42)[0] - e_phnum = struct.unpack_from(endian + "H", data, 44)[0] - for i in range(e_phnum): - off = e_phoff + i * e_phentsize - if off + 32 > len(data): - break - p_type, p_offset, p_vaddr, _paddr, p_filesz, _memsz, _flags, _align = ( - struct.unpack_from(endian + "IIIIIIII", data, off) - ) - if p_type == _PT_LOAD: - load_segments.append((p_offset, p_vaddr, p_filesz)) - elif p_type == _PT_DYNAMIC: - dynamic_offset = p_offset - dynamic_size = p_filesz - except struct.error: - return [], 0, 0 - return load_segments, dynamic_offset, dynamic_size - - @staticmethod - def _vaddr_to_offset( - vaddr: int, - segments: list[tuple[int, int, int]], - ) -> int | None: - for p_offset, seg_vaddr, p_filesz in segments: - if seg_vaddr <= vaddr < seg_vaddr + p_filesz: - return p_offset + (vaddr - seg_vaddr) - return None - - def linked_libraries(self) -> list[LinkedLibrary]: - data = self._load() - parsed = self._parse_header(data) - if parsed is None: - return [] - ei_class, endian, _, _ = parsed - load_segments, dynamic_offset, dynamic_size = self._load_segments(data) - - if not dynamic_offset or dynamic_offset + dynamic_size > len(data): - return [] - - strtab_vaddr = 0 - strtab_size = 0 - needed_indices: list[int] = [] - entry_size = 16 if ei_class == 2 else 8 - - pos = dynamic_offset - end = dynamic_offset + dynamic_size - while pos + entry_size <= end: - try: - if ei_class == 2: - d_tag, d_val = struct.unpack_from(endian + "QQ", data, pos) - else: - d_tag, d_val = struct.unpack_from(endian + "II", data, pos) - except struct.error: - break - pos += entry_size - if d_tag == _DT_NULL: - break - if d_tag == _DT_STRTAB: - strtab_vaddr = d_val - elif d_tag == _DT_STRSZ: - strtab_size = d_val - elif d_tag == _DT_NEEDED: - needed_indices.append(d_val) - - if not strtab_vaddr or not needed_indices: - return [] - - strtab_off = self._vaddr_to_offset(strtab_vaddr, load_segments) - if strtab_off is None: - return [] - - limit = strtab_size or (len(data) - strtab_off) - libs: list[LinkedLibrary] = [] - for idx in needed_indices: - start = strtab_off + idx - if start >= len(data): - continue - end_idx = data.find(b"\x00", start, min(start + 256, strtab_off + limit)) - if end_idx == -1: - continue - name = data[start:end_idx].decode("ascii", errors="replace") - if name: - libs.append(LinkedLibrary(name=name)) - return libs - - @classmethod - def linked_libraries_from_bytes(cls, data: bytes) -> list[LinkedLibrary]: - """Parse DT_NEEDED from raw ELF bytes (no file required).""" - analyzer = cls.__new__(cls) - analyzer.path = Path() - analyzer._data = data - return analyzer.linked_libraries() - - def scan_strings( - self, - patterns: list[tuple[str, re.Pattern[bytes]]], - *, - max_scan: int = 32_000_000, - ) -> list[StringMatch]: - matches: list[StringMatch] = [] - with self.path.open("rb") as fh: - try: - with mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ) as mm: - scan_data = mm[: min(len(mm), max_scan)] - for pattern_id, regex in patterns: - for m in regex.finditer(scan_data): - matched = m.group(0).decode("ascii", errors="replace") - matches.append( - StringMatch( - pattern_id=pattern_id, - matched=matched, - offset=m.start(), - ) - ) - except (ValueError, OSError): - data = self._load()[:max_scan] - for pattern_id, regex in patterns: - for m in regex.finditer(data): - matched = m.group(0).decode("ascii", errors="replace") - matches.append( - StringMatch( - pattern_id=pattern_id, - matched=matched, - offset=m.start(), - ) - ) - return matches - - @classmethod - def scan_bytes( - cls, - data: bytes, - patterns: list[tuple[str, re.Pattern[bytes]]], - *, - max_scan: int = 32_000_000, - ) -> list[StringMatch]: - """Scan raw bytes without a file on disk.""" - matches: list[StringMatch] = [] - scan_data = data[: min(len(data), max_scan)] - for pattern_id, regex in patterns: - for m in regex.finditer(scan_data): - matched = m.group(0).decode("ascii", errors="replace") - matches.append( - StringMatch( - pattern_id=pattern_id, - matched=matched, - offset=m.start(), - ) - ) - return matches - - @classmethod - def is_elf(cls, data: bytes) -> bool: - return len(data) >= 4 and data[:4] == b"\x7fELF" diff --git a/src/traust_engine/_util/redact.py b/src/traust_engine/_util/redact.py deleted file mode 100644 index aa92f4b..0000000 --- a/src/traust_engine/_util/redact.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Structural write-time redaction. - -Single-source secret patterns for every harness report writer. -Promoted from generate-team-report's Step-7 packaging redaction (our own -in-house patterns) and extended with the classes we lacked: Luhn-gated -payment card numbers, US SSNs, Azure/GCP key material, and URL-embedded -credentials. Convention: keep the first 5 characters, replace the rest -with ``...REDACTED`` so evidence stays traceable without being usable. - -Two entry points: - redact_text(text) -> (redacted_text, hits) — write-time scrubbing - scan_text(text) -> hits — detection only, used - by validate_report --strict to REJECT reports - that carry unredacted secrets - -`hits` is a list of {"category", "match_prefix"} — never the full -secret. -""" - -from __future__ import annotations - -import re - -_KEEP = 5 -_MARK = "...REDACTED" - -# (category, compiled regex) — value-capturing group 1 optional; when -# present only group 1 is replaced (assignment-style patterns) -PATTERNS: list[tuple[str, re.Pattern, bool]] = [ - ("aws-access-key", re.compile(r"AKIA[0-9A-Z]{12,}"), False), - ("github-token", re.compile(r"gh[posru]_[a-zA-Z0-9]{30,}"), False), - ("github-pat", re.compile(r"github_pat_[a-zA-Z0-9_]{20,}"), False), - ("gitlab-token", re.compile(r"glpat-[a-zA-Z0-9\-_]{20,}"), False), - ("slack-token", re.compile(r"xox[bpsa]-[a-zA-Z0-9\-]{10,}"), False), - ("llm-api-key", re.compile(r"sk-[a-zA-Z0-9]{20,}"), False), - ("google-api-key", re.compile(r"AIza[0-9A-Za-z_\-]{35}"), False), - ("bearer-token", re.compile(r"Bearer\s+([a-zA-Z0-9._\-]{20,})"), True), - ( - "jwt", - re.compile( - r"eyJ[a-zA-Z0-9_\-]{10,}\.[a-zA-Z0-9_\-]{10,}\." - r"[a-zA-Z0-9_\-]{10,}" - ), - False, - ), - ( - "password-assignment", - re.compile(r"(?:password|passwd)\s*[:=]\s*\\?['\"]([^'\"\\]{8,})\\?['\"]", re.IGNORECASE), - True, - ), - ( - "secret-assignment", - re.compile(r"secret\s*[:=]\s*\\?['\"]([^'\"\\]{8,})\\?['\"]", re.IGNORECASE), - True, - ), - ( - "token-assignment", - re.compile(r"token\s*[:=]\s*\\?['\"]([^'\"\\]{8,})\\?['\"]", re.IGNORECASE), - True, - ), - ( - "apikey-assignment", - re.compile(r"api[_-]?key\s*[:=]\s*\\?['\"]([^'\"\\]{8,})\\?['\"]", re.IGNORECASE), - True, - ), - # --- classes added by plan P2 (we lacked these) ------------------- - ("url-credentials", re.compile(r"://[A-Za-z0-9._%+-]+:([^@/\s'\"]{6,})@"), True), - ("us-ssn", re.compile(r"\b(?!000|666|9\d\d)\d{3}-(?!00)\d{2}-(?!0000)\d{4}\b"), False), - ("azure-client-secret", re.compile(r"\b[a-zA-Z0-9~_.\-]{3}8Q~[a-zA-Z0-9~_.\-]{30,}\b"), False), - ("gcp-sa-key", re.compile(r'"private_key_id"\s*:\s*"([a-f0-9]{40})"'), True), -] - -# classes where a match is near-certainly a real credential (structured -# vendor token formats). Heuristic classes (assignments, SSN, PAN, -# url-credentials) can legitimately appear in quoted audit evidence — -# calibrated 2026-07-24: 400-report corpus dry run had 0 high-confidence -# hits and 14 heuristic hits, all evidence quotes. -HIGH_CONFIDENCE = frozenset( - { - "aws-access-key", - "github-token", - "github-pat", - "gitlab-token", - "slack-token", - "google-api-key", - "jwt", - "pem-private-key", - "azure-client-secret", - "gcp-sa-key", - } -) - -_PEM = re.compile( - r"(-----BEGIN [A-Z ]*PRIVATE KEY-----)(.*?)(-----END [A-Z ]*PRIVATE " - r"KEY-----)", - re.DOTALL, -) - -# candidate PANs: 13-19 digits, optionally space/dash separated -_PAN_CANDIDATE = re.compile(r"\b(?:\d[ -]?){13,19}\b") - - -def _luhn_ok(digits: str) -> bool: - total, alt = 0, False - for d in reversed(digits): - n = int(d) - if alt: - n *= 2 - if n > 9: - n -= 9 - total += n - alt = not alt - return total % 10 == 0 - - -def _redact_value(value: str) -> str: - return value[:_KEEP] + _MARK - - -def _apply(text: str, collect: list, replace: bool) -> str: - for category, rx, group1 in PATTERNS: - - def _sub(m, _c=category, _g=group1): - whole = m.group(0) - val = m.group(1) if _g and m.lastindex else whole - if _MARK in whole: - return whole # already redacted - collect.append({"category": _c, "match_prefix": val[:_KEEP]}) - if not replace: - return whole - red = _redact_value(val) - return whole.replace(val, red) if _g and m.lastindex else red - - text = rx.sub(_sub, text) - - def _pem(m): - collect.append({"category": "pem-private-key", "match_prefix": "-----"}) - if not replace: - return m.group(0) - return f"{m.group(1)}\n[PRIVATE KEY MATERIAL REDACTED]\n{m.group(3)}" - - text = _PEM.sub(_pem, text) - - def _pan(m): - digits = re.sub(r"[ -]", "", m.group(0)) - if not (13 <= len(digits) <= 19 and _luhn_ok(digits)): - return m.group(0) # not a card number — leave untouched - collect.append({"category": "payment-card", "match_prefix": digits[:_KEEP]}) - if not replace: - return m.group(0) - return _redact_value(digits) - - text = _PAN_CANDIDATE.sub(_pan, text) - return text - - -def redact_text(text: str) -> tuple[str, list[dict]]: - """Redact in place; returns (clean_text, hits).""" - hits: list[dict] = [] - return _apply(text, hits, replace=True), hits - - -def scan_text(text: str) -> list[dict]: - """Detect only — powers validate_report --strict rejection.""" - hits: list[dict] = [] - _apply(text, hits, replace=False) - return hits diff --git a/src/traust_engine/_util/safe_exec.py b/src/traust_engine/_util/safe_exec.py deleted file mode 100644 index 2b37034..0000000 --- a/src/traust_engine/_util/safe_exec.py +++ /dev/null @@ -1,807 +0,0 @@ -"""safe_exec — validate and run target-derived commands without a shell. - -Two jobs: - - 1. VALIDATE an argv or a PoC-derived command string against a named - profile (config/safe-exec-profiles.yaml — policy as data). - 2. RUN the validated command — single argv or an allowlisted pipeline — - via subprocess chaining with a scrubbed environment. No shell is - ever involved, so shell metacharacters have no interpreter. - -Deny layers (validation order): - raw-pattern denies (string form) → substitution, expansion, /dev/tcp - shell operators (string form) → only `|` permitted, as a pipeline - hard-deny binaries → never grantable by any profile - shells → never grantable (pipelines are - executed natively, no shell needed) - profile allowlist → basename (or explicit ./path head) - git hardening (when git is allowed)→ network subcommands denied; - dangerous -c/--config-env keys denied - protected env assignments → PATH/LD_PRELOAD/GIT_SSH_COMMAND/… - recursion cap → safe_exec cannot re-enter itself - -Modes: the library tells the truth unconditionally; enforcement policy -belongs to the caller. The CLI honors SAFE_EXEC_MODE=warn|enforce -(default enforce) — warn reports the verdict and exits 0 so batch lanes -can calibrate. SAFE_EXEC_DISABLED= bypasses `run` but shouts to -stderr and appends to a per-user 0700 bypass log — never silent, and -library callers (validate-findings adapters) do not honor it. - -Usage: - safe_exec.py check --profile validation-step -- oc get pods -A - safe_exec.py check --profile validation-step --string 'oc get po | grep x' - safe_exec.py run --profile go-fuzz --timeout 900 -- make build - safe_exec.py list-profiles -""" - -from __future__ import annotations - -import dataclasses -import logging -import os -import re -import shlex -import subprocess -import sys -from pathlib import Path - -from traust_contracts import SafeExecProfiles - -_log = logging.getLogger(__name__) -RECURSION_ENV = "SAFE_EXEC_DEPTH" -MAX_RECURSION = 3 -BYPASS_LOG_DIR = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local" / "state")) -BYPASS_LOG = BYPASS_LOG_DIR / "traust-engine" / "safe-exec-bypass.log" - -# --------------------------------------------------------------------------- -# Deny tables. HARD entries are never grantable: a profile that lists one -# is a configuration error, refused at load. -# --------------------------------------------------------------------------- - -HARD_DENY_BINARIES = frozenset( - { - # privilege / system mutation - "sudo", - "su", - "doas", - "chown", - "chgrp", - "dd", - "mkfs", - "mount", - "umount", - "systemctl", - "service", - "crontab", - "launchctl", - "reboot", - "shutdown", - "insmod", - "rmmod", - "sysctl", - # process/host tampering - "kill", - "pkill", - "killall", - "renice", - "nohup", - # raw network tools (profiles needing egress use scoped tools, not these) - "nc", - "ncat", - "netcat", - "socat", - "telnet", - "ssh", - "scp", - "sftp", - "rsync", - "ftp", - "wget", - # package managers (runtime installs are S7 territory, never build steps) - "pip", - "pip3", - "dnf", - "yum", - "apt", - "apt-get", - "brew", - "gem", - "cargo-install", - } -) - -SHELLS = frozenset( - { - "sh", - "bash", - "zsh", - "ksh", - "dash", - "fish", - "csh", - "tcsh", - } -) - -# Interpreters are deny-by-default but PROFILE-grantable (python-test -# legitimately runs python3/pytest). Shells are not. -INTERPRETERS = frozenset( - { - "python", - "python2", - "python3", - "perl", - "ruby", - "node", - "deno", - "lua", - "php", - "awk", - "gawk", - "mawk", - "xargs", - "env", - "eval", - "exec", - "command", - "time", - "timeout", - "nice", - "setsid", - "script", - } -) - -GIT_NETWORK_SUBCOMMANDS = frozenset( - { - "push", - "pull", - "fetch", - "clone", - "remote", - "submodule", - "ls-remote", - } -) - -# Dangerous `git -c key=value` / `--config-env` keys: each is a code-exec -# or credential-theft primitive. Matched case-insensitively; entries -# ending in "." are prefixes. -GIT_DANGEROUS_C_KEYS = ( - "core.pager", - "core.editor", - "core.sshcommand", - "core.hookspath", - "core.fsmonitor", - "core.askpass", - "core.alternateobjectdirectories", - "credential.", - "filter.", - "diff.external", - "difftool.", - "mergetool.", - "merge.", - "remote.", - "http.proxy", - "http.sslcainfo", - "http.sslcert", - "http.sslkey", - "protocol.", - "sendemail.", - "alias.", - "gpg.program", - "ssh.variant", - "uploadpack.", - "sshcommand", -) - -PROTECTED_ENV_VARS = frozenset( - { - "PATH", - "LD_PRELOAD", - "LD_LIBRARY_PATH", - "LD_AUDIT", - "DYLD_INSERT_LIBRARIES", - "DYLD_LIBRARY_PATH", - "PYTHONPATH", - "PYTHONSTARTUP", - "PERL5LIB", - "NODE_OPTIONS", - "RUBYOPT", - "BASH_ENV", - "ENV", - "IFS", - "SHELL", - "GIT_SSH", - "GIT_SSH_COMMAND", - "GIT_PROXY_COMMAND", - "GIT_EXTERNAL_DIFF", - "GIT_PAGER", - "GIT_ASKPASS", - "GIT_CONFIG_GLOBAL", - "GIT_CONFIG_SYSTEM", - "GIT_INDEX_FILE", - "GIT_OBJECT_DIRECTORY", - "JAVA_TOOL_OPTIONS", - "MAVEN_OPTS", - "GRADLE_OPTS", - "_JAVA_OPTIONS", - "SAFE_EXEC_MODE", - "SAFE_EXEC_DISABLED", - RECURSION_ENV, - } -) - -# Raw denies applied to STRING-form commands before tokenization. -RAW_DENY_PATTERNS = ( - (re.compile(r"`"), "backtick command substitution"), - (re.compile(r"\$\("), "command substitution $( )"), - (re.compile(r"\$\{"), "parameter expansion ${ }"), - (re.compile(r"[<>]\("), "process substitution"), - (re.compile(r"\$'"), "ANSI-C quoting"), - (re.compile(r"/dev/(tcp|udp)/"), "raw socket via /dev/tcp|udp"), - # NB: no newline deny — nothing here ever reaches a shell, so a - # quoted newline is printf data and an unquoted one is whitespace; - # either way every resulting token passes the vetting below. -) - -# Environment kept when run() scrubs (plus profile keep_env). -BASE_KEEP_ENV = ( - "PATH", - "HOME", - "LANG", - "LC_ALL", - "TMPDIR", - "TERM", - "http_proxy", - "https_proxy", - "no_proxy", - "HTTP_PROXY", - "HTTPS_PROXY", - "NO_PROXY", -) - -_ENV_PREFIX_RX = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)=(.*)$", re.S) - - -@dataclasses.dataclass(frozen=True) -class Profile: - name: str - description: str - allow: frozenset - allowed_path_heads: frozenset - allow_pipelines: bool - keep_env: tuple - - def permits(self, head: str) -> bool: - if "/" in head: - return head in self.allowed_path_heads - return head in self.allow - - -@dataclasses.dataclass(frozen=True) -class Verdict: - ok: bool - reason: str = "" - # tokenized pipeline segments (only for string-form validation) - segments: tuple = () - - -# Embedded fallback so library consumers (validate-findings adapters) -# keep functioning if the config file is absent in a stripped checkout. -# The YAML file is authoritative; keep this entry in sync with it -# (tests/test_safe_exec.py::test_fallback_matches_config enforces). -_FALLBACK_PROFILES = { - "validation-step": Profile( - name="validation-step", - description="embedded fallback — see config/safe-exec-profiles.yaml", - allow=frozenset( - { - "curl", - "oc", - "kubectl", - "jq", - "grep", - "base64", - "head", - "tail", - "tr", - "wc", - "cat", - "sleep", - "echo", - "printf", - } - ), - allowed_path_heads=frozenset(), - allow_pipelines=True, - # VF_OAUTH_TOKEN kept for probe soundness (F15) — lab-scoped, - # short-TTL; see config/safe-exec-profiles.yaml - keep_env=("KUBECONFIG", "VF_OAUTH_TOKEN"), - ), -} - - -def profiles_from_section(section: SafeExecProfiles) -> dict: - out = {} - for name, spec in (section.profiles or {}).items(): - allow = frozenset(spec.get("allow") or []) - bad = sorted(allow & (HARD_DENY_BINARIES | SHELLS)) - if bad: - raise ValueError( - f"profile {name!r} grants hard-denied binaries {bad} — " - "hard denies are never grantable; fix " - "safe-exec-profiles.yaml" - ) - out[name] = Profile( - name=name, - description=str(spec.get("description", "")), - allow=allow, - allowed_path_heads=frozenset(spec.get("allowed_path_heads") or []), - allow_pipelines=bool(spec.get("allow_pipelines", False)), - keep_env=tuple(spec.get("keep_env") or ()), - ) - return out - - -_profiles_from_section = profiles_from_section # tests / legacy alias - - -_PROFILE_CACHE: dict | None = None - - -def bind_profiles(section: SafeExecProfiles | None) -> None: - """Seed the module-global cache (tests and standalone CLI only). - - Engine callers should use ``ContextOps.safe_exec_profile_map()`` — it is - scoped to one ``HarnessEngine`` instance and does not clobber other contexts - in the same process. - """ - global _PROFILE_CACHE - if section is None: - _PROFILE_CACHE = dict(_FALLBACK_PROFILES) - return - _PROFILE_CACHE = profiles_from_section(section) - - -def reset_profiles() -> None: - """Clear the module-global cache (tests only).""" - global _PROFILE_CACHE - _PROFILE_CACHE = None - - -def profiles(*, profile_map: dict | None = None) -> dict: - if profile_map is not None: - return profile_map - global _PROFILE_CACHE - if _PROFILE_CACHE is None: - _log.warning( - "safe_exec profiles not bound — falling back to built-in profiles. " - "This is a security-policy fallback: the operator's allowlists are " - "NOT in effect. Pass profile_map= or bind via HarnessEngine." - ) - _PROFILE_CACHE = dict(_FALLBACK_PROFILES) - return _PROFILE_CACHE - - -def get_profile(name: str, *, profile_map: dict | None = None) -> Profile: - p = profiles(profile_map=profile_map).get(name) - if p is None: - raise KeyError( - f"unknown safe_exec profile {name!r}; " - "known: {sorted(profiles(profile_map=profile_map))} " - "(see safe-exec-profiles.yaml in $TRAUST_CONFIG_HOME)" - ) - return p - - -# --------------------------------------------------------------------------- -# validation -# --------------------------------------------------------------------------- - - -def _check_git_argv(argv: list) -> str: - """Extra hardening when the head binary is git. Returns deny reason - or ''.""" - i = 1 - subcommand = None - while i < len(argv): - tok = argv[i] - if tok in ("-c", "--config-env"): - if i + 1 < len(argv): - key = argv[i + 1].split("=", 1)[0].strip().lower() - for danger in GIT_DANGEROUS_C_KEYS: - if key == danger or (danger.endswith(".") and key.startswith(danger)): - return f"git config key {key!r} is a code-exec/credential primitive" - i += 2 - continue - return "dangling git -c" - if tok == "-C": - i += 2 - continue - if tok.startswith("-"): - i += 1 - continue - subcommand = tok - break - if subcommand in GIT_NETWORK_SUBCOMMANDS: - return ( - f"git network subcommand {subcommand!r} is denied under " - "safe_exec (clones/fetches belong to S3-gated skill " - "steps, not target-derived commands)" - ) - return "" - - -# curl hardening (assessment 2026-07-31 live-F5): the validation-step -# profile legitimately probes lab endpoints, but upload/file-read/config -# forms turn a "safe" probe into arbitrary local-file exfiltration. -# Host allowlisting is the P1 follow-up; these flags are never needed. -CURL_DENY_FLAGS = frozenset( - { - "-F", - "--form", - "--form-string", - "-T", - "--upload-file", - "--config", - "-K", - "--netrc", - "-n", - "--netrc-file", - "--output-dir", - "--create-dirs", - } -) - - -def _check_curl_argv(argv: list) -> str: - for tok in argv[1:]: - flag = tok.split("=", 1)[0] - if flag in CURL_DENY_FLAGS: - return ( - f"curl flag {flag!r} is denied under safe_exec " - "(local-file read/upload/config — exfiltration " - "primitives; assessment 2026-07-31 F5)" - ) - if tok.startswith(("-d@", "--data@")): - return "curl -d@ is denied under safe_exec" - if flag in ("-d", "--data", "--data-binary", "--data-raw", "--data-urlencode"): - idx = argv.index(tok) - if "=" in tok: - val = tok.split("=", 1)[1] - else: - val = argv[idx + 1] if idx + 1 < len(argv) else "" - if val.startswith("@"): - return "curl --data @ is denied under safe_exec" - if tok.startswith("file://"): - return "curl file:// scheme is denied under safe_exec" - return "" - - -# kube hardening (assessment 2026-07-31 live-F1/F2 defense-in-depth): -# cluster/credential override flags never belong in target-derived -# commands — the adapter supplies context and kubeconfig. The execute -# layer denies these too; this is the second fence. -KUBE_DENY_FLAGS = frozenset( - { - "--kubeconfig", - "--context", - "--cluster", - "--user", - "--server", - "-s", - "--token", - "--as", - "--as-group", - "--as-uid", - "--insecure-skip-tls-verify", - "--tls-server-name", - "--certificate-authority", - "--client-certificate", - "--client-key", - "--username", - "--password", - } -) - - -def _check_kube_argv(argv: list) -> str: - for tok in argv[1:]: - flag = tok.split("=", 1)[0] - if flag in KUBE_DENY_FLAGS: - return ( - f"kubectl/oc flag {flag!r} is denied under safe_exec " - "— target-derived commands run only against the " - "engagement-pinned context (assessment 2026-07-31 F2)" - ) - return "" - - -def validate_argv(argv, profile: Profile) -> Verdict: - """Validate one command (no pipeline) as an argv list.""" - if not argv: - return Verdict(False, "empty command") - if int(os.environ.get(RECURSION_ENV, "0")) >= MAX_RECURSION: - return Verdict(False, f"safe_exec recursion depth cap ({MAX_RECURSION}) reached") - head = None - for tok in argv: - m = _ENV_PREFIX_RX.match(tok) - if m and head is None: - var = m.group(1) - if var.upper() in PROTECTED_ENV_VARS or var.upper().startswith("GIT_"): - return Verdict(False, f"assignment to protected env var {var!r}") - continue - head = tok - break - if head is None: - return Verdict(False, "command has no executable head") - base = head if "/" in head else Path(head).name - name = Path(base).name - if name in HARD_DENY_BINARIES: - return Verdict(False, f"{name!r} is hard-denied (never grantable)") - if name in SHELLS: - return Verdict( - False, f"shell {name!r} is never grantable — pipelines run natively under safe_exec" - ) - if name in ("safe_exec", "safe_exec.py"): - return Verdict(False, "safe_exec may not re-invoke itself") - if not profile.permits(head if "/" in head else name): - if name in INTERPRETERS: - return Verdict(False, f"interpreter {name!r} not granted by profile {profile.name!r}") - return Verdict( - False, - f"binary {head!r} not in profile {profile.name!r} allowlist {sorted(profile.allow)}", - ) - if name == "git": - reason = _check_git_argv([head, *list(argv[argv.index(head) + 1 :])]) - if reason: - return Verdict(False, reason) - if name == "curl": - reason = _check_curl_argv(list(argv[argv.index(head) :])) - if reason: - return Verdict(False, reason) - if name in ("kubectl", "oc"): - reason = _check_kube_argv(list(argv[argv.index(head) :])) - if reason: - return Verdict(False, reason) - return Verdict(True) - - -def vet_command_string(cmd: str, profile: Profile) -> Verdict: - """Vet a PoC-derived command STRING. On success, `segments` holds the - tokenized pipeline (one tuple per `|` segment).""" - for rx, why in RAW_DENY_PATTERNS: - if rx.search(cmd or ""): - return Verdict(False, f"{why} not allowed in step text") - lex = shlex.shlex(cmd, posix=True, punctuation_chars=True) - lex.whitespace_split = True - try: - tokens = list(lex) - except ValueError as e: - return Verdict(False, f"unparseable step command: {e}") - if not tokens: - return Verdict(False, "empty step command") - for t in tokens: - if t and all(c in ";&<>|()" for c in t) and t != "|": - return Verdict(False, f"shell operator {t!r} not allowed in step text") - segments: list = [[]] - for t in tokens: - if t == "|": - segments.append([]) - else: - segments[-1].append(t) - if len(segments) > 1 and not profile.allow_pipelines: - return Verdict(False, f"profile {profile.name!r} does not allow pipelines") - for seg in segments: - v = validate_argv(seg, profile) - if not v.ok: - return v - return Verdict(True, segments=tuple(tuple(s) for s in segments)) - - -# --------------------------------------------------------------------------- -# execution -# --------------------------------------------------------------------------- - - -def _scrubbed_env(profile: Profile, extra_env: dict | None = None) -> dict: - env = {k: os.environ[k] for k in (*BASE_KEEP_ENV, *profile.keep_env) if k in os.environ} - env[RECURSION_ENV] = str(int(os.environ.get(RECURSION_ENV, "0")) + 1) - if extra_env: - for k, v in extra_env.items(): - if k.upper() in PROTECTED_ENV_VARS: - raise ValueError(f"extra_env may not set protected {k!r}") - env[k] = v - return env - - -def run_segments( - segments, - profile: Profile, - *, - timeout: int = 120, - cwd=None, - input_: str | None = None, - extra_env: dict | None = None, -): - """Execute validated pipeline segments via subprocess chaining — - no shell. Returns (rc, stdout, stderr); rc/stderr come from the - final segment, non-zero upstream rcs are appended to stderr.""" - env = _scrubbed_env(profile, extra_env) - if len(segments) == 1: - try: - proc = subprocess.run( - list(segments[0]), - capture_output=True, - text=True, - timeout=timeout, - input=input_, - cwd=cwd, - env=env, - ) - except subprocess.TimeoutExpired as e: - out = ( - e.stdout - if isinstance(e.stdout, str) - else (e.stdout or b"").decode(errors="replace") - ) - err = ( - e.stderr - if isinstance(e.stderr, str) - else (e.stderr or b"").decode(errors="replace") - ) - return 124, out, err + f"\n[timeout after {timeout}s]" - except OSError as e: - return 127, "", f"[exec failed: {e}]" - return proc.returncode, proc.stdout, proc.stderr - procs = [] - prev_stdout = subprocess.PIPE if input_ is not None else None - try: - for _i, seg in enumerate(segments): - stdin = procs[-1].stdout if procs else prev_stdout - procs.append( - subprocess.Popen( - list(seg), - stdin=stdin, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - cwd=cwd, - env=env, - ) - ) - if procs[:-1]: - # let upstream see SIGPIPE naturally - procs[-2].stdout.close() - if input_ is not None: - procs[0].stdin.write(input_) - procs[0].stdin.close() - out, err = procs[-1].communicate(timeout=timeout) - except subprocess.TimeoutExpired: - for p in procs: - p.kill() - return 124, "", f"[pipeline timeout after {timeout}s]" - except OSError as e: - for p in procs: - p.kill() - return 127, "", f"[exec failed: {e}]" - rc = procs[-1].returncode - tail_err = err or "" - for i, p in enumerate(procs[:-1]): - p.wait(timeout=5) - if p.returncode not in (0, None): - tail_err += f"\n[pipeline segment {i} exited {p.returncode}]" - return rc, out, tail_err - - -def run( - cmd, - profile_name: str, - *, - timeout: int = 120, - cwd=None, - input_: str | None = None, - extra_env: dict | None = None, - honor_bypass: bool = False, - profile_map: dict | None = None, -): - """Validate then execute. `cmd` is an argv list or a command string. - Library callers default to NOT honoring SAFE_EXEC_DISABLED.""" - profile = get_profile(profile_name, profile_map=profile_map) - bypass = os.environ.get("SAFE_EXEC_DISABLED", "").strip() - if bypass and honor_bypass: - _log_bypass(cmd, profile_name, bypass) - segments = ( - [tuple(cmd)] - if not isinstance(cmd, str) - else vet_command_string(cmd, profile).segments or [tuple(shlex.split(cmd))] - ) - return run_segments( - segments, profile, timeout=timeout, cwd=cwd, input_=input_, extra_env=extra_env - ) - if isinstance(cmd, str): - v = vet_command_string(cmd, profile) - segments = v.segments - else: - v = validate_argv(list(cmd), profile) - segments = (tuple(cmd),) - if not v.ok: - return 126, "", f"[safe_exec blocked: {v.reason}]" - return run_segments( - segments, profile, timeout=timeout, cwd=cwd, input_=input_, extra_env=extra_env - ) - - -def _log_bypass(cmd, profile_name: str, reason: str) -> None: - line = f"SAFE_EXEC BYPASS profile={profile_name} reason={reason!r} cmd={cmd!r}" - print(f"*** {line}", file=sys.stderr) - try: - BYPASS_LOG.parent.mkdir(parents=True, exist_ok=True) - BYPASS_LOG.parent.chmod(0o700) - with BYPASS_LOG.open("a", encoding="utf-8") as fh: - import datetime - - fh.write(f"{datetime.datetime.now().astimezone().isoformat()} {line}\n") - except OSError: - pass # the stderr shout already happened - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - - -def check_command( - profile: str, - cmd: str | list[str], - *, - timeout: int = 120, - cwd: str | None = None, -) -> int: - profile_obj = get_profile(profile) - mode = os.environ.get("SAFE_EXEC_MODE", "enforce").lower() - if isinstance(cmd, str): - verdict = vet_command_string(cmd, profile_obj) - else: - verdict = validate_argv(cmd, profile_obj) - if verdict.ok: - print("OK") - return 0 - if mode == "warn": - print(f"WARN (would block): {verdict.reason}") - return 0 - print(f"BLOCK: {verdict.reason}") - return 1 - - -def execute_command( - profile: str, - cmd: str | list[str], - *, - timeout: int = 120, - cwd: str | None = None, -) -> int: - profile_obj = get_profile(profile) - mode = os.environ.get("SAFE_EXEC_MODE", "enforce").lower() - if isinstance(cmd, str): - verdict = vet_command_string(cmd, profile_obj) - else: - verdict = validate_argv(cmd, profile_obj) - if not verdict.ok and mode == "warn": - print(f"WARN (would block, running anyway — warn mode): {verdict.reason}", file=sys.stderr) - segs = verdict.segments or ( - [tuple(shlex.split(cmd))] if isinstance(cmd, str) else [tuple(cmd)] - ) - rc, out, err = run_segments(segs, profile_obj, timeout=timeout, cwd=cwd) - else: - rc, out, err = run(cmd, profile, timeout=timeout, cwd=cwd, honor_bypass=True) - sys.stdout.write(out or "") - sys.stderr.write(err or "") - return rc - - -def list_profiles_cmd() -> int: - for name, p in sorted(profiles().items()): - print(f"{name:18s} allow={sorted(p.allow)} pipelines={p.allow_pipelines} — {p.description}") - return 0 diff --git a/src/traust_engine/_util/script_loader.py b/src/traust_engine/_util/script_loader.py deleted file mode 100644 index e2c0e83..0000000 --- a/src/traust_engine/_util/script_loader.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Load modules that formerly lived under traust/scripts/.""" - -from __future__ import annotations - -import importlib -import importlib.util -import sys -from pathlib import Path -from types import ModuleType - -_MIGRATED: dict[str, str] = { - "corpus": "traust_engine.corpus.resolver", - "metrics_ledger": "traust_engine.metrics.history", - "check_drift": "traust.cli.check_drift", - "emit_doc_variance": "traust.cli.emit_doc_variance", - "check_skill_security": "traust.cli.check_skill_security", - "countersign": "traust.cli.countersign", -} - -_SKILL_LOCAL: dict[str, tuple[str, str]] = { - "compliance_assert": ("compliance-check", "scripts"), - "bootstrap_benchmark_targets": ("recall-benchmark", "scripts"), - "match_benchmark": ("recall-benchmark", "scripts"), - "validate_compliance_assessment": ("compliance-check", "scripts"), - "collect_cloud_inventory": ("compliance-check", "scripts"), - "collect_iac_inventory": ("compliance-check", "scripts"), - "extract_app_interface_env": ("compliance-check", "scripts"), - "run_compliance_check": ("compliance-check", "scripts"), - "enrich_findings_cves": ("secure-code-audit", "scripts"), -} - - -def load_script(name: str, harness_root: Path) -> ModuleType: - """Import a migrated engine/CLI module or a skill script.""" - if name in _MIGRATED: - mod = importlib.import_module(_MIGRATED[name]) - sys.modules[name] = mod - return mod - - if name in _SKILL_LOCAL: - skill, sub = _SKILL_LOCAL[name] - # A skill sits either at skills// or one level deeper under - # a workflow stage directory, skills/-// (legacy - # harnessing/ paths are still tried for older checkouts). - candidates: list[Path] = [] - for base in ("skills", "harnessing"): - flat = harness_root / base / skill / sub / f"{name}.py" - candidates.append(flat) - candidates.extend(sorted(harness_root.glob(f"{base}/*/{skill}/{sub}/{name}.py"))) - path = next((p for p in candidates if p.is_file()), candidates[0]) - if not path.is_file(): - raise FileNotFoundError(path) - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise ImportError(f"cannot load {name} from {path}") - mod = importlib.util.module_from_spec(spec) - sys.modules[name] = mod - spec.loader.exec_module(mod) - return mod - - matches: list[Path] = [] - for base in ("skills", "harnessing"): - matches.extend(sorted(harness_root.glob(f"{base}/**/scripts/{name}.py"))) - if matches: - path = matches[0] - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise ImportError(f"cannot load {name} from {path}") - mod = importlib.util.module_from_spec(spec) - sys.modules[name] = mod - spec.loader.exec_module(mod) - return mod - - raise FileNotFoundError(f"no module {name!r} in migrated map or skills/**/scripts/") diff --git a/src/traust_engine/adapters/__init__.py b/src/traust_engine/adapters/__init__.py index 6073c1f..376740c 100644 --- a/src/traust_engine/adapters/__init__.py +++ b/src/traust_engine/adapters/__init__.py @@ -1 +1,5 @@ -"""Scanner wrappers returning typed observations; core owns artifact admission and intake.""" +"""Scanner wrappers the sweep stages (opengrep) and the portfolio graph (checkov) run. + +PENDING(decision: sweep stages | portfolio graph layers) — kept for traust; +deleted when traust retires or ports it. +""" diff --git a/src/traust_engine/adapters/cloud_config.py b/src/traust_engine/adapters/cloud_config.py deleted file mode 100644 index f314620..0000000 --- a/src/traust_engine/adapters/cloud_config.py +++ /dev/null @@ -1,382 +0,0 @@ -"""Checkov as a traust-core tool provider: the offline scan behind a cloud-config audit. - -Every rule about Checkov's output lives in core (`models.cloud_config`); this module runs the -binaries and lays out the scan tree. Not to be confused with `adapters/checkov.py`, the -Kubernetes hardening scanner, which never ran Checkov. -""" - -from __future__ import annotations - -import json -import os -import re -import shutil -import subprocess -import tempfile -from collections.abc import Iterator, Mapping, Sequence -from pathlib import Path, PurePosixPath -from typing import Any - -from traust_core.v1.context import Context -from traust_core.v1.errors import ServiceError, TraustError, ValidationError -from traust_core.v1.models.cloud_config import ( - CHECKOV, - CHECKOV_VERSION, - CheckovScan, - CloudConfigFacts, - IacInventory, - require_checkov, -) -from traust_core.v1.providers.base import Acquisition, Readiness, Result -from traust_core.v1.providers.clock import Clock -from traust_core.v1.providers.process import ProcessRunner -from traust_core.v1.providers.tools import DatabaseStatus, NoDatabase, ToolRequest - -CHECKOV_TIMEOUT = 1800.0 -PROBE_TIMEOUT = 60.0 # version probes, git HEAD, and one `bicep build` -# The `bicep` CLI is installed by hand (docs: requirements); verify the download against the -# sha256 vetted for its platform. Intake row: docs/external-dependencies.md (MIT, Azure/bicep). -# bicep-osx-arm64: 59072cc82da704ab45d6bb11133da15b169795e42ef585dedc73d98e19fcf1be -BICEP_VERSION = "0.45.15" -# Offline always: no policy downloads, no platform key, gitleaks is the one secret scanner. -_OFFLINE = ("--output", "json", "--skip-download", "--compact") -# Core's IacInventory applies the same skip rule; pruning here only spares the walk. -_SKIP_DIRS = frozenset({"vendor", "node_modules", "third_party"}) -_YAML_READ_CAP = 1_000_000 # bytes; `kind:` sits near the top in practice -# The bicep fallback's yield: extra outputs, extra roots, aliases, notes, files recovered. -_Fallback = tuple[tuple[bytes, ...], tuple[str, ...], dict[str, str], dict[str, str], int] - - -class CheckovProvider: - """The pinned Checkov over one checkout, as `CloudConfigFacts`; read-only toward it.""" - - name = CHECKOV - - def __init__( - self, - checkov: ProcessRunner, - probe: ProcessRunner, - clock: Clock, - bicep: Sequence[Path] = (), - ) -> None: - self._checkov = checkov - self._probe = probe - self._clock = clock - self._bicep = tuple(bicep) - - def check(self) -> Readiness: - try: - require_checkov(self._version()) - except TraustError as e: - return Readiness(ready=False, detail=str(e)) - return Readiness(ready=True) - - def acquire(self, request: ToolRequest) -> Result[CloudConfigFacts]: - workspace = request.checkout() - if not workspace.is_dir(): - raise ValidationError(f"checkov: workspace is not a directory: {workspace}") - reported = self._version() - require_checkov(reported) # before the scan: another checkov never touches the checkout - facts = CloudConfigFacts.of(self._scan(workspace, reported)) - return Result( - facts, - Acquisition( - provider=self.name, - acquired_at=self._clock.now(), - version=facts.checkov_version, - source=str(workspace.resolve()), - ), - ) - - def version(self) -> str | None: - try: - found = re.search(r"\d+\.\d+\.\d+", self._version()) - except ServiceError: - return None # not installed: readiness says so - return found.group(0) if found else None - - def database(self) -> DatabaseStatus | None: - return None - - def update_database(self) -> DatabaseStatus: - raise NoDatabase(f"{self.name} keeps no vulnerability database to update") - - def _version(self) -> str: - try: - out = self._probe.run(["checkov", "--version"], check=False) - except ServiceError as e: - raise ServiceError( - _cause( - e, - missing=f"checkov not on PATH (pip install checkov=={CHECKOV_VERSION})", - timeout="checkov --version timed out", - ) - ) from e - return (out.stdout + out.stderr).decode(errors="replace") - - def _scan(self, workspace: Path, version: str) -> CheckovScan: - resolved = workspace.resolve() - files = list(_first_party(resolved)) - inventory = IacInventory.count(files, lambda p: _leading_text(resolved / p)) - observed: dict[str, Any] = { - "target": resolved.name, - "version": version, - "inventory": inventory, - "target_head": self._git_head(workspace), - } - with tempfile.TemporaryDirectory(prefix="cca-scan-") as tmp: - scan_dir, aliases, notes = _scan_tree(resolved, Path(tmp), _containerfiles(files)) - extra = (str(scan_dir),) if scan_dir != resolved else () - observed["roots"] = (str(workspace), str(resolved), *extra) - stdout, error = self._run_checkov(scan_dir, "--skip-framework", "secrets") - if error is None: - try: - parsed = json.loads(stdout) - except json.JSONDecodeError: - error = "unparseable checkov JSON output" - if error is not None: - return CheckovScan(**observed, error=error, aliases=aliases, notes=notes) - arm, roots, arm_aliases, note, recovered = self._bicep_fallback( - resolved, parsed, (*extra, str(resolved), str(workspace)) - ) - observed["roots"] += roots - return CheckovScan( - **observed, - outputs=(stdout, *arm), - aliases={**aliases, **arm_aliases}, - notes={**notes, **note}, - bicep_recovered=recovered, - ) - - def _run_checkov(self, directory: Path, *flags: str) -> tuple[bytes, str | None]: - """One offline checkov run over a directory: its stdout, or why there is none.""" - argv = ["checkov", "--directory", str(directory), *_OFFLINE, *flags] - try: - done = self._checkov.run(argv, check=False) - except ServiceError as e: - return b"", _cause( - e, missing="checkov not on PATH", timeout=f"timeout after {CHECKOV_TIMEOUT:.0f}s" - ) - if not done.stdout.strip(): - stderr = done.stderr.decode(errors="replace").strip() - return b"", ( - stderr.splitlines()[-1][:200] if stderr else f"exit {done.returncode}, no output" - ) - return done.stdout, None - - def _git_head(self, workspace: Path) -> str | None: - try: - done = self._probe.run(["git", "-C", str(workspace), "rev-parse", "HEAD"], check=False) - except ServiceError: - return None # not a git checkout, or no git: the facts say "unknown" - return (done.stdout.decode().strip() or None) if done.returncode == 0 else None - - def _find_bicep(self) -> Path | None: - """The first candidate that reports the pinned version; any other bicep counts as absent.""" - for cand in self._bicep: - if not cand.is_file(): - continue - try: - done = self._probe.run([str(cand), "--version"], check=False) - except TraustError: - continue - if BICEP_VERSION in done.stdout.decode(errors="replace"): - return cand - return None - - def _bicep_fallback(self, resolved: Path, parsed: Any, strip: tuple[str, ...]) -> _Fallback: - """Checkov 3.3.6's bicep parser rejects much real-world Bicep: transpile each rejected - file offline with the pinned CLI (`--no-restore`: registry modules are never fetched) - and re-scan the ARM through the arm framework, citing the source `.bicep`.""" - failures = _bicep_parse_failures(parsed, strip) - if not failures: - return (), (), {}, {}, 0 - bicep = self._find_bicep() - if bicep is None: - return _note( - f"{len(failures)} .bicep file(s) failed checkov's parser and the pinned bicep " - f"CLI (v{BICEP_VERSION}) is not installed — transpile fallback unavailable, " - "files stay NOT ASSESSED (docs/setup.md has the install)" - ) - with tempfile.TemporaryDirectory(prefix="cca-bicep-") as tmp: - ok, failed = [], [] - for rel in failures: - src = resolved / rel.lstrip("/") - dst = Path(tmp) / (rel.lstrip("/") + ".arm.json") - dst.parent.mkdir(parents=True, exist_ok=True) - built = src.is_file() and self._transpile(bicep, src, dst) - (ok if built else failed).append(rel) - if not ok: - return _note( - f"0/{len(failures)} parse-failed .bicep file(s) transpiled (bicep " - f"v{BICEP_VERSION} --no-restore; likely external registry modules — fetched " - "by design never); files stay NOT ASSESSED" - ) - arm, error = self._run_checkov(Path(tmp), "--framework", "arm") - if error is not None: - return _note( - f"{len(ok)}/{len(failures)} parse-failed .bicep file(s) transpiled but the " - f"arm re-scan failed: {error}; files stay NOT ASSESSED" - ) - note = ( - f"{len(ok)}/{len(failures)} parse-failed .bicep file(s) transpiled with the pinned " - f"bicep CLI v{BICEP_VERSION} (--no-restore, offline) and re-scanned via the arm " - "framework; fact paths cite the source .bicep but LINE RANGES refer to the " - "generated ARM template" - + (f"; still unparsed (stay NOT ASSESSED): {', '.join(failed)}" if failed else "") - ) - aliases = {"/" + rel.lstrip("/") + ".arm.json": rel for rel in ok} - return (arm,), (tmp,), aliases, {"bicep_transpile_fallback": note}, len(ok) - - def _transpile(self, bicep: Path, src: Path, dst: Path) -> bool: - argv = [str(bicep), "build", str(src), "--no-restore", "--outfile", str(dst)] - try: - done = self._probe.run(argv, check=False) - except ServiceError: - return False - return done.returncode == 0 and dst.is_file() - - -def register_checkov(ctx: Context) -> None: - """Register `CheckovProvider` in `ctx.tools`, on runners with Checkov's own time budgets.""" - - def build() -> CheckovProvider: - bicep = _bicep_candidates() - keep = ctx.config.process.keep_env - return CheckovProvider( - ProcessRunner(["checkov"], keep_env=keep, timeout_seconds=CHECKOV_TIMEOUT), - ProcessRunner( - ["checkov", "git", "bicep", *(p.name for p in bicep)], - keep_env=keep, - timeout_seconds=PROBE_TIMEOUT, - ), - ctx.clock, - bicep, - ) - - ctx.tools.register(CHECKOV, build) - - -def _bicep_candidates() -> list[Path]: - """Where the pinned bicep may be: `$CCA_BICEP`, PATH, then the per-user pinned install dir.""" - found = (os.environ.get("CCA_BICEP"), shutil.which("bicep")) - home = Path.home() / ".cache" / "cca-tools" / f"bicep-{BICEP_VERSION}" / "bicep" - return [*(Path(p) for p in found if p), home] - - -def _cause(e: ServiceError, *, missing: str, timeout: str) -> str: - if isinstance(e.__cause__, FileNotFoundError): - return missing - if isinstance(e.__cause__, subprocess.TimeoutExpired): - return timeout - return str(e) - - -def _note(text: str) -> _Fallback: - return (), (), {}, {"bicep_transpile_fallback": text}, 0 - - -def _first_party(root: Path) -> Iterator[PurePosixPath]: - """Checkout-relative files outside vendored, hidden (but `.tekton`) and symlinked dirs.""" - for top, dirs, names in os.walk(root, followlinks=False): - dirs[:] = sorted( - d - for d in dirs - if d not in _SKIP_DIRS - and (not d.startswith(".") or d == ".tekton") - and not Path(top, d).is_symlink() - ) - for name in sorted(names): - if (path := Path(top, name)).is_file(): - yield PurePosixPath(path.relative_to(root).as_posix()) - - -def _containerfiles(files: Sequence[PurePosixPath]) -> list[PurePosixPath]: - """Checkov 3.3.6's dockerfile framework only matches `Dockerfile` / `Dockerfile.*`, so it - silently skips the Red Hat `Containerfile` convention.""" - return sorted( - f - for f in files - if ".tekton" not in f.parts - and (f.name == "Containerfile" or f.name.startswith("Containerfile.")) - ) - - -def _leading_text(path: Path) -> str: - try: - with path.open(encoding="utf-8", errors="replace") as fh: - return fh.read(_YAML_READ_CAP) - except OSError: - return "" - - -def _scan_tree( - resolved: Path, tmp: Path, containerfiles: Sequence[PurePosixPath] -) -> tuple[Path, dict[str, str], dict[str, str]]: - """Where checkov scans: the checkout itself, or a temp tree when a 3.3.6 blind spot needs - one. Returns (scan dir, {alias path: real path}, {metadata key: workaround note}).""" - hidden = any(part.startswith(".") for part in resolved.parts[1:]) - if not (hidden or containerfiles): - return resolved, {}, {} - scan_dir = tmp / (resolved.name.lstrip(".") or "target") - aliases: dict[str, str] = {} - notes: dict[str, str] = {} - if containerfiles: - scan_dir.mkdir() - _mirror(resolved, scan_dir) - skipped = [] - for rel in containerfiles: - alias = scan_dir / rel.parent / ("Dockerfile" + rel.name[len("Containerfile") :]) - if alias.exists() or alias.is_symlink(): - skipped.append(rel.as_posix()) # a real Dockerfile twin is scanned instead - continue - alias.symlink_to(resolved / rel) - prefix = "" if str(rel.parent) == "." else "/" + rel.parent.as_posix() - aliases[f"{prefix}/{alias.name}"] = f"{prefix}/{rel.name}" - notes["containerfile_alias_workaround"] = ( - f"{len(aliases)} Containerfile(s) aliased to Dockerfile-named symlinks in temp scan " - f"tree {scan_dir} (checkov {CHECKOV_VERSION}'s dockerfile framework skips " - "Containerfile names); fact paths are mapped back to the real Containerfile paths" - + ( - f"; not aliased (Dockerfile-name twin already exists): {', '.join(skipped)}" - if skipped - else "" - ) - ) - else: - scan_dir.symlink_to(resolved) - if hidden: - # checkov 3.3.6's kubernetes framework evaluates zero checks, silently, when any - # component of the scan path is dot-prefixed. - notes["hidden_path_workaround"] = ( - f"hidden path component in {resolved} suppresses checkov's kubernetes framework; " - f"scanned via non-hidden temp path {scan_dir}" - ) - return scan_dir, aliases, notes - - -def _mirror(src: Path, dst: Path) -> None: - """Every real directory as a real directory (checkov's discovery never descends a directory - symlink), every file and symlinked dir as a symlink back to the checkout.""" - for entry in sorted(src.iterdir()): - if entry.is_dir() and not entry.is_symlink(): - (dst / entry.name).mkdir() - _mirror(entry, dst / entry.name) - else: - (dst / entry.name).symlink_to(entry) - - -def _bicep_parse_failures(parsed: Any, strip: tuple[str, ...]) -> list[str]: - """Checkout paths (leading /) of the .bicep files checkov failed to parse.""" - rels = set() - for block in parsed if isinstance(parsed, list) else [parsed]: - if not isinstance(block, Mapping): - continue - for path in (block.get("results") or {}).get("parsing_errors") or []: - path = str(path) - if not path.endswith(".bicep"): - continue - if root := next((r for r in strip if r and path.startswith(r)), None): - path = "/" + path[len(root) :].lstrip("/") - rels.add(path) - return sorted(rels) diff --git a/src/traust_engine/adapters/compliance_sources.py b/src/traust_engine/adapters/compliance_sources.py deleted file mode 100644 index d6d838f..0000000 --- a/src/traust_engine/adapters/compliance_sources.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Core's compliance sources: ADR register files at their pin, and official product docs. - -Every rule about what these files mean lives in core (`models.compliance`: `parse_adr`, -`is_decision_record`, `doc_text`); this module only reads them, over the same network paths the -legacy `index_adrs.py` (gh api tree + raw.githubusercontent) and `fetch_product_docs.py` (curl to -docs.redhat.com) used. -""" - -from __future__ import annotations - -import json -import re -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from urllib.parse import quote -from uuid import UUID - -from traust_core.v1.context import Context -from traust_core.v1.errors import NotFoundError, ServiceError, ValidationError -from traust_core.v1.models.compliance import OFFICIAL_DOCS, is_decision_record -from traust_core.v1.providers.base import Acquisition, Readiness, Result, on_path -from traust_core.v1.providers.clock import Clock -from traust_core.v1.providers.collaboration import ProductDocument, RegisterFile -from traust_core.v1.providers.http import HttpFetcher -from traust_core.v1.providers.process import ProcessRunner - -GH_TIMEOUT = 120.0 -CURL_TIMEOUT = 90.0 # curl's own --max-time is 60 s -_ADR_READ_CAP = 262_144 # bytes per register file, as legacy read them -_GITHUB_REPO = re.compile(r"github\.com[/:]([\w.-]+/[\w.-]+?)(?:\.git|/|$)") -_PIN = re.compile(r"[0-9a-f]{40}") -_SLUG = re.compile(r"[A-Za-z0-9._-]+") - - -class GithubAdrSource: - """ADR register files read at the register's pinned commit from GitHub; never a checkout.""" - - name = "github-adr" - - def __init__(self, gh: ProcessRunner, http: HttpFetcher, clock: Clock) -> None: - self._gh = gh - self._http = http - self._clock = clock - - def check(self) -> Readiness: - return on_path("gh", missing=" (the ADR register tree is read with gh api)") - - def files(self, repo: str, pin: str, paths: Sequence[str]) -> Result[list[RegisterFile]]: - slug = github_slug(repo) - if not _PIN.fullmatch(pin): - raise ValidationError(f"ADR register {repo}: pin {pin!r} is not a 40-hex commit") - found = [ - RegisterFile( - path=path, - text=self._raw(slug, pin, path), - url=f"https://github.com/{slug}/blob/{pin}/{path}", - ) - for path in self._tree(slug, pin) - if is_decision_record(path, paths) - ] - acquisition = Acquisition( - provider=self.name, acquired_at=self._clock.now(), version=pin, source=repo - ) - return Result(found, acquisition) - - def _tree(self, slug: str, pin: str) -> list[str]: - try: - out = self._gh.run(["gh", "api", f"repos/{slug}/git/trees/{pin}?recursive=1"]) - tree = json.loads(out.stdout) - except (ServiceError, ValueError) as e: - raise ServiceError(f"ADR register {slug}@{pin}: tree unreadable: {e}") from e - if tree.get("truncated"): - raise ServiceError(f"ADR register {slug}@{pin}: GitHub truncated the tree listing") - return sorted(e["path"] for e in tree["tree"] if e.get("type") == "blob") - - def _raw(self, slug: str, pin: str, path: str) -> str: - url = f"https://raw.githubusercontent.com/{slug}/{pin}/{quote(path)}" - try: - body = self._http.get(url) - except ServiceError as e: - raise ServiceError(f"ADR register {slug}@{pin}: {path} unreadable: {e}") from e - return body[:_ADR_READ_CAP].decode("utf-8", "replace") - - -def github_slug(repo: str) -> str: - """`org/name` of a GitHub repository URL; any other forge has no register reader yet.""" - if found := _GITHUB_REPO.search(repo): - return found.group(1) - raise ValidationError(f"ADR register {repo}: not a GitHub repository (no reader for its forge)") - - -@dataclass(frozen=True, slots=True) -class DocsTarget: - """One product's docs.redhat.com slug and version; no guides means every guide listed.""" - - product_slug: str - version: str - guides: tuple[str, ...] = () - - -class RedHatDocSource: - """docs.redhat.com guides as single-page html, fetched with stock curl (the CDN refuses - python-urllib and custom user agents); nothing is cached or stored.""" - - name = "docs.redhat.com" - - def __init__( - self, curl: ProcessRunner, clock: Clock, targets: Mapping[UUID, DocsTarget] - ) -> None: - self._curl = curl - self._clock = clock - self._targets = dict(targets) - - def check(self) -> Readiness: - return on_path("curl") - - def fetch(self, product_id: UUID) -> Result[list[ProductDocument]]: - target = self._targets.get(product_id) - if target is None: - raise NotFoundError(f"product {product_id}: no docs.redhat.com slug/version declared") - slug, version = _segment(target.product_slug), _segment(target.version) - base = f"{OFFICIAL_DOCS}en/documentation/{slug}/{version}" - pages = [ - ProductDocument( - product_slug=slug, version=version, guide=guide, url=url, html=self._get(url) - ) - for guide in map(_segment, target.guides or self._guides(base, slug, version)) - for url in (f"{base}/html-single/{guide}/index",) - ] - acquisition = Acquisition( - provider=self.name, acquired_at=self._clock.now(), version=version, source=base - ) - return Result(pages, acquisition) - - def _guides(self, landing: str, slug: str, version: str) -> list[str]: - pattern = rf'href="/en/documentation/{re.escape(slug)}/{re.escape(version)}/html/([^"/]+)"' - if guides := sorted(set(re.findall(pattern, self._get(landing)))): - return guides - raise NotFoundError( - f"no guides found at {landing}: wrong slug/version, or the page layout changed" - ) - - def _get(self, url: str) -> str: - argv = ["curl", "-sSL", "--fail", "--proto", "=https", "--max-time", "60", url] - try: - return self._curl.run(argv).stdout.decode("utf-8", "replace") - except ServiceError as e: - raise ServiceError(f"fetch failed for {url}: {e}") from e - - -def _segment(value: str) -> str: - if not _SLUG.fullmatch(value): - raise ValidationError(f"docs.redhat.com path segment {value!r} is not a plain slug") - return value - - -def register_compliance_sources(ctx: Context, docs: Mapping[UUID, DocsTarget]) -> None: - """Set the ADR and doc sources the compliance doors read through.""" - keep = ctx.config.process.keep_env - ctx.adr_source = GithubAdrSource( - ProcessRunner(["gh"], keep_env=keep, timeout_seconds=GH_TIMEOUT), ctx.http, ctx.clock - ) - ctx.doc_source = RedHatDocSource( - ProcessRunner(["curl"], keep_env=keep, timeout_seconds=CURL_TIMEOUT), ctx.clock, docs - ) diff --git a/src/traust_engine/adapters/crypto_audit.py b/src/traust_engine/adapters/crypto_audit.py deleted file mode 100644 index 94f2a0b..0000000 --- a/src/traust_engine/adapters/crypto_audit.py +++ /dev/null @@ -1,1040 +0,0 @@ -# DESIGN: This tool collects raw crypto-relevant data. It never interprets -# whether data is "good", "bad", "PQC-capable", or "compliant". Consumer -# skills (pqc-readiness, compliance-check, etc.) own all interpretation. -# If you're adding a pattern/threshold/classification here, it belongs in a skill. -"""crypto_audit.py — Unopinionated crypto data collector. - -Three tiers of structural data extraction: - - source — repository/build-time provider census (crypto_probe.py) - image — container image structural data (libs, packages, certs, policy) - cluster — runtime state via oc exec (TLS negotiation, crypto-policy) - -All subcommands emit crypto-audit/v1 JSON with a ``tier`` field. - -Usage: - crypto_audit.py source /path/to/repo [--component NAME] [--output facts.json] - crypto_audit.py image registry.example.com/app:tag [--no-pull] [-o facts.json] - crypto_audit.py cluster --context ctx [--namespaces ns1 ns2] [--discover] -""" - -from __future__ import annotations - -import json -import re -import shutil -import subprocess -import sys -import tempfile -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from traust_engine._util.elf import ElfAnalyzer -from traust_engine.adapters import crypto_probe - -SCHEMA = "crypto-audit/v1" - -_TLS_CONNECT_TIMEOUT = 3 - - -# --------------------------------------------------------------------------- -# Output helpers -# --------------------------------------------------------------------------- - - -def _utc_now() -> str: - return datetime.now(UTC).isoformat() - - -def _summary(facts: list[dict[str, Any]]) -> dict[str, Any]: - return { - "total_facts": len(facts), - "probe_ids": sorted({f["probe_id"] for f in facts}), - "providers": sorted({f["provider"] for f in facts}), - } - - -def build_output( - *, - tier: str, - target: str, - facts: list[dict[str, Any]], - errors: list[str], - component: str = "", - metadata: dict[str, Any] | None = None, -) -> dict[str, Any]: - out: dict[str, Any] = { - "schema": SCHEMA, - "tier": tier, - "target": target, - "timestamp": _utc_now(), - "metadata": metadata or {}, - "facts": facts, - "errors": errors, - "summary": _summary(facts), - } - if component: - out["component"] = component - return out - - -def _write_output(payload: dict[str, Any], output_path: str, quiet: bool) -> None: - text = json.dumps(payload, indent=2) - if output_path: - Path(output_path).write_text(text) - if not quiet: - print(f"[crypto-audit] written to {output_path}", file=sys.stderr) - else: - print(text) - - -# --------------------------------------------------------------------------- -# Container runtime (minimal) -# --------------------------------------------------------------------------- - - -class ContainerRuntime: - def __init__(self, runtime: str = "auto", platform: str = "") -> None: - if runtime == "auto" or not runtime: - self.runtime = self._detect() - else: - self.runtime = runtime - if not self.runtime: - raise RuntimeError("Neither podman nor docker found in PATH") - self.platform = platform - - @staticmethod - def _detect() -> str: - for rt in ("podman", "docker"): - if shutil.which(rt): - return rt - return "" - - def _cmd(self) -> list[str]: - return [self.runtime] - - def _plat(self) -> list[str]: - return ["--platform", self.platform] if self.platform else [] - - @staticmethod - def _run(cmd, *, timeout=120, **kw) -> subprocess.CompletedProcess: - return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, **kw) - - def pull(self, image: str, *, timeout=300) -> bool: - return ( - self._run([*self._cmd(), "pull", *self._plat(), image], timeout=timeout).returncode == 0 - ) - - def create(self, image: str) -> str: - r = self._run([*self._cmd(), "create", *self._plat(), image, "/bin/true"]) - if r.returncode != 0: - raise RuntimeError(f"create failed: {r.stderr.strip()}") - return r.stdout.strip()[:64] - - def cp_out(self, cid: str, src: str, dst: str) -> bool: - return self._run([*self._cmd(), "cp", f"{cid}:{src}", dst]).returncode == 0 - - def inspect(self, image: str): - r = self._run([*self._cmd(), "inspect", image, "--format", "json"]) - if r.returncode != 0: - return None - try: - data = json.loads(r.stdout) - return data[0] if isinstance(data, list) else data - except (json.JSONDecodeError, IndexError): - return None - - def run_cmd(self, image: str, cmd: list[str], *, entrypoint: str | None = None, timeout=30): - c = [*self._cmd(), "run", "--rm", *self._plat()] - if entrypoint: - c += ["--entrypoint", entrypoint] - c.append(image) - return self._run(c + cmd, timeout=timeout) - - def remove(self, cid: str) -> None: - self._run([*self._cmd(), "rm", "-f", cid], timeout=30) - - -# --------------------------------------------------------------------------- -# Source tier -# --------------------------------------------------------------------------- - - -def _source_fact_to_dict(fact: crypto_probe.CryptoFact) -> dict[str, Any]: - source = fact.file if fact.line <= 0 else f"{fact.file}:{fact.line}" - row: dict[str, Any] = { - "probe_id": fact.probe_id, - "detail": fact.detail, - "provider": fact.provider, - "source": source, - } - if fact.version: - row["version"] = fact.version - extra = dict(fact.extra) if fact.extra else {} - if fact.line > 0: - extra.update(file=fact.file, line=fact.line) - if extra: - row["extra"] = extra - return row - - -def run_source( - repo_path: Path, - *, - sbom_path: Path | None = None, - component: str = "", -) -> dict[str, Any]: - facts = crypto_probe.probe_repo(repo_path) - if sbom_path: - facts.extend(crypto_probe.probe_sbom(sbom_path)) - fact_dicts = [_source_fact_to_dict(f) for f in facts] - metadata: dict[str, Any] = {"repo_dir": str(repo_path)} - if sbom_path: - metadata["sbom"] = str(sbom_path) - return build_output( - tier="source", - target=str(repo_path), - facts=fact_dicts, - errors=[], - component=component, - metadata=metadata, - ) - - -# --------------------------------------------------------------------------- -# Image tier — structural data only -# --------------------------------------------------------------------------- - - -def _collect_image_metadata(rt: ContainerRuntime, image: str) -> tuple[list[dict], dict]: - """Collect env vars, labels, architecture from image inspect.""" - facts: list[dict[str, Any]] = [] - metadata: dict[str, Any] = {} - data = rt.inspect(image) - if not data: - return facts, metadata - - config = data.get("Config") or data.get("config") or {} - labels = config.get("Labels") or {} - env_list = config.get("Env") or [] - entrypoint = config.get("Entrypoint") or [] - - metadata["architecture"] = data.get("Architecture") or data.get("architecture") or "unknown" - metadata["os"] = data.get("Os") or data.get("os") or "unknown" - metadata["entrypoint"] = entrypoint - metadata["labels"] = labels - - for env_entry in env_list: - facts.append( - { - "probe_id": "IMG_ENV", - "detail": env_entry[:200], - "provider": "image-metadata", - "source": "image inspect env", - "extra": {"env": env_entry}, - } - ) - - for key, val in labels.items(): - facts.append( - { - "probe_id": "IMG_LABEL", - "detail": f"{key}={val}"[:200], - "provider": "image-metadata", - "source": "image inspect labels", - "extra": {"key": key, "value": val}, - } - ) - - return facts, metadata - - -def _collect_linked_libs(rt: ContainerRuntime, cid: str, tmpdir: str) -> list[dict]: - """Extract ELF binaries and report their linked libraries.""" - facts: list[dict[str, Any]] = [] - bin_dirs = ["usr/bin", "usr/sbin", "usr/local/bin", "bin", "sbin"] - dst_base = Path(tmpdir) / "bins" - dst_base.mkdir(parents=True, exist_ok=True) - - for d in bin_dirs: - dst = dst_base / d - dst.mkdir(parents=True, exist_ok=True) - rt.cp_out(cid, f"/{d}/.", str(dst)) - - for bin_path in sorted(dst_base.rglob("*")): - if not bin_path.is_file() or bin_path.stat().st_size < 10_000: - continue - try: - data = bin_path.read_bytes() - except OSError: - continue - if not ElfAnalyzer.is_elf(data): - continue - libs = ElfAnalyzer.linked_libraries_from_bytes(data) - if libs: - lib_names = [lib.name for lib in libs] - rel = bin_path.relative_to(dst_base) - facts.append( - { - "probe_id": "IMG_LINKED_LIBS", - "detail": ", ".join(lib_names)[:200], - "provider": "elf", - "source": f"/{rel}", - "extra": {"binary": f"/{rel}", "libs": lib_names}, - } - ) - return facts - - -def _collect_packages(rt: ContainerRuntime, image: str) -> list[dict]: - """Get installed package list via rpm/dpkg/apk.""" - facts: list[dict[str, Any]] = [] - - r = rt.run_cmd( - image, - ["-qa", "--queryformat", "%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}\\n"], - entrypoint="rpm", - timeout=30, - ) - if r.returncode == 0 and r.stdout.strip(): - for line in r.stdout.splitlines(): - line = line.strip() - if line: - facts.append( - { - "probe_id": "IMG_PACKAGES", - "detail": line[:200], - "provider": "rpm", - "source": "rpm -qa", - "extra": {"format": "rpm", "nevra": line}, - } - ) - return facts - - r = rt.run_cmd( - image, - ["-W", "-f", "${Package} ${Version}\\n"], - entrypoint="dpkg-query", - timeout=30, - ) - if r.returncode == 0 and r.stdout.strip(): - for line in r.stdout.splitlines(): - line = line.strip() - if line: - facts.append( - { - "probe_id": "IMG_PACKAGES", - "detail": line[:200], - "provider": "dpkg", - "source": "dpkg-query", - "extra": {"format": "deb"}, - } - ) - return facts - - r = rt.run_cmd(image, ["list", "--installed"], entrypoint="apk", timeout=30) - if r.returncode == 0 and r.stdout.strip(): - for line in r.stdout.splitlines(): - line = line.strip() - if line: - facts.append( - { - "probe_id": "IMG_PACKAGES", - "detail": line[:200], - "provider": "apk", - "source": "apk list", - "extra": {"format": "apk"}, - } - ) - return facts - - -def _collect_certs(rt: ContainerRuntime, cid: str, tmpdir: str) -> list[dict]: - """List certificate files from /etc/pki and /etc/ssl.""" - facts: list[dict[str, Any]] = [] - cert_dst = Path(tmpdir) / "certs" - cert_dst.mkdir(parents=True, exist_ok=True) - - for src_dir in ("/etc/pki", "/etc/ssl/certs"): - rt.cp_out(cid, src_dir, str(cert_dst / src_dir.lstrip("/"))) - - for cert_file in sorted(cert_dst.rglob("*")): - if cert_file.is_file() and cert_file.suffix.lower() in ( - ".pem", - ".crt", - ".cert", - ".key", - ): - rel = cert_file.relative_to(cert_dst) - facts.append( - { - "probe_id": "IMG_CERT_FILES", - "detail": f"/{rel}", - "provider": "filesystem", - "source": "cp /etc/pki + /etc/ssl", - "extra": {"path": f"/{rel}", "size": cert_file.stat().st_size}, - } - ) - return facts - - -def _collect_crypto_files(rt: ContainerRuntime, cid: str, tmpdir: str) -> list[dict]: - """Check for crypto-policy and FIPS indicator files.""" - facts: list[dict[str, Any]] = [] - policy_dst = Path(tmpdir) / "crypto-policies" - policy_dst.mkdir(parents=True, exist_ok=True) - - if rt.cp_out(cid, "/etc/crypto-policies/state/current", str(policy_dst / "current")): - current = (policy_dst / "current").read_text(errors="replace").strip() - if current: - facts.append( - { - "probe_id": "IMG_CRYPTO_POLICY", - "detail": current, - "provider": "crypto-policies", - "source": "/etc/crypto-policies/state/current", - "extra": {"policy": current}, - } - ) - - fips_dst = Path(tmpdir) / "fips" - fips_dst.mkdir(parents=True, exist_ok=True) - if rt.cp_out(cid, "/etc/system-fips", str(fips_dst / "system-fips")): - facts.append( - { - "probe_id": "IMG_FIPS_MARKER", - "detail": "/etc/system-fips present", - "provider": "fips", - "source": "/etc/system-fips", - } - ) - - for fips_path in ( - "/usr/lib64/ossl-modules/fips.so", - "/usr/lib/ossl-modules/fips.so", - ): - check_dst = fips_dst / Path(fips_path).name - if rt.cp_out(cid, fips_path, str(check_dst)): - facts.append( - { - "probe_id": "IMG_FIPS_MODULE", - "detail": f"FIPS module at {fips_path}", - "provider": "openssl-fips", - "source": fips_path, - } - ) - break - - return facts - - -def run_image( - image: str, - *, - runtime: str = "", - platform: str = "", - no_pull: bool = False, - component: str = "", -) -> dict[str, Any]: - errors: list[str] = [] - facts: list[dict[str, Any]] = [] - metadata: dict[str, Any] = {"image": image} - tmpdir = tempfile.mkdtemp(prefix="crypto-audit-image-") - - try: - rt = ContainerRuntime(runtime=runtime or "auto", platform=platform) - except RuntimeError as e: - errors.append(str(e)) - return build_output( - tier="binary", - target=image, - facts=facts, - errors=errors, - component=component, - metadata=metadata, - ) - - try: - if not no_pull and not rt.pull(image): - errors.append("pull failed") - return build_output( - tier="binary", - target=image, - facts=facts, - errors=errors, - component=component, - metadata=metadata, - ) - - meta_facts, img_meta = _collect_image_metadata(rt, image) - facts.extend(meta_facts) - metadata.update(img_meta) - - cid = rt.create(image) - metadata["container_id"] = cid[:12] - - try: - facts.extend(_collect_linked_libs(rt, cid, tmpdir)) - facts.extend(_collect_packages(rt, image)) - facts.extend(_collect_certs(rt, cid, tmpdir)) - facts.extend(_collect_crypto_files(rt, cid, tmpdir)) - finally: - rt.remove(cid) - - except Exception as e: - errors.append(f"{type(e).__name__}: {e}") - finally: - shutil.rmtree(tmpdir, ignore_errors=True) - - return build_output( - tier="binary", - target=image, - facts=facts, - errors=errors, - component=component, - metadata=metadata, - ) - - -# --------------------------------------------------------------------------- -# Cluster tier — raw TLS + system state collection -# --------------------------------------------------------------------------- - - -def _oc_cmd(context: str, args: list[str]) -> list[str]: - cmd = ["oc"] - if context: - cmd += ["--context", context] - cmd += args - return cmd - - -def _run_oc( - context: str, args: list[str], *, timeout: int = 60 -) -> subprocess.CompletedProcess[str]: - return subprocess.run(_oc_cmd(context, args), capture_output=True, text=True, timeout=timeout) - - -def _oc_exec(context: str, namespace: str, pod: str, shell_cmd: str, *, timeout: int = 30): - r = _run_oc( - context, - ["exec", "-n", namespace, pod, "--", "sh", "-c", shell_cmd], - timeout=timeout, - ) - return r.returncode, r.stdout, r.stderr - - -def _find_apiserver_pod(context: str) -> tuple[str, str] | None: - for ns in ("openshift-kube-apiserver", "kube-system"): - r = _run_oc( - context, - [ - "get", - "pods", - "-n", - ns, - "-o", - 'jsonpath={range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\\n"}{end}', - ], - timeout=30, - ) - if r.returncode == 0: - for pod in r.stdout.splitlines(): - pod = pod.strip() - if pod and ("apiserver" in pod or "guard" in pod): - return ns, pod - return None - - -def _list_running_pods(context: str, namespace: str) -> list[str]: - r = _run_oc( - context, - [ - "get", - "pods", - "-n", - namespace, - "-o", - 'jsonpath={range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\\n"}{end}', - ], - timeout=30, - ) - if r.returncode != 0: - return [] - return [p.strip() for p in r.stdout.splitlines() if p.strip()] - - -def _parse_tls_output(raw: str) -> dict[str, Any]: - """Parse full openssl s_client output into structured negotiation data.""" - result: dict[str, Any] = {} - for line in raw.splitlines(): - line = line.strip() - if not line: - continue - if line.startswith("Protocol") and ":" in line: - result["protocol"] = line.split(":", 1)[-1].strip() - elif line.startswith("Ciphersuite"): - result["ciphersuite"] = line.split(":", 1)[-1].strip() - elif "Cipher is" in line: - result.setdefault("ciphersuite", line.split("Cipher is")[-1].strip()) - elif line.startswith("Cipher") and ":" in line and "is" not in line.lower(): - result.setdefault("ciphersuite", line.split(":", 1)[-1].strip()) - elif "Negotiated TLS1.3 group" in line: - result["key_exchange_group"] = line.split(":", 1)[-1].strip() - elif line.startswith(("Server Temp Key", "Peer Temp Key")): - result.setdefault("key_exchange_group", line.split(":", 1)[-1].strip()) - elif "Peer signature type" in line or line.startswith("Signature type"): - result["peer_signature_type"] = line.split(":", 1)[-1].strip() - elif "Peer signing digest" in line or line.startswith("Hash used"): - result["peer_signature_digest"] = line.split(":", 1)[-1].strip() - elif "Server public key is" in line: - m = re.search(r"(\d+)\s*bit", line) - if m: - result["server_cert_key_bits"] = int(m.group(1)) - elif line.startswith("subject="): - result["server_cert_subject"] = line.split("=", 1)[-1].strip()[:200] - elif line.startswith("issuer="): - result["server_cert_issuer"] = line.split("=", 1)[-1].strip()[:200] - elif "ALPN protocol" in line: - result["alpn"] = line.split(":", 1)[-1].strip() if ":" in line else "" - elif "New," in line and "Cipher is" in line: - result.setdefault("protocol", line.split(",")[0].replace("New", "").strip()) - return result - - -def _tls_probe( - context: str, - ns: str, - pod: str, - host: str, - port: int, - *, - groups: str = "", - timeout: int = _TLS_CONNECT_TIMEOUT, -) -> dict[str, Any] | None: - cmd = f"echo | timeout {timeout} openssl s_client -connect {host}:{port}" - if groups: - cmd += f" -groups {groups}" - cmd += " 2>&1" - rc, out, _ = _oc_exec(context, ns, pod, cmd, timeout=timeout + 10) - if rc != 0 and not out.strip(): - return None - parsed = _parse_tls_output(out) - if not parsed.get("protocol") and not parsed.get("ciphersuite"): - return None - return parsed - - -def _classify_port(port_name: str, port_number: int) -> str: - name_lower = (port_name or "").lower() - if any(kw in name_lower for kw in ("metrics", "health", "readiness")): - return "observability" - if any(kw in name_lower for kw in ("https", "tls", "secure")): - return "service" - if port_number in (443, 8443, 6443, 9093): - return "service" - return "unknown" - - -def _discover_services(context: str, namespace: str, errors: list[str]) -> list[dict[str, Any]]: - r = _run_oc(context, ["get", "svc", "-n", namespace, "-o", "json"], timeout=30) - if r.returncode != 0: - errors.append(f"oc get svc -n {namespace} failed: {r.stderr.strip()[:200]}") - return [] - try: - data = json.loads(r.stdout) - except json.JSONDecodeError as e: - errors.append(f"svc JSON parse error in {namespace}: {e}") - return [] - - endpoints: list[dict[str, Any]] = [] - for item in data.get("items", []): - spec = item.get("spec", {}) - svc_name = item.get("metadata", {}).get("name", "") - cluster_ip = spec.get("clusterIP", "") - if not cluster_ip or cluster_ip == "None": - continue - for port_entry in spec.get("ports", []): - port_num = port_entry.get("port") - port_name = port_entry.get("name", "") - if not port_num: - continue - endpoints.append( - { - "service": svc_name, - "namespace": namespace, - "host": f"{svc_name}.{namespace}.svc", - "port": port_num, - "port_name": port_name, - "port_class": _classify_port(port_name, port_num), - } - ) - return endpoints - - -def _probe_pod_crypto( - context: str, - namespace: str, - pod: str, - errors: list[str], -) -> list[dict[str, Any]]: - """Collect crypto-policy, openssl version, go version from a pod.""" - facts: list[dict[str, Any]] = [] - source_prefix = f"{namespace}/{pod}" - - policy_cmd = ( - "cat /etc/crypto-policies/state/current 2>/dev/null; " - "echo '---'; " - "cat /proc/sys/crypto/fips_enabled 2>/dev/null; " - "echo '---'; " - "openssl version 2>/dev/null" - ) - rc, out, _ = _oc_exec(context, namespace, pod, policy_cmd) - if rc == 0 and out.strip(): - parts = out.split("---") - policy = parts[0].strip() if len(parts) > 0 else "" - fips = parts[1].strip() if len(parts) > 1 else "" - openssl_ver = parts[2].strip() if len(parts) > 2 else "" - - if policy: - facts.append( - { - "probe_id": "CLUSTER_CRYPTO_POLICY", - "detail": policy, - "provider": "crypto-policies", - "source": source_prefix, - "extra": { - "namespace": namespace, - "pod": pod, - "policy": policy, - "fips_enabled": fips == "1", - }, - } - ) - if openssl_ver: - ver_m = re.search(r"(OpenSSL|LibreSSL|BoringSSL)\s+([\d.]+\S*)", openssl_ver, re.I) - facts.append( - { - "probe_id": "CLUSTER_OPENSSL_VERSION", - "detail": openssl_ver[:200], - "provider": "openssl", - "source": source_prefix, - "version": ver_m.group(2) if ver_m else None, - "extra": {"namespace": namespace, "pod": pod}, - } - ) - - go_cmd = ( - "go version /proc/1/exe 2>/dev/null || " - "strings -n 12 /proc/1/exe 2>/dev/null | grep -m1 '^go1\\.' || true" - ) - rc, out, _ = _oc_exec(context, namespace, pod, go_cmd) - if rc == 0 and out.strip(): - go_m = re.search(r"go(1\.\d+(?:\.\d+)?)", out) - if go_m: - facts.append( - { - "probe_id": "CLUSTER_GO_VERSION", - "detail": out.strip()[:200], - "provider": "go", - "source": source_prefix, - "version": go_m.group(1), - "extra": {"namespace": namespace, "pod": pod}, - } - ) - - return facts - - -def run_cluster( - context: str, - namespaces: list[str], - *, - component: str = "", - discover: bool = False, - groups: str = "", -) -> dict[str, Any]: - errors: list[str] = [] - facts: list[dict[str, Any]] = [] - metadata: dict[str, Any] = {"context": context, "namespaces": namespaces} - - if not shutil.which("oc"): - errors.append("oc not found in PATH") - return build_output( - tier="runtime", - target=context, - facts=facts, - errors=errors, - component=component, - metadata=metadata, - ) - - apiserver = _find_apiserver_pod(context) - - if discover: - metadata["mode"] = "discover" - discovery_meta: dict[str, Any] = {"namespaces": {}} - for namespace in namespaces: - endpoints = _discover_services(context, namespace, errors) - ns_meta = { - "services_enumerated": len({e["service"] for e in endpoints}), - "endpoints_total": len(endpoints), - "tls_endpoints": 0, - "plaintext_endpoints": 0, - } - - exec_pod = None - pods = _list_running_pods(context, namespace) - if pods: - exec_pod = (namespace, pods[0]) - elif apiserver: - exec_pod = apiserver - if not exec_pod: - errors.append(f"no exec pod for namespace {namespace}") - discovery_meta["namespaces"][namespace] = ns_meta - continue - - for ep in endpoints: - negotiation = _tls_probe( - context, - exec_pod[0], - exec_pod[1], - ep["host"], - ep["port"], - groups=groups, - ) - if not negotiation: - ns_meta["plaintext_endpoints"] += 1 - continue - ns_meta["tls_endpoints"] += 1 - detail = ( - f"{ep['host']}:{ep['port']} " - f"{negotiation.get('protocol', '?')} " - f"{negotiation.get('ciphersuite', '?')} " - f"{negotiation.get('key_exchange_group', '?')}" - ) - facts.append( - { - "probe_id": "CLUSTER_TLS_NEGOTIATION", - "detail": detail[:200], - "provider": "tls", - "source": f"oc exec {exec_pod[1]} -> {ep['host']}:{ep['port']}", - "extra": { - "service": ep["service"], - "namespace": ep["namespace"], - "host": ep["host"], - "port": ep["port"], - "port_name": ep["port_name"], - "port_class": ep["port_class"], - "negotiation": negotiation, - }, - } - ) - - discovery_meta["namespaces"][namespace] = ns_meta - metadata["discovery"] = discovery_meta - - else: - metadata["mode"] = "default" - if apiserver: - ns, pod = apiserver - default_targets = [ - ("kubernetes.default.svc", 443, "kubernetes-api"), - ("oauth-openshift.openshift-authentication.svc", 443, "oauth"), - ("etcd-client.openshift-etcd.svc", 2379, "etcd"), - ("image-registry.openshift-image-registry.svc", 5000, "image-registry"), - ] - for host, port, service in default_targets: - negotiation = _tls_probe(context, ns, pod, host, port, groups=groups) - if not negotiation: - errors.append(f"TLS probe failed for {host}:{port}") - continue - detail = ( - f"{host}:{port} " - f"{negotiation.get('protocol', '?')} " - f"{negotiation.get('ciphersuite', '?')} " - f"{negotiation.get('key_exchange_group', '?')}" - ) - facts.append( - { - "probe_id": "CLUSTER_TLS_NEGOTIATION", - "detail": detail[:200], - "provider": "tls", - "source": f"oc exec {pod} -> {host}:{port}", - "extra": { - "service": service, - "namespace": host.split(".")[1] if "." in host else "", - "host": host, - "port": port, - "port_name": "", - "port_class": _classify_port("", port), - "negotiation": negotiation, - }, - } - ) - else: - errors.append("no running kube-apiserver pod found") - - for namespace in namespaces: - pods = _list_running_pods(context, namespace) - if not pods: - errors.append(f"no running pods in namespace {namespace}") - continue - facts.extend(_probe_pod_crypto(context, namespace, pods[0], errors)) - - return build_output( - tier="runtime", - target=context, - facts=facts, - errors=errors, - component=component, - metadata=metadata, - ) - - -# --------------------------------------------------------------------------- -# Typed dispatch (app owns CLI) -# --------------------------------------------------------------------------- - - -def audit_source( - path: Path | str, - *, - from_sbom: Path | str | None = None, - component: str = "", - output: Path | str | None = None, - quiet: bool = False, -) -> int: - repo = Path(path).resolve() - if not repo.is_dir(): - print(f"error: not a directory: {repo}", file=sys.stderr) - return 1 - sbom = Path(from_sbom).resolve() if from_sbom else None - if sbom and not sbom.is_file(): - print(f"error: SBOM not found: {sbom}", file=sys.stderr) - return 1 - if not quiet: - print(f"[crypto-audit] source tier: {repo}", file=sys.stderr) - payload = run_source(repo, sbom_path=sbom, component=component) - if not quiet: - print( - f"[crypto-audit] {payload['summary']['total_facts']} facts, " - f"{len(payload['errors'])} errors", - file=sys.stderr, - ) - _write_output(payload, str(output) if output else "", quiet) - return 0 - - -def audit_image( - image: str, - *, - runtime: str = "", - platform: str = "", - no_pull: bool = False, - component: str = "", - output: Path | str | None = None, - quiet: bool = False, -) -> int: - if not quiet: - print(f"[crypto-audit] image tier: {image}", file=sys.stderr) - payload = run_image( - image, - runtime=runtime, - platform=platform, - no_pull=no_pull, - component=component, - ) - if not quiet: - print( - f"[crypto-audit] {payload['summary']['total_facts']} facts, " - f"{len(payload['errors'])} errors", - file=sys.stderr, - ) - _write_output(payload, str(output) if output else "", quiet) - return 0 - - -def audit_cluster( - context: str, - namespaces: list[str], - *, - targets: Path | str, - discover: bool = False, - groups: str = "", - component: str = "", - output: Path | str | None = None, - quiet: bool = False, -) -> int: - if not targets: - raise ValueError("targets file is required for cluster audit") - - import datetime as _dt - - from traust_engine.validation.scope import Action as _Action - from traust_engine.validation.scope import Scope as _Scope - - scope = _Scope.from_targets_file(Path(targets)) - - audit_path = (Path(output).parent if output else Path.cwd()) / "crypto-audit-scope.jsonl" - allowed, denied = [], [] - for ns in namespaces: - ok, reason = scope.is_in_scope( - _Action(adapter="k8s", verb="exec", context=context, namespace=ns) - ) - with audit_path.open("a", encoding="utf-8") as fh: - fh.write( - json.dumps( - { - "ts": _dt.datetime.now(_dt.UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), - "action": f"k8s/exec ctx={context} ns={ns}", - "scope_allowed": ok, - "scope_reason": reason, - } - ) - + "\n" - ) - (allowed if ok else denied).append((ns, reason)) - if denied and not quiet: - for ns, reason in denied: - print(f"[crypto-audit] SCOPE DENIED ns={ns}: {reason}", file=sys.stderr) - if not allowed: - print( - "[crypto-audit] no in-scope namespaces — nothing to probe (fail-closed)", - file=sys.stderr, - ) - payload = { - "schema": "crypto-audit/v1", - "tier": "runtime", - "component": component or "", - "facts": [], - "errors": [f"scope denied: {ns} — {r}" for ns, r in denied], - "summary": {"total_facts": 0}, - } - else: - if not quiet: - print( - f"[crypto-audit] cluster tier: " - f"context={context} " - f"namespaces={','.join(ns for ns, _ in allowed)} " - f"mode={'discover' if discover else 'default'}", - file=sys.stderr, - ) - payload = run_cluster( - context, - [ns for ns, _ in allowed], - component=component, - discover=discover, - groups=groups, - ) - payload.setdefault("errors", []).extend(f"scope denied: {ns} — {r}" for ns, r in denied) - - if not quiet: - print( - f"[crypto-audit] {payload['summary']['total_facts']} facts, " - f"{len(payload['errors'])} errors", - file=sys.stderr, - ) - _write_output(payload, str(output) if output else "", quiet) - return 0 diff --git a/src/traust_engine/adapters/crypto_probe.py b/src/traust_engine/adapters/crypto_probe.py deleted file mode 100644 index 7cf1727..0000000 --- a/src/traust_engine/adapters/crypto_probe.py +++ /dev/null @@ -1,904 +0,0 @@ -"""crypto_probe.py — General-purpose crypto posture probing for a repository. - -Deterministic static analysis of build files, Dockerfiles, lock files, and -source to discover what crypto stack a repository uses. This is a **provider -census** tool — it identifies which crypto stacks are present and their -versions. TLS configuration details (cipher suites, protocol versions) and -code-level crypto API usage are handled by pqc-scan / opengrep rules. - -Each fact is a dict with at minimum: - probe_id — what was detected (CRYPTO_* namespace) - file — relative path within the repo - line — 1-based line number (0 if not applicable) - detail — human-readable description (max 200 chars) - provider — crypto stack identifier (go, openssl, jdk, node, rustls, ...) - -Optional per-probe fields: - version — detected version string - extra — dict of probe-specific metadata - -Usage: - crypto_probe.py --repo-dir DIR [--json] - python -c 'import crypto_probe; print(crypto_probe.probe_repo(Path("...")))' -""" - -from __future__ import annotations - -import json -import re -from dataclasses import dataclass, field -from pathlib import Path - -VENDOR_RX = re.compile( - r"(^|/)(vendor|node_modules|third_party|_vendor|3rdparty|third-party|" - r"bundled|external|site-packages|\.gomodcache)(/|$)" -) - -GO_DIRECTIVE_RX = re.compile(r"(?m)^go\s+(\d+)\.(\d+)") -FROM_RX = re.compile(r"(?mi)^\s*FROM\s+(\S+)") -FROM_ALIAS_RX = re.compile(r"(?mi)^\s*FROM\s+\S+\s+AS\s+(\S+)") -PKG_INSTALL_OPENSSL_RX = re.compile( - r"(?mi)\b(dnf|yum|microdnf|apt-get|apt|apk)\b[^\n]*" - r"(?:install|add)[^\n]*\b(openssl\S*|libssl\S*|gnutls\S*|libnss\S*)" -) -GOEXPERIMENT_RX = re.compile( - r"(?mi)GOEXPERIMENT\s*[=:]\s*(\S*(?:boringcrypto|opensslcrypto|cngcrypto)\S*)" -) -CGO_FIPS_RX = re.compile(r"(?mi)CGO_ENABLED\s*[=:]\s*1") -GODEBUG_TLS_RX = re.compile(r"GODEBUG\s*[=:].{0,80}?tls(mlkem|kyber)=0") -GODEBUG_FIPS_RX = re.compile(r"GODEBUG\s*[=:].{0,80}?fips140\s*=\s*(on|only)") -CRYPTO_POLICY_RX = re.compile( - r"update-crypto-policies\s+--set\s+(\S+)|\bFIPS[:=\s]*(mode|true|enabled)", re.I -) -NEVRA_CRYPTO_RX = re.compile( - r"(?m)(openssl(?:-libs|-fips-provider)?|crypto-policies|gnutls|nss)" - r"[^\n]{0,40}?(\d+[.:][\w.\-]+)" -) -CARGO_TLS_RX = re.compile( - r"(?m)(rustls|native-tls|openssl|ring|aws-lc-rs)\s*=\s*\{[^}]*" - r'version\s*=\s*"([^"]+)"' -) -CARGO_SIMPLE_RX = re.compile(r'(?m)(rustls|native-tls|ring|aws-lc-rs)\s*=\s*"([^"]+)"') -GRADLE_JAVA_RX = re.compile( - r"(?:sourceCompatibility|targetCompatibility|toolchain\s*\{[^}]*" - r"languageVersion\.set\(JavaLanguageVersion\.of\()\s*[=( ]*(\d+)" -) -PYTHON_CRYPTO_DEPS_RX = re.compile( - r"(?mi)^(cryptography|pyopenssl|pycryptodome|pynacl|paramiko|" - r"python-jose|pyjwt|certifi)(?:\[[\w,]+\])?\s*[=~>net(\d+)\.(\d+)") -CSPROJ_PKG_CRYPTO_RX = re.compile( - r' bool: - return bool(VENDOR_RX.search(rel)) - - -def _extract_min_semver(constraint: str) -> str | None: - """Extract the minimum version from a semver constraint like '>=18.0.0' or '^20'.""" - m = re.search(r"[>=^~]*\s*(\d+(?:\.\d+)*)", constraint) - return m.group(1) if m else None - - -# ---------- Go ---------- - - -def probe_go(repo: Path) -> list[CryptoFact]: - facts = [] - for gm in _rfiles(repo, "go.mod"): - rel = str(gm.relative_to(repo)) - if _is_vendor(rel): - continue - text = gm.read_text(errors="replace") - m = GO_DIRECTIVE_RX.search(text) - if m: - major, minor = int(m.group(1)), int(m.group(2)) - facts.append( - CryptoFact( - probe_id="CRYPTO_GO_TOOLCHAIN", - file=rel, - line=1, - detail=f"go {major}.{minor}", - provider="go", - version=f"{major}.{minor}", - ) - ) - if GOLANG_FIPS_RX.search(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_GO_FIPS_BACKEND", - file=rel, - line=1, - detail="golang-fips or boring crypto dependency in go.mod", - provider="go-fips", - extra={ - "crypto_backend_swap": True, - "note": "Go stdlib crypto replaced by OpenSSL via CGo", - }, - ) - ) - for gs in _rfiles(repo, "go.sum"): - rel = str(gs.relative_to(repo)) - if _is_vendor(rel): - continue - text = gs.read_text(errors="replace") - if GOLANG_FIPS_RX.search(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_GO_FIPS_BACKEND", - file=rel, - line=1, - detail="golang-fips or boring crypto dependency in go.sum", - provider="go-fips", - extra={"crypto_backend_swap": True}, - ) - ) - return facts - - -# ---------- Dockerfiles ---------- - - -def probe_dockerfiles(repo: Path) -> list[CryptoFact]: - facts = [] - for name in ("Dockerfile", "Containerfile"): - for df in _rfiles(repo, f"{name}*"): - rel = str(df.relative_to(repo)) - if _is_vendor(rel) or not df.is_file(): - continue - try: - text = df.read_text(errors="replace") - except OSError: - continue - - aliases = {a.lower() for a in FROM_ALIAS_RX.findall(text)} - for m in FROM_RX.finditer(text): - img = m.group(1) - if img.lower() in aliases or img.lower() == "scratch": - continue - line = text[: m.start()].count("\n") + 1 - facts.append( - CryptoFact( - probe_id="CRYPTO_BASE_IMAGE", - file=rel, - line=line, - detail=img, - provider="base-image", - ) - ) - - for m in PKG_INSTALL_OPENSSL_RX.finditer(text): - line = text[: m.start()].count("\n") + 1 - facts.append( - CryptoFact( - probe_id="CRYPTO_OPENSSL_INSTALL", - file=rel, - line=line, - detail=m.group(0)[:200], - provider="openssl", - ) - ) - - for m in DOCKERFILE_ARG_ENV_RX.finditer(text): - key, val = m.group(1), m.group(2) - combined = f"{key}={val}".lower() - if any(kw in combined for kw in CRYPTO_KEYWORDS): - line = text[: m.start()].count("\n") + 1 - facts.append( - CryptoFact( - probe_id="CRYPTO_DOCKERFILE_DIRECTIVE", - file=rel, - line=line, - detail=f"{key}={val}"[:200], - provider="container-build", - extra={"directive_type": "env", "key": key, "value": val}, - ) - ) - - for m in DOCKERFILE_LABEL_RX.finditer(text): - key, val = m.group(1), m.group(2).strip() - combined = f"{key}={val}".lower() - if any(kw in combined for kw in CRYPTO_KEYWORDS): - line = text[: m.start()].count("\n") + 1 - facts.append( - CryptoFact( - probe_id="CRYPTO_DOCKERFILE_DIRECTIVE", - file=rel, - line=line, - detail=f"{key}={val}"[:200], - provider="container-build", - extra={"directive_type": "label", "key": key, "value": val}, - ) - ) - return facts - - -# ---------- Node.js ---------- - - -def probe_node(repo: Path) -> list[CryptoFact]: - facts = [] - for nv in _rfiles(repo, ".nvmrc"): - rel = str(nv.relative_to(repo)) - if _is_vendor(rel): - continue - ver = nv.read_text(errors="replace").strip().lstrip("v") - m = re.match(r"(\d+(?:\.\d+)*)", ver) - if m: - facts.append( - CryptoFact( - probe_id="CRYPTO_NODE_VERSION", - file=rel, - line=1, - detail=f"node {ver}", - provider="node", - version=m.group(1), - ) - ) - for pj in _rfiles(repo, "package.json"): - rel = str(pj.relative_to(repo)) - if _is_vendor(rel): - continue - try: - eng = (json.loads(pj.read_text(errors="replace")).get("engines") or {}).get("node") - except (json.JSONDecodeError, AttributeError): - continue - if eng: - ver = _extract_min_semver(str(eng)) - if ver: - facts.append( - CryptoFact( - probe_id="CRYPTO_NODE_VERSION", - file=rel, - line=1, - detail=f"engines.node {eng}", - provider="node", - version=ver, - ) - ) - return facts - - -# ---------- JDK ---------- - - -def probe_jdk(repo: Path) -> list[CryptoFact]: - facts = [] - for pom in _rfiles(repo, "pom.xml"): - rel = str(pom.relative_to(repo)) - if _is_vendor(rel): - continue - text = pom.read_text(errors="replace") - m = re.search( - r"<(?:maven\.compiler\.(?:release|source)|" - r"java\.version)>\s*(\d+)", - text, - ) - if m: - facts.append( - CryptoFact( - probe_id="CRYPTO_JDK_VERSION", - file=rel, - line=1, - detail=f"jdk {m.group(1)}", - provider="jdk", - version=m.group(1), - ) - ) - for bg in _rfiles(repo, "build.gradle*"): - rel = str(bg.relative_to(repo)) - if _is_vendor(rel): - continue - text = bg.read_text(errors="replace") - m = GRADLE_JAVA_RX.search(text) - if m: - facts.append( - CryptoFact( - probe_id="CRYPTO_JDK_VERSION", - file=rel, - line=1, - detail=f"jdk {m.group(1)}", - provider="jdk", - version=m.group(1), - ) - ) - for sec in _rfiles(repo, "java.security"): - rel = str(sec.relative_to(repo)) - if _is_vendor(rel): - continue - try: - text = sec.read_text(errors="replace") - except OSError: - continue - for m in JVM_TLS_PROPS_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_JVM_SECURITY_PROPERTY", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=f"{m.group(1)}={m.group(2)[:150]}", - provider="jdk", - ) - ) - return facts - - -# ---------- Python ---------- - - -def probe_python(repo: Path) -> list[CryptoFact]: - facts = [] - for pv in _rfiles(repo, ".python-version"): - rel = str(pv.relative_to(repo)) - if _is_vendor(rel): - continue - ver = pv.read_text(errors="replace").strip() - m = re.match(r"(\d+(?:\.\d+)*)", ver) - if m: - facts.append( - CryptoFact( - probe_id="CRYPTO_PYTHON_VERSION", - file=rel, - line=1, - detail=f"python {ver}", - provider="python", - version=m.group(1), - ) - ) - - for reqfile in ("requirements.txt", "requirements-dev.txt", "requirements-security.txt"): - for rf in _rfiles(repo, reqfile): - rel = str(rf.relative_to(repo)) - if _is_vendor(rel): - continue - try: - text = rf.read_text(errors="replace") - except OSError: - continue - for m in PYTHON_CRYPTO_DEPS_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_PYTHON_CRYPTO_DEP", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=f"{m.group(1)}=={m.group(2)}", - provider="python", - version=m.group(2), - extra={"package": m.group(1)}, - ) - ) - - for pp in _rfiles(repo, "pyproject.toml"): - rel = str(pp.relative_to(repo)) - if _is_vendor(rel): - continue - try: - text = pp.read_text(errors="replace") - except OSError: - continue - pyver = re.search(r'requires-python\s*=\s*"([^"]+)"', text) - if pyver: - ver = _extract_min_semver(pyver.group(1)) - if ver: - facts.append( - CryptoFact( - probe_id="CRYPTO_PYTHON_VERSION", - file=rel, - line=text[: pyver.start()].count("\n") + 1, - detail=f"requires-python {pyver.group(1)}", - provider="python", - version=ver, - ) - ) - for m in PYTHON_CRYPTO_DEPS_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_PYTHON_CRYPTO_DEP", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=f"{m.group(1)}=={m.group(2)}", - provider="python", - version=m.group(2), - extra={"package": m.group(1)}, - ) - ) - - for pf in _rfiles(repo, "Pipfile"): - rel = str(pf.relative_to(repo)) - if _is_vendor(rel): - continue - try: - text = pf.read_text(errors="replace") - except OSError: - continue - pyver = re.search(r'python_version\s*=\s*"([^"]+)"', text) - if pyver: - facts.append( - CryptoFact( - probe_id="CRYPTO_PYTHON_VERSION", - file=rel, - line=text[: pyver.start()].count("\n") + 1, - detail=f"python_version {pyver.group(1)}", - provider="python", - version=pyver.group(1), - ) - ) - return facts - - -# ---------- Rust ---------- - - -def probe_rust(repo: Path) -> list[CryptoFact]: - facts = [] - for ct in _rfiles(repo, "Cargo.toml"): - rel = str(ct.relative_to(repo)) - if _is_vendor(rel): - continue - text = ct.read_text(errors="replace") - rv = re.search(r'(?m)^rust-version\s*=\s*"([^"]+)"', text) - if rv: - facts.append( - CryptoFact( - probe_id="CRYPTO_RUST_VERSION", - file=rel, - line=1, - detail=f"rust {rv.group(1)}", - provider="rust", - version=rv.group(1), - ) - ) - for m in CARGO_TLS_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_RUST_TLS_BACKEND", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=f"{m.group(1)} {m.group(2)}", - provider=m.group(1), - version=m.group(2), - ) - ) - for m in CARGO_SIMPLE_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_RUST_TLS_BACKEND", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=f"{m.group(1)} {m.group(2)}", - provider=m.group(1), - version=m.group(2), - ) - ) - return facts - - -# ---------- C/C++ ---------- - - -def probe_c(repo: Path) -> list[CryptoFact]: - """Probe C/C++ projects for TLS library linkage.""" - facts = [] - for cmake in _rfiles(repo, "CMakeLists.txt"): - rel = str(cmake.relative_to(repo)) - if _is_vendor(rel): - continue - text = cmake.read_text(errors="replace") - for lib in ("OpenSSL", "GnuTLS", "MbedTLS", "wolfSSL"): - if re.search(rf"find_package\(\s*{lib}", text, re.I): - facts.append( - CryptoFact( - probe_id="CRYPTO_C_TLS_LIBRARY", - file=rel, - line=1, - detail=f"find_package({lib})", - provider=lib.lower(), - ) - ) - for m in re.finditer( - r"pkg_check_modules\([^)]*\b(openssl|gnutls|" - r"mbedtls|wolfssl)\b", - text, - re.I, - ): - facts.append( - CryptoFact( - probe_id="CRYPTO_C_TLS_LIBRARY", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=f"pkg_check_modules {m.group(1)}", - provider=m.group(1).lower(), - ) - ) - for conf in _rfiles(repo, "configure.ac"): - rel = str(conf.relative_to(repo)) - if _is_vendor(rel): - continue - text = conf.read_text(errors="replace") - for lib in ("openssl", "gnutls", "mbedtls"): - if re.search(rf"AC_CHECK_LIB\(\[?{lib}", text, re.I): - facts.append( - CryptoFact( - probe_id="CRYPTO_C_TLS_LIBRARY", - file=rel, - line=1, - detail=f"AC_CHECK_LIB({lib})", - provider=lib, - ) - ) - return facts - - -# ---------- .NET / C# ---------- - - -def probe_dotnet(repo: Path) -> list[CryptoFact]: - facts = [] - for gj in _rfiles(repo, "global.json"): - rel = str(gj.relative_to(repo)) - if _is_vendor(rel): - continue - try: - doc = json.loads(gj.read_text(errors="replace")) - ver = (doc.get("sdk") or {}).get("version") - if ver: - facts.append( - CryptoFact( - probe_id="CRYPTO_DOTNET_SDK", - file=rel, - line=1, - detail=f"dotnet sdk {ver}", - provider="dotnet", - version=ver, - ) - ) - except (json.JSONDecodeError, AttributeError): - continue - for csproj in _rfiles(repo, "*.csproj"): - rel = str(csproj.relative_to(repo)) - if _is_vendor(rel): - continue - try: - text = csproj.read_text(errors="replace") - except OSError: - continue - m = CSPROJ_TFM_RX.search(text) - if m: - ver = f"{m.group(1)}.{m.group(2)}" - facts.append( - CryptoFact( - probe_id="CRYPTO_DOTNET_TFM", - file=rel, - line=1, - detail=f"net{ver}", - provider="dotnet", - version=ver, - ) - ) - for m in CSPROJ_PKG_CRYPTO_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_DOTNET_CRYPTO_PKG", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=f"{m.group(1)} {m.group(2)}", - provider="dotnet", - version=m.group(2), - extra={"package": m.group(1)}, - ) - ) - return facts - - -# ---------- Ruby ---------- - - -def probe_ruby(repo: Path) -> list[CryptoFact]: - facts = [] - for rv in _rfiles(repo, ".ruby-version"): - rel = str(rv.relative_to(repo)) - if _is_vendor(rel): - continue - ver = rv.read_text(errors="replace").strip() - m = re.match(r"(\d+(?:\.\d+)*)", ver) - if m: - facts.append( - CryptoFact( - probe_id="CRYPTO_RUBY_VERSION", - file=rel, - line=1, - detail=f"ruby {ver}", - provider="ruby", - version=m.group(1), - ) - ) - ruby_crypto_rx = re.compile( - r"(?m)^\s*gem\s+['\"](" - r"openssl|rbnacl|bcrypt|ed25519|jwt|jose|rsa" - r")['\"](?:\s*,\s*['\"]([~>= list[CryptoFact]: - facts = [] - for lock in _rfiles(repo, "rpms.lock.yaml"): - rel = str(lock.relative_to(repo)) - text = lock.read_text(errors="replace") - seen: set[tuple[str, str]] = set() - for m in NEVRA_CRYPTO_RX.finditer(text): - key = (m.group(1), m.group(2)) - if key in seen: - continue - seen.add(key) - facts.append( - CryptoFact( - probe_id="CRYPTO_RPM_NEVRA", - file=rel, - line=1, - detail=f"{m.group(1)} {m.group(2)}", - provider=m.group(1), - version=m.group(2), - ) - ) - return facts - - -# ---------- Environment / policy ---------- - - -def probe_env_and_policy(repo: Path) -> list[CryptoFact]: - """Scan text files for GODEBUG TLS kill-switches, crypto-policy refs, - and JVM security properties.""" - facts = [] - for f in _rfiles(repo, "*"): - if not f.is_file() or f.suffix not in TEXT_EXTS: - continue - rel = str(f.relative_to(repo)) - if _is_vendor(rel) or f.stat().st_size > MAX_FILE_SIZE: - continue - try: - text = f.read_text(errors="replace") - except OSError: - continue - for m in GODEBUG_TLS_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_GODEBUG_TLS_KILLSWITCH", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=m.group(0)[:200], - provider="go", - ) - ) - for m in GODEBUG_FIPS_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_FIPS140_GODEBUG", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=f"GODEBUG fips140={m.group(1)}", - provider="go", - extra={"fips_mode": m.group(1)}, - ) - ) - for m in GOEXPERIMENT_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_DOCKERFILE_DIRECTIVE", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=f"GOEXPERIMENT={m.group(1)}", - provider="container-build", - extra={"directive_type": "env", "key": "GOEXPERIMENT", "value": m.group(1)}, - ) - ) - for m in CRYPTO_POLICY_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_POLICY_SET", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=m.group(0)[:200], - provider="crypto-policies", - ) - ) - if f.name == "java.security" or f.suffix == ".properties": - for m in JVM_TLS_PROPS_RX.finditer(text): - facts.append( - CryptoFact( - probe_id="CRYPTO_JVM_SECURITY_PROPERTY", - file=rel, - line=text[: m.start()].count("\n") + 1, - detail=f"{m.group(1)}={m.group(2)[:150]}", - provider="jdk", - ) - ) - return facts - - -# ---------- SBOM ---------- - - -def probe_sbom(sbom_path: Path) -> list[CryptoFact]: - """Extract crypto-relevant components from a CycloneDX SBOM.""" - doc = json.loads(sbom_path.read_text()) - comps = doc.get("components") or [] - facts = [] - crypto_keywords = ( - "openssl", - "crypto-policies", - "gnutls", - "libssh", - "nss", - "bouncycastle", - "jose", - "golang.org/x/crypto", - "pycryptodome", - "cryptography", - "ring", - "rustls", - "mbedtls", - "wolfssl", - "aws-lc", - "pyopenssl", - "pynacl", - "bcrypt", - ) - for c in comps: - name = (c.get("name") or "").lower() - ver = c.get("version") or "" - if any(k in name for k in crypto_keywords): - facts.append( - CryptoFact( - probe_id="CRYPTO_SBOM_COMPONENT", - file=f"sbom:{c.get('purl') or name}", - line=0, - detail=f"{c.get('name')} {ver}"[:200], - provider=_sbom_provider(name), - version=ver, - ) - ) - return facts - - -def _sbom_provider(name: str) -> str: - """Map SBOM component name to a provider key.""" - if "openssl" in name or "libssl" in name: - return "openssl" - if "gnutls" in name: - return "gnutls" - if "nss" in name: - return "nss" - if "rustls" in name: - return "rustls" - if "ring" in name or "aws-lc" in name: - return "aws-lc-rs" - if "bouncycastle" in name: - return "bouncycastle" - if "cryptography" in name or "pyopenssl" in name: - return "python-openssl" - return name.split("/")[-1].split("-")[0] - - -# ---------- Orchestration ---------- - - -def probe_repo(repo: Path) -> list[CryptoFact]: - """Run all static probes against a repository directory.""" - facts = [] - facts.extend(probe_go(repo)) - facts.extend(probe_dockerfiles(repo)) - facts.extend(probe_node(repo)) - facts.extend(probe_jdk(repo)) - facts.extend(probe_python(repo)) - facts.extend(probe_rust(repo)) - facts.extend(probe_c(repo)) - facts.extend(probe_dotnet(repo)) - facts.extend(probe_ruby(repo)) - facts.extend(probe_rpm_locks(repo)) - facts.extend(probe_env_and_policy(repo)) - return facts diff --git a/src/traust_engine/adapters/detection_quality.py b/src/traust_engine/adapters/detection_quality.py deleted file mode 100644 index 6744333..0000000 --- a/src/traust_engine/adapters/detection_quality.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Core's detection-quality sources: the opengrep rule packs, and canary repositories. - -Core scores packs and builds the answer key (`DetectionQualityService`); this module reads rule -files and commits canary trees. Nothing here touches the network: a supplemental pack is read -only from the clone an opengrep scan already made (`opengrep.resolve_rules`). -""" - -from __future__ import annotations - -import re -import shutil -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import yaml -from pydantic import ValidationError as InvalidModel -from traust_core.v1.context import Context -from traust_core.v1.errors import ConfigError, ServiceError, ValidationError -from traust_core.v1.models.detection_quality import ( - CanaryInjection, - CanaryKind, - CanaryRepo, - CanarySource, - RuleId, - RulePack, - RuleRef, - SeededFinding, -) -from traust_core.v1.providers.base import Readiness, on_path -from traust_core.v1.providers.process import ProcessRunner - -from traust_engine.adapters.opengrep import pinned_rules_source, rules_cache_dir, rules_root -from traust_engine.assets import default_rule_pack_dir, harness_version - -GIT_TIMEOUT = 120.0 -TRAUST_PACK = "traust" -_CWE = re.compile(r"CWE-\d+", re.IGNORECASE) -# Never a command the repository's own config names (fsmonitor, hooks). -_HARDENED = ("-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null") - - -@dataclass(frozen=True, slots=True) -class Supplement: - """An enabled supplemental pack of the rule-pack allowlist: its pinned source and rule ids.""" - - name: str - source: str - rules: tuple[str, ...] - - -class OpengrepRulePacks: - """The traust pack (bundled, or `locations.opengrep_rules`) and each allowlisted supplement, - as rule ids with their languages and metadata CWEs.""" - - name = "opengrep" - - def __init__( - self, pack: Path, version: str, supplements: Sequence[Supplement], git: ProcessRunner - ) -> None: - self._pack = pack - self._version = version - self._supplements = tuple(supplements) - self._git = git - - def check(self) -> Readiness: - if self._pack.is_dir(): - return Readiness(ready=True) - return Readiness(ready=False, detail=f"rule pack {self._pack} is not a directory") - - def packs(self) -> list[RulePack]: - own = rule_refs(self._pack) - traust = RulePack(name=TRAUST_PACK, version=self._version, rules=tuple(own.values())) - return [traust, *map(self._supplement, self._supplements)] - - def _supplement(self, supplement: Supplement) -> RulePack: - try: - url, ref = pinned_rules_source(supplement.source) - except ValueError as e: - raise ConfigError(f"rule pack {supplement.name}: {e}") from e - dest = rules_cache_dir(url, ref) - if not dest.is_dir(): - raise ConfigError( - f"rule pack {supplement.name}: {supplement.source} is not fetched yet " - f"({dest}); an opengrep scan with the allowlist fetches it" - ) - self._verify(supplement.name, dest, ref) - found = rule_refs(rules_root(url, dest)) - if missing := sorted(set(supplement.rules) - set(found)): - raise ValidationError( - f"rule pack {supplement.name}@{ref[:12]}: allowlisted rules not in the pack: " - + ", ".join(missing) - ) - return RulePack( - name=supplement.name, - version=ref, - rules=tuple(found[r] for r in dict.fromkeys(supplement.rules)), - ) - - def _verify(self, name: str, dest: Path, ref: str) -> None: - """A cached pack counts only at its pin with a pristine worktree: a prior scan of a - hostile repository (or a local edit) must not change the rules under a pinned SHA.""" - try: - head = self._run_git(dest, "rev-parse", "HEAD").strip() - dirty = self._run_git(dest, "status", "--porcelain").strip() - except ServiceError as e: - raise ConfigError(f"rule pack {name}: cache {dest} unreadable: {e}") from e - if head != ref or dirty: - raise ConfigError( - f"rule pack {name}: cache {dest} fails pin/cleanliness verification " - f"(HEAD {head[:12]}, {'dirty' if dirty else 'clean'}); refusing unverifiable rules" - ) - - def _run_git(self, repo: Path, *args: str) -> str: - return self._git.run(["git", "-C", str(repo), *_HARDENED, *args]).stdout.decode() - - -def rule_refs(root: Path) -> dict[str, RuleRef]: - """Every rule under `root`, by id; opengrep test targets (`*.test.yaml`, tests/) and hidden - directories hold no rules, and a YAML file with no `rules:` list is not a rule file.""" - found: dict[str, RuleRef] = {} - for path in sorted([*root.rglob("*.yaml"), *root.rglob("*.yml")]): - parts = path.relative_to(root).parts - if ".test." in path.name or "tests" in parts or any(p.startswith(".") for p in parts): - continue - try: - doc = yaml.safe_load(path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, yaml.YAMLError) as e: - raise ValidationError(f"rule file {path}: unreadable: {e}") from e - if not (isinstance(doc, dict) and isinstance(doc.get("rules"), list)): - continue - for rule in doc["rules"]: - if not (isinstance(rule, dict) and rule.get("id")): - raise ValidationError(f"rule file {path}: a rule has no id") - cwe = (rule.get("metadata") or {}).get("cwe") or () - cwes = _CWE.findall(" ".join(map(str, [cwe] if isinstance(cwe, str) else cwe))) - found[str(rule["id"])] = RuleRef( - id=RuleId(str(rule["id"])), - languages=tuple(str(lang) for lang in rule.get("languages") or ()), - cwes=tuple(dict.fromkeys(c.upper() for c in cwes)), - ) - return found - - -# Fixed identity and date, so a canary's HEAD SHA depends only on its tree's content. -_CANARY_GIT_ENV = { - "GIT_AUTHOR_NAME": "canary-builder", - "GIT_AUTHOR_EMAIL": "canary@invalid", - "GIT_COMMITTER_NAME": "canary-builder", - "GIT_COMMITTER_EMAIL": "canary@invalid", - "GIT_AUTHOR_DATE": "2026-01-01T00:00:00 +0000", - "GIT_COMMITTER_DATE": "2026-01-01T00:00:00 +0000", - # The builder's own git config (autocrlf, signing, templates) must not reach the commit. - "GIT_CONFIG_GLOBAL": "/dev/null", - "GIT_CONFIG_NOSYSTEM": "1", -} -# Only committed content: bytecode caches would make the SHA depend on who last ran Python. -_NOT_COMMITTED = shutil.ignore_patterns("__pycache__", "*.pyc") - - -class GitCanaryMaterializer: - """Canary and seeded-fixture sources (`/{canary,seed}.yaml + tree/`), each committed - into a fresh local repository; the answer key never enters it.""" - - name = "canary-git" - - def __init__(self, sources_dir: Path, repos_dir: Path, git: ProcessRunner) -> None: - self._sources = sources_dir - self._repos = repos_dir - self._git = git - - def check(self) -> Readiness: - if not self._sources.is_dir(): - return Readiness(ready=False, detail=f"no canary sources at {self._sources}") - return on_path("git") - - def sources(self) -> list[CanarySource]: - if not self._sources.is_dir(): - raise ConfigError(f"no canary sources at {self._sources}") - return [ - canary_source(d, self._sources.name) - for d in sorted(self._sources.iterdir()) - if any((d / k.answer_key).is_file() for k in CanaryKind) - ] - - def materialize(self, source: CanarySource) -> CanaryRepo: - tree = self._sources / source.name / "tree" - if not tree.is_dir(): - raise ValidationError(f"canary {source.name}: no tree/ at {tree}") - dest = (self._repos / source.name).resolve() - if dest.exists(): - shutil.rmtree(dest) - self._repos.mkdir(parents=True, exist_ok=True) - # Links stay links: following one would commit a file from outside the source tree. - shutil.copytree(tree, dest, symlinks=True, ignore=_NOT_COMMITTED) - for args in ( - ("init", "-q", "-b", "main"), - ("add", "-A"), - # Never signed: a signature carries its own timestamp, so the SHA would differ per run. - ("-c", "commit.gpgsign=false", "commit", "-q", "-m", f"import {source.name}"), - ): - self._run_git(dest, *args) - sha = self._run_git(dest, "rev-parse", "HEAD").strip() - return CanaryRepo(repo_url=f"file://{dest}", sha=sha) - - def _run_git(self, repo: Path, *args: str) -> str: - argv = ["git", "-C", str(repo), *_HARDENED, *args] - return self._git.run(argv, env=_CANARY_GIT_ENV).stdout.decode() - - -def canary_source(directory: Path, sources_name: str) -> CanarySource: - """One `/` source directory: its answer key, and the files of tree/ that get committed.""" - keys = [k for k in CanaryKind if (directory / k.answer_key).is_file()] - if len(keys) != 1: - raise ValidationError(f"canary {directory.name}: both canary.yaml and seed.yaml — use one") - kind = keys[0] - try: - meta: Mapping[str, Any] = yaml.safe_load( - (directory / kind.answer_key).read_text(encoding="utf-8") - ) - if meta["name"] != directory.name: - raise ValidationError( - f"canary {directory.name}: {kind.answer_key} names {meta['name']!r}, " - "not its directory" - ) - expected = meta["expected"] if isinstance(meta["expected"], list) else [meta["expected"]] - injection = meta.get("injection") - tree = directory / "tree" - if not tree.is_dir(): - raise ValidationError(f"canary {directory.name}: no tree/ beside its answer key") - if any(".git" in p.relative_to(tree).parts for p in tree.rglob(".git")): - raise ValidationError( - f"canary {directory.name}: a .git entry inside tree/ would configure the git " - "that commits it" - ) - return CanarySource( - name=directory.name, - kind=kind, - recipe=f"{sources_name}/{directory.name}", - expected=tuple( - SeededFinding( - cwes=tuple(str(c) for c in e["cwes"]), - paths=tuple(e["paths"]), - severity=e["severity"], - title=e.get("title", ""), - ) - for e in expected - ), - tree=tuple( - sorted( - p.relative_to(tree).as_posix() - for p in tree.rglob("*") - if (p.is_file() or p.is_symlink()) - and "__pycache__" not in p.relative_to(tree).parts - and p.suffix != ".pyc" - ) - ), - language=meta.get("language"), - injection=CanaryInjection( - injection_class=injection["class"], tokens=tuple(injection["tokens"]) - ) - if injection - else None, - ) - except (OSError, yaml.YAMLError, KeyError, TypeError, InvalidModel) as e: - raise ValidationError(f"canary {directory.name}: unreadable {kind.answer_key}: {e}") from e - - -def register_detection_quality( - ctx: Context, - *, - pack: Path, - supplements: Sequence[Mapping[str, Any]], - canary_sources: Path, - canary_repos: Path, -) -> None: - """Set the rule-pack source and canary materialiser the detection-quality doors read through. - - `supplements` are `opengrep.load_supplemental_packs` entries (name, source, rules). - """ - git = ProcessRunner(["git"], keep_env=ctx.config.process.keep_env, timeout_seconds=GIT_TIMEOUT) - version = harness_version() if pack == default_rule_pack_dir() else f"local:{pack}" - ctx.rule_packs = OpengrepRulePacks( - pack, - version, - [Supplement(s["name"], s["source"], tuple(s["rules"])) for s in supplements], - git, - ) - ctx.canaries = GitCanaryMaterializer(canary_sources, canary_repos, git) diff --git a/src/traust_engine/adapters/fleet_fix.py b/src/traust_engine/adapters/fleet_fix.py deleted file mode 100644 index ce4ee19..0000000 --- a/src/traust_engine/adapters/fleet_fix.py +++ /dev/null @@ -1,176 +0,0 @@ -"""Core's fleet-fix applier and ast-grep golden runner: the subprocess half of traust -apply_fleet_fix.py. - -Core owns the spec, the line rewrite (`rewrite_pinned_refs`), the golden assertions and the -file_glob / max_files_changed guards (`RemediationService`); this module clones, resolves refs with -`git ls-remote`, runs ast-grep and diffs. It never commits, forks or pushes. -""" - -from __future__ import annotations - -import re -import tempfile -from pathlib import Path, PurePosixPath -from typing import Any - -from traust_core.v1.context import Context -from traust_core.v1.errors import ServiceError, ValidationError -from traust_core.v1.models.artifacts import FleetFixArtifact -from traust_core.v1.models.common import RepoPath -from traust_core.v1.models.remediation import FleetFixDiff, in_scope, rewrite_pinned_refs -from traust_core.v1.providers.clock import Clock -from traust_core.v1.providers.process import ProcessRunner - -CLONE_TIMEOUT = 600.0 -LS_REMOTE_TIMEOUT = 60.0 -GIT_TIMEOUT = 120.0 -AST_GREP_TIMEOUT = 600.0 -# URLs come from target-repository files (the spec's url_regex): hostile by definition. -# https-only plus GIT_ALLOW_PROTOCOL blocks ext::/ssh helper execution; `--` and the leading -# character checks block option injection (audit A1, a confirmed RCE class). -_HTTPS_URL = re.compile(r"https://[^\s'\"]+") -_REF = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*") -_SHA = re.compile(r"[0-9a-f]{40}") -_NETWORK_ENV = {"GIT_ALLOW_PROTOCOL": "https", "GIT_TERMINAL_PROMPT": "0"} -# Never a command the checkout's own config names (fsmonitor, hooks). -_HARDENED = ("-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null") - - -class GitFleetFixApplier: - """A fleet-fix spec applied to a fresh https clone of one commit, as a diff; also runs an - ast_grep spec over one golden fixture for core's golden tests.""" - - def __init__(self, git: ProcessRunner, ast_grep: ProcessRunner, clock: Clock) -> None: - self._git = git - self._ast_grep = ast_grep - self._clock = clock - - def apply(self, spec: FleetFixArtifact, repository_url: str, sha: str) -> FleetFixDiff: - if not _HTTPS_URL.fullmatch(repository_url): - raise ValidationError(f"fleet fix {spec.id}: {repository_url!r} is not an https URL") - if not _SHA.fullmatch(sha): - raise ValidationError(f"fleet fix {spec.id}: {sha!r} is not a 40-hex commit") - with tempfile.TemporaryDirectory(prefix="fleetfix-") as tmp: - repo = Path(tmp) / "repo" - self._run_git( - Path(tmp), - "clone", - "--quiet", - "--no-checkout", - "--", - repository_url, - str(repo), - timeout=CLONE_TIMEOUT, - ) - self._run_git(repo, "checkout", "--quiet", "--detach", sha) - guards = spec.guards - if guards.require_clean_tree is not False and self._run_git( - repo, "status", "--porcelain" - ): - raise ValidationError(f"fleet fix {spec.id}: {repository_url}@{sha} is not clean") - files = self._in_scope(spec, repo) - if spec.matcher.kind == "pinned_ref_line": - unresolved = self._rewrite(spec, repo, files) - else: - self._run_ast_grep(spec, [repo / f for f in files]) - unresolved = [] - changed = self._run_git(repo, "diff", "--name-only", "-z").split("\0") - return FleetFixDiff( - files_changed=tuple(RepoPath(f) for f in changed if f), - unresolved=tuple(unresolved), - diff=self._run_git(repo, "diff", "--no-ext-diff", "--no-textconv"), - ) - - def after(self, spec: FleetFixArtifact, file: str, before: str) -> str: - path = PurePosixPath(file) - if path.is_absolute() or ".." in path.parts: - raise ValidationError(f"fleet fix {spec.id}: golden fixture path {file!r} escapes") - with tempfile.TemporaryDirectory(prefix="fleetfix-golden-") as tmp: - target = Path(tmp, *path.parts) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(before, encoding="utf-8") - self._run_ast_grep(spec, [target]) - return target.read_text(encoding="utf-8") - - def _in_scope(self, spec: FleetFixArtifact, repo: Path) -> list[str]: - """Tracked regular files the spec's file_glob allows; links are never followed.""" - tracked = self._run_git(repo, "ls-files", "-z").split("\0") - return [ - f - for f in tracked - if f - and in_scope(spec.matcher.file_glob, f) - and (repo / f).is_file() - and not (repo / f).is_symlink() - ] - - def _rewrite(self, spec: FleetFixArtifact, repo: Path, files: list[str]) -> list[str]: - date = self._clock.now().date().isoformat() - ref = (spec.resolver.ref if spec.resolver else None) or "main" - cache: dict[str, str | None] = {} - - def resolve(url: str) -> str | None: - if url not in cache: - cache[url] = self.ls_remote(url, ref) - return cache[url] - - unresolved: list[str] = [] - for f in files: - text = (repo / f).read_text(encoding="utf-8", errors="replace") - new, missed = rewrite_pinned_refs(spec, f, text, date, resolve) - unresolved += missed - if new != text: - (repo / f).write_text(new, encoding="utf-8") - return unresolved - - def ls_remote(self, url: str, ref: str) -> str | None: - """The commit `ref` names on an https remote; None (reported unresolved by core) when the - URL or ref fails the guards or the remote doesn't answer.""" - if not (_HTTPS_URL.fullmatch(url) and _REF.fullmatch(ref)): - return None - argv = ["git", *_HARDENED, "ls-remote", "--", url, f"refs/heads/{ref}", ref] - try: - out = self._git.run( - argv, env=_NETWORK_ENV, check=False, timeout_seconds=LS_REMOTE_TIMEOUT - ) - except ServiceError: - return None - shas = (line.split("\t")[0].strip() for line in out.stdout.decode().splitlines()) - return next((s for s in shas if _SHA.fullmatch(s)), None) - - def _run_ast_grep(self, spec: FleetFixArtifact, paths: list[Path]) -> None: - if not paths: - return - m: Any = spec.matcher - argv = [ - "ast-grep", - "run", - f"--pattern={m.pattern}", - f"--rewrite={spec.rewrite.template}", - f"--lang={m.lang}", - "--update-all", - *map(str, paths), - ] - # An empty cwd: ast-grep looks for sgconfig.yml (custom languages load native libraries) - # upward from it, and the target repository must not supply one. - with tempfile.TemporaryDirectory(prefix="fleetfix-cwd-") as cwd: - out = self._ast_grep.run(argv, cwd=Path(cwd), check=False) - if out.returncode not in (0, 1): # 1: no match - tail = out.stderr.decode(errors="replace")[-400:] - raise ServiceError(f"fleet fix {spec.id}: ast-grep exited {out.returncode}: {tail}") - - def _run_git(self, repo: Path, *args: str, timeout: float = GIT_TIMEOUT) -> str: - argv = ["git", "-C", str(repo), *_HARDENED, *args] - return self._git.run(argv, env=_NETWORK_ENV, timeout_seconds=timeout).stdout.decode() - - -def register_fleet_fix(ctx: Context) -> None: - """Set the fleet-fix applier and the ast-grep golden runner (one object serves both).""" - keep = ctx.config.process.keep_env - applier = GitFleetFixApplier( - ProcessRunner(["git"], keep_env=keep, timeout_seconds=CLONE_TIMEOUT), - ProcessRunner(["ast-grep"], keep_env=keep, timeout_seconds=AST_GREP_TIMEOUT), - ctx.clock, - ) - ctx.fleet_fix_applier = applier - ctx.fleet_fix_goldens = applier diff --git a/src/traust_engine/adapters/gitleaks.py b/src/traust_engine/adapters/gitleaks.py deleted file mode 100644 index 6c5de24..0000000 --- a/src/traust_engine/adapters/gitleaks.py +++ /dev/null @@ -1,291 +0,0 @@ -"""Committed-credential candidates via gitleaks (secret pre-scan stage). - -Runs `gitleaks` against a pinned target clone — working tree by default, -full git history with --mode history — and reduces its JSON report to -secret-FREE candidates: location, rule id, gitleaks fingerprint, entropy, -and a `liveness_class` tag that routes testable credential classes to the -validate-findings credential-liveness verifier. The matched secret value -itself is NEVER written to the output: downstream consumers that need it -(the liveness probe) re-read it from the checkout at probe time. - -Inline ``gitleaks:allow`` comments in the target are IGNORED -(``--ignore-gitleaks-allow``): the target is the code under audit, and a -suppression it ships for itself is repository content, not an exemption -(docs/adversarial-content-doctrine.md in traust). Candidates on such lines -carry ``inline_allow: true`` so the judge sees the marker and weighs it. - -This is a CANDIDATE GENERATOR, never a finder or verdict of record -(docs/deterministic-inferential-mix.md: deterministic tools route, -gate, tag, or index — never conclude). A regex hit may be a test fixture, -an example placeholder, or a revoked credential — the audit/triage agent -weighs it; the liveness verifier upgrades it to evidence. - -The default ruleset is resolved via ``locations.gitleaks_config`` (injected -context), ``GITLEAKS_CONFIG`` env, or the bundled ``gitleaks-default.toml``. -Override with ``--config`` on the CLI. - -Usage: - python3 run_gitleaks.py --repo [--mode dir|history] - [--out ] [--config ] - [--timeout SECONDS] [--gitleaks BIN] - -Output defaults to -gitleaks.json in the CWD. -Exit 0 on a completed scan (with or without candidates); 1 on tool failure. -""" - -import contextlib -import json -import subprocess -import tempfile -from pathlib import Path -from typing import Any - -from traust_contracts import Locations -from traust_contracts.models import AdapterResult, Location - -from traust_engine.adapters._contract_bridge import ( - build_scan_result, - format_lines, - map_severity, - utc_now, -) -from traust_engine.assets import harness_version as engine_harness_version - -# gitleaks rule id (prefix) -> credential class the validate-findings -# liveness verifier knows how to probe read-only. Anything unmapped is -# UNTESTABLE by design — the list grows only with the probe registry. -LIVENESS_CLASSES = { - "aws-access-token": "aws", - "github-pat": "github", - "github-fine-grained-pat": "github", - "github-oauth": "github", - "github-app-token": "github", - "gitlab-pat": "gitlab", - "gitlab-ptt": "gitlab", - "gitlab-rrt": "gitlab", - "slack-bot-token": "slack", - "slack-user-token": "slack", - "traust-openshift-sha256-token": "openshift", -} - - -def resolve_gitleaks_config( - config: Path | None = None, - loc: Locations | None = None, -) -> Path: - """Config path: explicit arg → locations → env → bundled default.""" - if config is not None: - path = config - else: - from traust_engine import locations as locs - from traust_engine.assets import default_gitleaks_config - - raw = loc.gitleaks_config if loc else None - if raw: - configured = locs.gitleaks_config_path(loc) - if configured is None: - raise ValueError( - f"locations.gitleaks_config={raw!r} has no local path form " - "(remote URIs are not supported for gitleaks config)" - ) - path = configured - else: - path = default_gitleaks_config() - if not path.is_file(): - raise ValueError( - "gitleaks config required (config, locations.gitleaks_config, or bundled default)" - ) - return path - - -def harness_version(): - return engine_harness_version() - - -def repo_head(repo): - try: - return subprocess.run( - ["git", "-C", str(repo), "rev-parse", "HEAD"], - capture_output=True, - text=True, - timeout=15, - check=True, - ).stdout.strip() - except (subprocess.SubprocessError, OSError): - return None - - -def tool_version(binary): - try: - out = subprocess.run([binary, "version"], capture_output=True, text=True, timeout=15).stdout - return out.strip().splitlines()[0] if out.strip() else None - except (subprocess.SubprocessError, OSError): - return None - - -INLINE_ALLOW_MARKER = "gitleaks:allow" - - -def _has_inline_allow(finding: dict, repo: Path, path: str, mode: str) -> bool: - """True when the matched line carries the target's own suppression marker. - - gitleaks redacts only the secret, so the marker survives in ``Line`` / - ``Match``. Dir mode falls back to reading the line from the checkout; - only the boolean leaves this function, never line content. - """ - for key in ("Line", "Match"): - if INLINE_ALLOW_MARKER in (finding.get(key) or ""): - return True - start = finding.get("StartLine") - if mode != "dir" or not path or not isinstance(start, int) or start < 1: - return False - end = finding.get("EndLine") if isinstance(finding.get("EndLine"), int) else start - try: - with (repo / path).open(encoding="utf-8", errors="replace") as fh: - for lineno, line in enumerate(fh, 1): - if lineno > end: - break - if lineno >= start and INLINE_ALLOW_MARKER in line: - return True - except OSError: - return False - return False - - -def parse_report(findings: list, repo: Path, mode: str) -> list[dict]: - """Reduce gitleaks findings to secret-free candidates.""" - candidates = [] - for f in findings: - rule = f.get("RuleID") or "" - path = f.get("File") or "" - with contextlib.suppress(ValueError): - path = str(Path(path).resolve().relative_to(repo.resolve())) - cand = { - "rule_id": rule, - "description": f.get("Description") or "", - "path": path, - "start_line": f.get("StartLine"), - "end_line": f.get("EndLine"), - "entropy": f.get("Entropy"), - # commit:file:rule:line — gitleaks' stable identity; carries - # no secret material and is the dedup/correlation key - "fingerprint": f.get("Fingerprint") or "", - "liveness_class": LIVENESS_CLASSES.get(rule), - "evidence": "secret_pattern_match", - "inline_allow": _has_inline_allow(f, repo, path, mode), - } - if mode == "history": - cand["commit"] = f.get("Commit") or None - cand["commit_date"] = f.get("Date") or None - # author email routes ownership; the *message* stays out — - # commit messages are repository content (injection surface) - cand["author_email"] = f.get("Email") or None - candidates.append(cand) - candidates.sort(key=lambda c: (c["path"], c["start_line"] or 0, c["rule_id"], c["fingerprint"])) - return candidates - - -def _run_gitleaks( - repo: Path, - mode: str, - config: Path, - timeout: int, - gitleaks_bin: str, -) -> list[dict]: - """Run gitleaks and return raw finding list.""" - with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tf: - report_path = Path(tf.name) - try: - cmd = [ - gitleaks_bin, - "git" if mode == "history" else "dir", - str(repo), - "--config", - str(config), - "--report-format", - "json", - "--report-path", - str(report_path), - "--redact=100", - # The target's own gitleaks:allow comments are data under audit, - # never exemptions; parse_report flags them as inline_allow. - "--ignore-gitleaks-allow", - "--exit-code", - "0", - "--no-banner", - "--log-level", - "error", - ] - proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) - if proc.returncode != 0: - raise RuntimeError( - f"gitleaks failed (exit {proc.returncode}): {proc.stderr.strip()[:500]}" - ) - findings = json.loads(report_path.read_text(encoding="utf-8")) - return findings or [] - finally: - report_path.unlink(missing_ok=True) - - -def _describe(cand: dict) -> str: - text = cand.get("description", "") - if cand.get("inline_allow"): - note = ( - "The target marks this line gitleaks:allow; a suppression shipped " - "by the audited repo is not an exemption, judge the candidate." - ) - text = f"{text} ({note})" if text else note - return text - - -def _candidates_to_findings(candidates: list[dict]) -> list[dict[str, Any]]: - findings: list[dict[str, Any]] = [] - for cand in candidates: - loc = ( - Location( - path=cand["path"], - lines=format_lines(cand.get("start_line"), cand.get("end_line")), - ) - if Location is not None - else { - "path": cand["path"], - "lines": format_lines(cand.get("start_line"), cand.get("end_line")), - } - ) - findings.append( - { - "id": f"gitleaks/{cand['rule_id']}", - "title": cand.get("description") or cand["rule_id"], - "severity": map_severity("high"), - "locations": [loc], - "description": _describe(cand), - "category": cand.get("rule_id", ""), - } - ) - return findings - - -def scan( - repo: Path, - mode: str = "dir", - config: Path | None = None, - timeout: int = 900, - gitleaks_bin: str = "gitleaks", - *, - loc: Locations | None = None, -) -> AdapterResult: - """Run gitleaks and return typed scan result.""" - repo = repo.resolve() - config = resolve_gitleaks_config(config, loc) - if not repo.is_dir(): - raise ValueError(f"not a directory: {repo}") - - raw = _run_gitleaks(repo, mode, config, timeout, gitleaks_bin) - candidates = parse_report(raw, repo, mode) - return build_scan_result( - str(repo), - "gitleaks", - _candidates_to_findings(candidates), - scanned_at=utc_now(), - scanner_version=tool_version(gitleaks_bin) or "", - ) diff --git a/src/traust_engine/adapters/govulncheck.py b/src/traust_engine/adapters/govulncheck.py deleted file mode 100644 index ed4f2fe..0000000 --- a/src/traust_engine/adapters/govulncheck.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Dependency-CVE reachability candidates via govulncheck (reachability stage 1). - -Runs `govulncheck -json ./...` against a pinned target clone and reduces -its finding stream to one candidate per OSV advisory, classified by the -most specific evidence govulncheck produced: - - symbol_reachable — a vulnerable function is on a static - call path from the target's own code - package_imported_not_observed — the vulnerable package is imported, - but no call to a vulnerable symbol - was observed by static analysis - module_required_not_observed — the module is required by go.mod only - -This is a CANDIDATE GENERATOR, never a finder or verdict of record -(docs/deterministic-inferential-mix.md: deterministic tools route, -gate, tag, or index — never conclude). `not_observed` means exactly that: -static analysis did not observe a call path. It does not mean unreachable -— reflection, plugins, and config-driven dispatch are invisible to it — -so downstream consumers may lower a prior on this signal but never -auto-dismiss a finding. Design and staging: -docs/reachability.md. - -Usage: - python3 run_govulncheck.py --repo [--out ] - [--timeout SECONDS] [--govulncheck BIN] - -Output defaults to -govulncheck.json in the CWD. -Exit 0 on a completed scan (with or without candidates); 1 on tool failure. -""" - -import json -import subprocess -from pathlib import Path -from typing import Any - -from traust_contracts.models import AdapterResult, Location - -from traust_engine.adapters._contract_bridge import ( - build_scan_result, - map_severity, - utc_now, -) -from traust_engine.assets import harness_version as engine_harness_version - -REACH_ORDER = [ - "module_required_not_observed", - "package_imported_not_observed", - "symbol_reachable", -] - - -def harness_version(): - return engine_harness_version() - - -def repo_head(repo): - try: - return subprocess.run( - ["git", "-C", str(repo), "rev-parse", "HEAD"], - capture_output=True, - text=True, - timeout=15, - check=True, - ).stdout.strip() - except (subprocess.SubprocessError, OSError): - return None - - -def classify(finding): - """Map one govulncheck finding record to a reachability level.""" - trace = finding.get("trace") or [] - frame = trace[0] if trace else {} - if frame.get("function"): - return "symbol_reachable" - if frame.get("package"): - return "package_imported_not_observed" - return "module_required_not_observed" - - -def trace_symbols(trace): - """Render a govulncheck trace as compact call-path strings, caller first.""" - out = [] - for frame in reversed(trace): - fn = frame.get("function") - if not fn: - continue - recv = frame.get("receiver") or "" - name = f"{recv}.{fn}" if recv else fn - pkg = frame.get("package") or "" - pos = frame.get("position") or {} - loc = f" ({pos['filename']}:{pos['line']})" if pos.get("filename") else "" - out.append(f"{pkg}.{name}{loc}" if pkg else f"{name}{loc}") - return out - - -def iter_records(text): - """Yield JSON objects from a govulncheck -json stream (a concatenation - of pretty-printed objects, not JSONL).""" - decoder = json.JSONDecoder() - pos, end = 0, len(text) - while pos < end: - while pos < end and text[pos].isspace(): - pos += 1 - if pos >= end: - break - try: - rec, pos = decoder.raw_decode(text, pos) - except json.JSONDecodeError: - break - yield rec - - -def parse_stream(text): - """Reduce the govulncheck -json stream to per-OSV candidates.""" - osv_meta = {} # osv id -> advisory metadata - candidates = {} # osv id -> candidate dict (most specific evidence wins) - tool = {} - - for rec in iter_records(text): - if not isinstance(rec, dict): - continue - if "config" in rec: - cfg = rec["config"] - tool = { - "name": cfg.get("scanner_name", "govulncheck"), - "version": cfg.get("scanner_version"), - "go_version": cfg.get("go_version"), - "db": cfg.get("db"), - "db_last_modified": cfg.get("db_last_modified"), - "scan_level": cfg.get("scan_level"), - } - elif "osv" in rec: - osv = rec["osv"] - osv_meta[osv.get("id")] = { - "aliases": osv.get("aliases") or [], - "summary": osv.get("summary") or osv.get("details", "")[:200], - } - elif "finding" in rec: - f = rec["finding"] - osv_id = f.get("osv") - if not osv_id: - continue - level = classify(f) - trace = f.get("trace") or [] - frame = trace[0] if trace else {} - cand = candidates.setdefault( - osv_id, - { - "osv_id": osv_id, - "reachability": level, - "module": frame.get("module"), - "found_version": frame.get("version"), - "fixed_version": f.get("fixed_version"), - "packages": [], - "example_trace": [], - }, - ) - if REACH_ORDER.index(level) > REACH_ORDER.index(cand["reachability"]): - cand["reachability"] = level - pkg = frame.get("package") - if pkg and pkg not in cand["packages"]: - cand["packages"].append(pkg) - if level == "symbol_reachable": - symbols = trace_symbols(trace) - # keep the shortest observed call path as the example - if symbols and ( - not cand["example_trace"] or len(symbols) < len(cand["example_trace"]) - ): - cand["example_trace"] = symbols - - for osv_id, cand in candidates.items(): - meta = osv_meta.get(osv_id, {}) - cand["aliases"] = meta.get("aliases", []) - cand["summary"] = meta.get("summary", "") - - order = {lvl: i for i, lvl in enumerate(reversed(REACH_ORDER))} - ranked = sorted(candidates.values(), key=lambda c: (order[c["reachability"]], c["osv_id"])) - return tool, ranked - - -def _reachability_to_severity(reachability: str) -> str: - return { - "symbol_reachable": "high", - "package_imported_not_observed": "medium", - "module_required_not_observed": "low", - }.get(reachability, "medium") - - -def _candidates_to_findings(candidates: list[dict]) -> list[dict[str, Any]]: - findings: list[dict[str, Any]] = [] - for cand in candidates: - locs = [] - for site in cand.get("example_trace", []): - locs.append( - Location(path=site, description="call path") - if Location is not None - else {"path": site, "description": "call path"} - ) - for pkg in cand.get("packages", []): - locs.append( - Location(path=pkg, description="imported package") - if Location is not None - else {"path": pkg, "description": "imported package"} - ) - desc = ( - f"module={cand.get('module')} " - f"version={cand.get('found_version')} " - f"reachability={cand['reachability']}" - ) - findings.append( - { - "id": cand["osv_id"], - "title": cand.get("summary") or cand["osv_id"], - "severity": map_severity(_reachability_to_severity(cand["reachability"])), - "locations": locs, - "description": desc, - "category": cand["reachability"], - } - ) - return findings - - -def command(govulncheck_bin: str = "govulncheck", db_dir: Path | None = None) -> list[str]: - """The argv a govulncheck scan runs, from inside the checkout. - - *db_dir*, when given, adds ``-db file://`` so the scan reads the feeds-cache mount - explicitly rather than querying vuln.go.dev live (scanner-currency-plan.md, "Target state"). - """ - cmd = [govulncheck_bin, "-json"] - if db_dir is not None: - cmd += ["-db", f"file://{db_dir}"] - return [*cmd, "./..."] - - -def _run_govulncheck( - repo: Path, - timeout: int, - govulncheck_bin: str, - *, - profile_map: dict | None = None, - freeze: bool = True, - db_dir: Path | None = None, -) -> tuple[int, str, str, dict, list]: - """Execute govulncheck under safe_exec and parse its JSON stream.""" - from traust_engine._util import safe_exec - from traust_engine.toolchain import govulncheck_db_present - from traust_engine.toolchain._errors import ScannerDatabaseUnavailable - - if db_dir is not None and not govulncheck_db_present(db_dir): - raise ScannerDatabaseUnavailable(f"govulncheck: no vulnerability database at {db_dir}") - - cmd = command(govulncheck_bin, db_dir=db_dir) - extra_env = {} - if freeze: - from traust_engine.toolchain import freeze_env - - extra_env = freeze_env("govulncheck", db_dir=str(db_dir) if db_dir else None) - - rc, out, err = safe_exec.run( - cmd, - "go-scan", - timeout=timeout, - cwd=str(repo), - extra_env=extra_env if extra_env else None, - profile_map=profile_map, - ) - tool, candidates = parse_stream(out) - return rc, out, err, tool, candidates - - -def scan( - repo: Path, - timeout: int = 600, - govulncheck_bin: str = "govulncheck", - *, - freeze: bool = True, - profile_map: dict | None = None, - db_dir: Path | None = None, -) -> AdapterResult: - """Run govulncheck and return typed scan result. - - *freeze* (default True) injects env vars to prevent DB auto-update - (govulncheck uses ``GOVULNDB`` for a local DB mirror). - *profile_map* must come from ``AdaptersOps.safe_exec_profile_map()`` when - running under a loaded engine; omit only in unit tests. - *db_dir*, when given, points the scan at a feeds-cache mount (``-db file://``) and - raises :class:`ScannerDatabaseUnavailable` before running when that mount has no usable - database — never a silent zero-finding scan. - """ - repo = repo.resolve() - if not (repo / "go.mod").exists() and not list(repo.glob("*/go.mod")): - raise ValueError(f"no go.mod under {repo}") - - rc, _out, err, tool, candidates = _run_govulncheck( - repo, - timeout, - govulncheck_bin, - profile_map=profile_map, - freeze=freeze, - db_dir=db_dir, - ) - if not tool: - raise RuntimeError(f"govulncheck produced no scan output (exit {rc}): {err.strip()[:500]}") - - return build_scan_result( - str(repo), - "govulncheck", - _candidates_to_findings(candidates), - scanned_at=utc_now(), - scanner_version=tool.get("version", ""), - ) diff --git a/src/traust_engine/adapters/grype.py b/src/traust_engine/adapters/grype.py deleted file mode 100644 index 4790812..0000000 --- a/src/traust_engine/adapters/grype.py +++ /dev/null @@ -1,160 +0,0 @@ -"""Container/SBOM vulnerability candidates via grype (SCA stage). - -Runs ``grype`` against an SBOM (CycloneDX/SPDX JSON) or a directory and -reduces its JSON output to normalised candidates — one per (CVE, package) -pair — with severity, fixed-in version, and location metadata. - -This is a CANDIDATE GENERATOR, never a finder or verdict of record. -A matched CVE is context for the audit/triage agent to weigh — the -vulnerable code path may be unreachable or the package unused. - -Usage: - python3 -m traust_engine.adapters.grype --target [--out FILE] -""" - -from __future__ import annotations - -import json -import os -import subprocess -from pathlib import Path -from typing import Any - -from traust_contracts.models import AdapterResult - -from traust_engine.adapters._contract_bridge import ( - build_scan_result, - map_severity, - utc_now, -) - - -def _tool_version(binary: str) -> str | None: - try: - out = subprocess.run( - [binary, "--version"], capture_output=True, text=True, timeout=15 - ).stdout - import re - - m = re.search(r"(\d+\.\d+\.\d+)", out) - return m.group(1) if m else None - except (subprocess.SubprocessError, OSError): - return None - - -def command(target: str, grype_bin: str = "grype", output_format: str = "json") -> list[str]: - """The argv a grype scan of `target` runs.""" - return [grype_bin, target, "-o", output_format] - - -def _run_grype( - target: str, - timeout: int, - grype_bin: str, - *, - freeze: bool = True, - output_format: str = "json", - db_dir: Path | None = None, -) -> dict[str, Any]: - from traust_engine.toolchain import freeze_env, grype_db_present - from traust_engine.toolchain._errors import ScannerDatabaseUnavailable - - if db_dir is not None and not grype_db_present(db_dir): - raise ScannerDatabaseUnavailable(f"grype: no vulnerability database at {db_dir}") - - env = dict(os.environ) - if freeze: - env.update(freeze_env("grype", db_dir=str(db_dir) if db_dir else None)) - - cmd = command(target, grype_bin, output_format) - proc = subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=timeout, - env=env, - ) - if proc.returncode not in (0, 1): - raise RuntimeError(f"grype failed (exit {proc.returncode}): {proc.stderr.strip()[:500]}") - return json.loads(proc.stdout) - - -def _parse_matches(doc: dict[str, Any]) -> list[dict[str, Any]]: - findings: list[dict[str, Any]] = [] - for match in doc.get("matches", []): - vuln = match.get("vulnerability", {}) - artifact = match.get("artifact", {}) - related = match.get("relatedVulnerabilities", []) - - vuln_id = vuln.get("id", "UNKNOWN") - severity_hint = vuln.get("severity", "unknown").lower() - description = vuln.get("description", "") - if not description and related: - description = related[0].get("description", "") - - fixed_in = ( - ", ".join(fix.get("version", "?") for fix in vuln.get("fix", {}).get("versions", [])) - or None - ) - - pkg_name = artifact.get("name", "") - pkg_version = artifact.get("version", "") - pkg_type = artifact.get("type", "") - locations = artifact.get("locations", []) - - loc_dicts = [{"path": loc.get("path", "")} for loc in locations] - if not loc_dicts: - loc_dicts.append({"path": f"{pkg_type}:{pkg_name}"}) - - cwes = [] - for url in vuln.get("urls", []): - if "cwe.mitre.org" in url: - import re - - m = re.search(r"CWE-(\d+)", url) - if m: - cwes.append(f"CWE-{m.group(1)}") - - remediation = f"Upgrade {pkg_name} to {fixed_in}" if fixed_in else "" - - findings.append( - { - "id": f"{vuln_id}:{pkg_name}@{pkg_version}", - "title": f"{vuln_id} in {pkg_name} {pkg_version}", - "severity": map_severity(severity_hint), - "cwes": cwes, - "locations": loc_dicts, - "description": description[:2000], - "remediation": remediation, - "category": "dependency_vulnerability", - } - ) - return findings - - -def scan( - target: str | Path, - timeout: int = 600, - grype_bin: str = "grype", - *, - freeze: bool = True, - db_dir: Path | None = None, -) -> AdapterResult: - """Run grype and return typed scan result. - - *target* can be an SBOM path (``sbom:``) or a directory. - *freeze* (default True) disables grype's DB auto-update. - *db_dir*, when given, points grype at a feeds-cache mount instead of its native cache - (``GRYPE_DB_CACHE_DIR``) and raises :class:`ScannerDatabaseUnavailable` before running when - that mount has no usable database — never a silent zero-finding scan. - """ - target_str = str(target) - doc = _run_grype(target_str, timeout, grype_bin, freeze=freeze, db_dir=db_dir) - - return build_scan_result( - target_str, - "grype", - _parse_matches(doc), - scanned_at=utc_now(), - scanner_version=_tool_version(grype_bin) or "", - ) diff --git a/src/traust_engine/adapters/joern.py b/src/traust_engine/adapters/joern.py deleted file mode 100644 index f22b0fb..0000000 --- a/src/traust_engine/adapters/joern.py +++ /dev/null @@ -1,499 +0,0 @@ -"""Joern call-site reachability facts for JVM and C/C++ code — the -analogue of govulncheck's symbol tier in /impact-analysis. - -Builds a code property graph of the repo's first-party sources -(joern-parse: javasrc2cpg for Java, c2cpg for C/C++) and reports every -call site whose resolved callee matches the vulnerable symbols -(Java `class#method`, C function names) or package prefixes under -investigation. Facts only — the wrapper never -classifies; /impact-analysis judges the output. - -EVIDENCE ASYMMETRY (by design, stated in every artifact): a found call -path is PROMOTING evidence (first-party code demonstrably invokes the -vulnerable API — cite the site). An absent path is NEVER proof of -safety: Java dependency injection (Spring/CDI), reflection, and -MethodHandles hide edges from any static analyzer, Joern included. The -tier can upgrade a classification; it must never downgrade one. - -Usage: - traust adapters joern --repo DIR --out FILE \ - [--language java|c] \ - [--symbols 'com.foo.Bar#method' | 'EVP_EncryptUpdate' ...] \ - [--packages com.foo,com.bar] [--module group:artifact] \ - [--timeout 900] [--max-sites 200] - -C symbol matching is exact-name (c2cpg methodFullName is the bare -function name; prefix matching on short C names would over-match). - -Exit 0 always writes the artifact: status "ran", "skipped: " -(joern absent, no Java sources, no patterns derivable), or -"error: " (CPG build/query failure) — mirroring the -deterministic_steps convention so callers record honesty, not crashes. -""" - -from __future__ import annotations - -import contextlib -import json -import os -import re -import shutil -import subprocess -import tempfile -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from traust_contracts.models import AdapterResult, Location - -from traust_engine.adapters._contract_bridge import ( - build_scan_result, - map_severity, - utc_now, -) - -# Query script executed inside joern. TSV out (SITE/COUNT rows) so the -# Scala side needs no JSON library; Python parses and structures. -QUERY_SC = r""" -@main def exec(cpgFile: String, symbolsFile: String, outFile: String) = { - importCpg(cpgFile) - val patterns = scala.io.Source.fromFile(symbolsFile).getLines().filter(_.nonEmpty).toList - val sb = new StringBuilder - for (p <- patterns) { - val exact = p.startsWith("=") - val contains = p.startsWith("~") - val q = if (exact || contains) p.drop(1) else p - // Exact mode compares the CALLEE NAME, not the whole methodFullName. - // c2cpg emits ".ConsumeFieldMessage:" - // for C++ member calls and never qualifies with "::", so comparing the - // whole string made every C++ member query fail by construction, while - // still matching plain C where the name IS the whole prefix. Measured - // 2026-08-14 on protobuf text_format.cc: 3325 calls, 0 matches. - def calleeName(fn: String): String = { - val base = fn.split(":").headOption.getOrElse(fn) - base.split('.').lastOption.getOrElse(base) - } - val calls = (if (exact) cpg.call.filter(c => calleeName(c.methodFullName) == q) - else if (contains) cpg.call.filter(_.methodFullName.contains(q)) - else cpg.call.filter(_.methodFullName.startsWith(q))).l - for (c <- calls.take(500)) { - val file = c.location.filename - val line = Option(c.lineNumber).flatten.map(_.toString).getOrElse("-1") - sb.append(List("SITE", p, file, line, c.method.fullName, c.methodFullName) - .mkString("\t")).append("\n") - } - sb.append(List("COUNT", p, calls.size.toString).mkString("\t")).append("\n") - } - val w = new java.io.PrintWriter(outFile) - w.write(sb.toString) - w.close() -} -""" - -_TEST_PATH = re.compile(r"(^|/)(src/test|tests?|testdata|it)(/|$)", re.I) - -# joern-parse frontend, passed EXPLICITLY per --language. Never rely on -# auto-detection: for a Java tree it reports `language: JAVA` and invokes -# **jimple2cpg** (the BYTECODE frontend) whenever compiled artifacts are -# present, not javasrc2cpg. -_FRONTEND = {"java": "JAVASRC", "c": "NEWC"} - -# Coordinate tokens that carry no package-identity information — they -# appear in half of all Maven artifact ids and would match any import. -_GENERIC_TOKENS = frozenset( - { - "core", - "api", - "apis", - "starter", - "common", - "commons", - "util", - "utils", - "client", - "server", - "impl", - "lib", - "libs", - "java", - "javax", - "base", - "bom", - "parent", - "all", - "main", - "runtime", - "support", - "extension", - "extensions", - "plugin", - "plugins", - "module", - "modules", - "spi", - "sdk", - "com", - "org", - "io", - "net", - "ch", - "me", - } -) -_IMPORT_RX = re.compile(r"^\s*import\s+(?:static\s+)?([\w.]+)\s*;", re.M) - - -def derive_packages_from_imports(repo: Path, module: str, max_files: int = 4000) -> list[dict]: - """Package prefixes the repo ACTUALLY imports that plausibly belong to - `module`, each with the file that evidences it.""" - group, _, artifact = module.partition(":") - toks = { - t.lower() - for t in re.split(r"[.\-_]", group + "-" + artifact) - if len(t) > 2 and t.lower() not in _GENERIC_TOKENS - } - if not toks: - return [] - hits: dict[str, str] = {} - score: dict[str, int] = {} - scanned = 0 - for f in repo.rglob("*.java"): - if scanned >= max_files: - break - scanned += 1 - try: - text = f.read_text(encoding="utf-8", errors="replace") - except OSError: - continue - for m in _IMPORT_RX.finditer(text): - fqn = m.group(1) - parts = fqn.split(".") - low = [p.lower() for p in parts] - idxs = [i for i, p in enumerate(low) if p in toks] - if not idxs: - continue - pkg = ".".join(parts[: max(idxs) + 1]) - if pkg.count(".") < 1: - continue - hits.setdefault(pkg, str(f.relative_to(repo))) - score[pkg] = max( - score.get(pkg, 0), - len({p for p in low[: max(idxs) + 1]} & toks), - ) - if not hits: - return [] - best = max(score.values()) - out = [] - for pkg, ev in sorted(hits.items()): - if score.get(pkg, 0) < best: - continue - if any(o != pkg and o.startswith(pkg + ".") and score.get(o, 0) >= best for o in hits): - continue - out.append( - { - "pattern": pkg + ".", - "kind": "package", - "source": f"import-derived:{pkg}", - "evidence_file": ev, - "tokens_matched": score.get(pkg, 0), - } - ) - return out[:12] - - -@dataclass -class PatternInput: - symbols: list[str] | None = None - packages: str = "" - module: str | None = None - language: str = "java" - repo: Path | None = None - - -def patterns_from(inp: PatternInput) -> list[dict]: - """Derive methodFullName patterns. Symbol patterns are strongest - (the vulnerable method/function itself); package prefixes weaker - (any call into the library's API); a bare Maven group is weakest - (group id only approximates the package root — recorded as - heuristic). A leading '=' marks an exact-match pattern for the - query layer (used for C function names).""" - pats: list[dict] = [] - for s in inp.symbols or []: - if inp.language == "c": - # c2cpg methodFullName is the bare function name — exact - # match only; prefix matching on short C names over-matches - pats.append({"pattern": "=" + s, "kind": "symbol", "source": s}) - elif "#" in s: - cls, _, meth = s.partition("#") - if "." in cls: - pats.append({"pattern": f"{cls}.{meth}:", "kind": "symbol", "source": s}) - else: - pats.append( - { - "pattern": f"~.{cls}.{meth}:", - "kind": "symbol", - "source": s, - "match": "contains (class not fully qualified)", - } - ) - else: - pats.append( - {"pattern": s if s.endswith(".") else s + ".", "kind": "package", "source": s} - ) - for p in inp.packages.split(",") if inp.packages else []: - p = p.strip() - if p: - pats.append( - {"pattern": p if p.endswith(".") else p + ".", "kind": "package", "source": p} - ) - if not pats and inp.module: - if inp.language == "java": - with contextlib.suppress(OSError, AttributeError, TypeError): - pats.extend(derive_packages_from_imports(inp.repo, inp.module)) - if not pats: - group = inp.module.partition(":")[0] - if "." in group: - pats.append( - { - "pattern": group + ".", - "kind": "group-heuristic", - "source": inp.module, - "match": "group id only approximates the " - "package root; no matching import " - "found in the tree", - } - ) - # de-dup, deterministic order - seen, out = set(), [] - for p in pats: - if p["pattern"] not in seen: - seen.add(p["pattern"]) - out.append(p) - return out - - -def emit(out: Path, doc: dict) -> None: - doc.setdefault( - "soundness", - { - "found_path": "promoting evidence — first-party code invokes the " - "matched API; cite the call site", - "absent_path": "NEVER proof of safety — Java DI/reflection hide " - "edges from static analysis; this tier upgrades " - "classifications, never downgrades them", - "line_numbers": "APPROXIMATE — javasrc2cpg line attribution " - "drifts on some constructs; the " - "file + caller method are authoritative. Anyone " - "citing a site must verify the exact line by " - "reading the file (citation-gate discipline).", - }, - ) - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(doc, indent=1) + "\n", encoding="utf-8") - print(f"joern-reachability: {doc['status']} → {out}") - - -def _run_joern_scan( - repo: Path, - *, - symbols: list[str] | None, - timeout: int, - max_sites: int = 200, - language: str = "java", - packages: str = "", - module: str = "", -) -> dict[str, Any]: - """Run joern reachability scan and return artifact dict.""" - base: dict[str, Any] = { - "tool": "joern", - "repo": str(repo), - "language": language, - "module": module or None, - } - - joern = shutil.which("joern") - parse = shutil.which("joern-parse") - if not (joern and parse): - return {**base, "status": "skipped: joern not on PATH"} - - exts = ("*.java",) if language == "java" else ("*.c", "*.h", "*.cc", "*.cpp", "*.hpp") - if not any(f for pat in exts for f in repo.rglob(pat)): - return {**base, "status": f"skipped: no {language} sources"} - - pats = patterns_from( - PatternInput( - symbols=symbols or [], - packages=packages, - module=module or None, - language=language, - repo=repo, - ) - ) - if not pats: - return {**base, "status": "skipped: no symbol/package patterns derivable"} - - try: - ver = subprocess.run( - [joern, "--version"], - capture_output=True, - text=True, - timeout=120, - stdin=subprocess.DEVNULL, - ) - m = re.search(r"\b(\d+\.\d+\.\d+)\b", (ver.stdout or "") + (ver.stderr or "")) - base["version"] = m.group(1) if m else "unknown" - except (subprocess.SubprocessError, OSError): - base["version"] = "unknown" - - with tempfile.TemporaryDirectory(prefix="joern-reach-") as td: - tdp = Path(td) - cpg = tdp / "cpg.bin" - frontend = _FRONTEND[language] - base["frontend"] = frontend - built = subprocess.run( - [parse, str(repo), "--language", frontend, "--output", str(cpg)], - capture_output=True, - text=True, - timeout=timeout, - stdin=subprocess.DEVNULL, - env={**os.environ, "JAVA_OPTS": os.environ.get("JOERN_JAVA_OPTS", "-Xmx8g")}, - ) - if built.returncode != 0 or not cpg.is_file(): - raw = (built.stderr or built.stdout).strip() - return { - **base, - "status": "error: joern-parse failed", - "detail_head": raw[:400], - "detail": raw[-400:], - } - - script = tdp / "query.sc" - script.write_text(QUERY_SC, encoding="utf-8") - symfile = tdp / "patterns.txt" - symfile.write_text("\n".join(p["pattern"] for p in pats) + "\n", encoding="utf-8") - tsv = tdp / "sites.tsv" - q = subprocess.run( - [ - joern, - "--script", - str(script), - "--param", - f"cpgFile={cpg}", - "--param", - f"symbolsFile={symfile}", - "--param", - f"outFile={tsv}", - ], - capture_output=True, - text=True, - timeout=timeout, - stdin=subprocess.DEVNULL, - env={**os.environ, "JAVA_OPTS": os.environ.get("JOERN_JAVA_OPTS", "-Xmx8g")}, - ) - if q.returncode != 0 or not tsv.is_file(): - return { - **base, - "status": "error: joern query failed", - "detail": (q.stderr or q.stdout).strip()[-400:], - } - - by_pattern: dict[str, dict] = { - p["pattern"]: {**p, "call_sites": [], "total_calls": 0} for p in pats - } - repo_prefix = str(repo.resolve()) + os.sep - for row in tsv.read_text(encoding="utf-8").splitlines(): - parts = row.split("\t") - if parts[0] == "COUNT" and len(parts) == 3: - if parts[1] in by_pattern: - by_pattern[parts[1]]["total_calls"] = int(parts[2]) - elif parts[0] == "SITE" and len(parts) == 6: - _, pat, fname, line, caller, callee = parts - if pat not in by_pattern: - continue - rel = fname[len(repo_prefix) :] if fname.startswith(repo_prefix) else fname - sites = by_pattern[pat]["call_sites"] - if len(sites) < max_sites: - sites.append( - { - "file": rel, - "line": int(line), - "caller": caller, - "callee": callee, - "test_path": bool(_TEST_PATH.search(rel)), - } - ) - - targets = [] - for p in pats: - entry = by_pattern[p["pattern"]] - entry["call_sites"].sort(key=lambda s: (s["file"], s["line"])) - nontest = [s for s in entry["call_sites"] if not s["test_path"]] - capped = entry["total_calls"] > len(entry["call_sites"]) - targets.append( - { - "source": entry["source"], - "kind": entry["kind"], - "pattern": entry["pattern"], - "referenced": bool(nontest), - "referenced_test_only": bool(entry["call_sites"]) and not nontest, - "total_calls": entry["total_calls"], - "call_sites": entry["call_sites"], - **({"sites_capped_at": max_sites} if capped else {}), - } - ) - - return {**base, "status": "ran", "targets": targets} - - -def _targets_to_findings(targets: list[dict]) -> list[dict[str, Any]]: - findings: list[dict[str, Any]] = [] - for target in targets: - for site in target.get("call_sites", []): - loc = ( - Location( - path=site["file"], - lines=str(site.get("line", "")), - description=site.get("caller", ""), - ) - if Location is not None - else { - "path": site["file"], - "lines": str(site.get("line", "")), - "description": site.get("caller", ""), - } - ) - findings.append( - { - "id": f"joern/{target['pattern']}:{site['file']}:{site['line']}", - "title": f"Call to {target['source']} at {site['file']}", - "severity": map_severity("high" if target.get("referenced") else "medium"), - "locations": [loc], - "description": ( - f"callee={site.get('callee')} " - f"pattern={target['pattern']} " - f"kind={target.get('kind')}" - ), - "category": target.get("kind", ""), - } - ) - return findings - - -def scan( - repo: Path, - symbols: list[str] | None = None, - timeout: int = 1200, -) -> AdapterResult: - """Run joern reachability scan and return typed scan result.""" - repo = repo.resolve() - doc = _run_joern_scan(repo, symbols=symbols, timeout=timeout) - if doc.get("status") != "ran": - raise RuntimeError(doc.get("status", "joern scan failed")) - - return build_scan_result( - str(repo), - "joern", - _targets_to_findings(doc.get("targets", [])), - scanned_at=utc_now(), - scanner_version=doc.get("version", ""), - ) diff --git a/src/traust_engine/adapters/ops.py b/src/traust_engine/adapters/ops.py index 5170554..262de96 100644 --- a/src/traust_engine/adapters/ops.py +++ b/src/traust_engine/adapters/ops.py @@ -1,7 +1,10 @@ """``HarnessEngine.adapters`` — scanner config injection, bound to context. Adapters stay stateless; this namespace only surfaces typed config from the -injected context (rule-pack allowlist, safe_exec profiles, workspace root). +injected context (rule-pack allowlist, rule pack) for the rule-lane stage. + +PENDING(decision: sweep stages) — kept for traust; +deleted when traust retires or ports it. """ from __future__ import annotations diff --git a/src/traust_engine/adapters/osv.py b/src/traust_engine/adapters/osv.py deleted file mode 100644 index de7a2fe..0000000 --- a/src/traust_engine/adapters/osv.py +++ /dev/null @@ -1,877 +0,0 @@ -"""Dependency-CVE candidates from the OSV API (multi-ecosystem SCA stage). - -Collects every declared package and version from a target checkout's -lockfiles and manifests (the engine's ``portfolio.parsers``, the same -parsers the portfolio graph uses, plus ``go.mod``), looks them up through -the shared OSV API client (``traust_engine.osv``), and reduces the result to -one candidate per (OSV advisory, package), tagged with the files the package -was declared in. No scanner binary and no local vulnerability database are -involved (owner decision 2026-10-20). - -This is a CANDIDATE GENERATOR, never a finder or verdict of record -(docs/deterministic-inferential-mix.md: deterministic tools route, -gate, tag, or index — never conclude). Every candidate is merely -*declared*, evidence class ``dependency_declared``: the package may be -unused, dev-only, or the vulnerable code path unexercised. - -**Coverage is reported, never assumed.** A lookup that fails, is -rate-limited or comes back partial is counted in the provenance block -(``unreachable`` / ``quota-deferred`` / ``error``) and the scan is -``degraded``. When the API cannot be used at all (one probe before the scan, -or not a single package answered) the scan is ``failed``: no dependency -results, and ``scan()`` raises. Neither is ever "no advisories". Versions -taken from a manifest rather than a lockfile are declared floors, tagged -``version_basis: manifest-floor``. - -The OSV API is the default. The osv-scanner binary path (``run_scanner`` / -``parse_output``) serves air-gapped deployments, which run osv-scanner -offline against a local OSV database the adopter provides; the harness does -not install either (owner decision 2026-10-20). The engine reads no -configuration: the caller chooses the path. -""" - -import json -import os -import re -import subprocess -from collections import Counter, defaultdict -from pathlib import Path -from typing import Any - -from traust_contracts.models import AdapterResult, Location - -from traust_engine.adapters._contract_bridge import ( - build_scan_result, - map_severity, - utc_now, -) -from traust_engine.assets import harness_version as engine_harness_version -from traust_engine.osv import OsvClient, Query, max_score -from traust_engine.portfolio import parsers as manifest_parsers -from traust_engine.portfolio.graph import parse_gomod - - -def harness_version(): - return engine_harness_version() - - -def repo_head(repo): - try: - return subprocess.run( - ["git", "-C", str(repo), "rev-parse", "HEAD"], - capture_output=True, - text=True, - timeout=15, - check=True, - ).stdout.strip() - except (subprocess.SubprocessError, OSError): - return None - - -def fixed_version(vuln: dict, pkg_name: str, ecosystem: str): - """Best-effort fixed version for pkg from the OSV affected ranges.""" - for aff in vuln.get("affected") or []: - p = aff.get("package") or {} - if p.get("name") != pkg_name: - continue - if ecosystem and p.get("ecosystem") and p["ecosystem"] != ecosystem: - continue - for rng in aff.get("ranges") or []: - for ev in rng.get("events") or []: - if ev.get("fixed"): - return ev["fixed"] - return None - - -def parse_output(doc: dict, repo: Path) -> list[dict]: - """Reduce osv-scanner JSON to per-(advisory, package) candidates.""" - candidates: dict[tuple, dict] = {} - for result in doc.get("results") or []: - src = result.get("source") or {} - src_path = src.get("path") or "" - try: - src_rel = str(Path(src_path).resolve().relative_to(repo.resolve())) - except ValueError: - src_rel = src_path - for pkg_entry in result.get("packages") or []: - pkg = pkg_entry.get("package") or {} - name = pkg.get("name") or "" - eco = pkg.get("ecosystem") or "" - ver = pkg.get("version") or "" - sev_by_group = {} - for grp in pkg_entry.get("groups") or []: - for vid in grp.get("ids") or []: - sev_by_group[vid] = grp.get("max_severity") - for vuln in pkg_entry.get("vulnerabilities") or []: - vid = vuln.get("id") - if not vid: - continue - key = (vid, eco, name) - cand = candidates.setdefault( - key, - { - "osv_id": vid, - "aliases": vuln.get("aliases") or [], - "summary": (vuln.get("summary") or (vuln.get("details") or "")[:200]), - "ecosystem": eco, - "package": name, - "found_version": ver, - "fixed_version": fixed_version(vuln, name, eco), - "max_severity": sev_by_group.get(vid), - "evidence": "dependency_declared", - "sources": [], - }, - ) - if src_rel and src_rel not in cand["sources"]: - cand["sources"].append(src_rel) - return sorted(candidates.values(), key=lambda c: (c["ecosystem"], c["package"], c["osv_id"])) - - -def run_scanner(binary: str, repo: Path, timeout: int, *, freeze: bool = False): - """Invoke osv-scanner, tolerating both the v2 (`scan source`) and v1 - CLI shapes. Returns (stdout, stderr, returncode) of the first shape - that produces output, or the last attempt's result. - - When *freeze* is True, ``--offline`` is appended so osv-scanner uses - only its local DB cache and never phones home mid-scan. - """ - from traust_engine.toolchain import freeze_flags - - extra = freeze_flags("osv-scanner") if freeze else [] - shapes = [ - [binary, "scan", "source", "--recursive", "--format", "json", *extra, str(repo)], - [binary, "--recursive", "--format", "json", *extra, str(repo)], - ] - last = None - for cmd in shapes: - proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) - last = (proc, " ".join(cmd)) - if proc.stdout.strip().startswith("{"): - return proc, " ".join(cmd) - return last - - -def _cvss_to_severity(max_severity: str | None) -> str: - if not max_severity: - return "medium" - try: # a numeric CVSS score ("9.8"): band it (CVSS v3 qualitative scale) - score = float(max_severity) - except (TypeError, ValueError): - score = None - if score is not None: - if score >= 9.0: - return "critical" - if score >= 7.0: - return "high" - if score >= 4.0: - return "medium" - return "low" if score > 0 else "medium" - sev = str(max_severity).upper() - if sev in ("CRITICAL",): - return "critical" - if sev in ("HIGH", "HIGHEST"): - return "high" - if sev in ("MEDIUM", "MODERATE"): - return "medium" - if sev in ("LOW",): - return "low" - return "medium" - - -def _candidates_to_findings(candidates: list[dict]) -> list[dict[str, Any]]: - findings: list[dict[str, Any]] = [] - for cand in candidates: - locs = [] - for src in cand.get("sources", []): - loc = Location(path=src) if Location is not None else {"path": src} - locs.append(loc) - desc_parts = [ - f"{cand.get('package')}@{cand.get('found_version')}", - f"ecosystem={cand.get('ecosystem')}", - ] - if cand.get("fixed_version"): - desc_parts.append(f"fixed in {cand['fixed_version']}") - findings.append( - { - "id": cand["osv_id"], - "title": cand.get("summary") or cand["osv_id"], - "severity": map_severity(_cvss_to_severity(cand.get("max_severity"))), - "locations": locs, - "description": "; ".join(desc_parts), - "source_findings": cand.get("aliases", []), - "category": cand.get("ecosystem", ""), - } - ) - return findings - - -# --------------------------------------------------------------------------- -# API-based adapter (the default; owner decision 2026-10-20) -# --------------------------------------------------------------------------- - -# Parser ecosystem -> OSV ecosystem name. docker and helm have no OSV -# ecosystem; they stay graph-only, as in the fleet sweep. -OSV_ECOSYSTEM = { - "npm": "npm", - "pypi": "PyPI", - "maven": "Maven", - "cargo": "crates.io", - "ruby": "RubyGems", - "nuget": "NuGet", - "go": "Go", - "actions": "GitHub Actions", -} - -# Files whose versions are resolved, not declared ranges. A directory that -# holds one of these for an ecosystem is read from it alone; that -# ecosystem's manifests in the same directory are skipped. -LOCKFILES = { - "package-lock.json", - "npm-shrinkwrap.json", - "pnpm-lock.yaml", - "yarn.lock", - "poetry.lock", - "Pipfile.lock", - "uv.lock", - "gradle.lockfile", - "Cargo.lock", - "Gemfile.lock", - "packages.lock.json", - "go.mod", -} - -SKIP_DIRS = {".git", "node_modules"} - -_EXACT_VERSION = re.compile(r"^v?[0-9][0-9A-Za-z.+_~-]*$") -_GO_DIRECTIVE = re.compile(r"^go\s+(\d+\.\d+(?:\.\d+)?)\s*$", re.M) -_GO_TOOLCHAIN = re.compile(r"^toolchain\s+go(\d+\.\d+(?:\.\d+)?)\S*\s*$", re.M) - - -def _requirements_all_pinned(text: str) -> bool: - lines = [ - ln.split("#", 1)[0].split(";", 1)[0].strip() - for ln in text.splitlines() - if ln.strip() and not ln.strip().startswith(("#", "-")) - ] - lines = [ln for ln in lines if ln] - return bool(lines) and all("==" in ln for ln in lines) - - -# A bare requirement (``requests``, ``flask[async]``) carries no version. The -# shared parser drops those lines, so they are counted here: they are -# declarations the lookup cannot check, never silently absent. -_BARE_REQUIREMENT = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?$") - - -def _bare_requirements(text: str) -> list[str]: - names = [] - for raw in text.splitlines(): - line = raw.split("#", 1)[0].split(";", 1)[0].strip() - m = _BARE_REQUIREMENT.match(line) if line and not line.startswith("-") else None - if m: - names.append(re.sub(r"[-_.]+", "-", m.group(1)).lower()) - return names - - -# A requirements file that pins every line can still list only the direct -# dependencies (``feast[minimal]==0.62.0`` pulls in dozens more). Only a -# compiled file (pip-compile, uv pip compile: ``# via`` annotations or -# ``--hash=`` lines) is known to carry the whole tree. -_COMPILED_REQUIREMENTS = re.compile( - r"^\s*#\s+via\b|--hash=|^#.*\b(?:pip-compile|uv pip compile)\b", re.M -) - - -def discover_manifests(repo: Path) -> list[tuple[str, str, Any]]: - """(repo-relative path, ecosystem, parser) for every manifest in the tree. - - Walks the checkout without running git in it (an audited repository's - git config is untrusted), skipping ``.git`` and ``node_modules``. The - parser is ``"gomod"`` for go.mod files. - """ - found: list[tuple[str, str, Any]] = [] - for root, dirs, files in os.walk(repo): - dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS) - for name in sorted(files): - rel = str((Path(root) / name).relative_to(repo)) - if name == "go.mod": - found.append((rel, "go", "gomod")) - continue - hit = manifest_parsers.parser_for_path(rel) - if hit and hit[0] in OSV_ECOSYSTEM: - found.append((rel, hit[0], hit[1])) - return found - - -_GO_REPLACE = re.compile( - r"^(?P\S+)(?:\s+(?Pv\S+))?\s*=>\s*(?P\S+)(?:\s+(?Pv\S+))?\s*$" -) - - -def gomod_replaces(text: str) -> dict[tuple[str, str | None], tuple[str, str | None]]: - """``replace`` directives: (old path, old version or None) -> (new, new version). - - A new path starting ``./``, ``../`` or ``/`` is a local directory (version - None): the module is built from the checkout itself, so it is not a - published package to look up. The shared ``parse_gomod`` ignores - ``replace``, hence this reader. - """ - out: dict[tuple[str, str | None], tuple[str, str | None]] = {} - in_block = False - for raw in text.splitlines(): - line = raw.split("//", 1)[0].strip() - if line.startswith("replace ("): - in_block = True - continue - if in_block and line == ")": - in_block = False - continue - if in_block: - candidate = line - elif line.startswith("replace "): - candidate = line[len("replace ") :] - else: - continue - m = _GO_REPLACE.match(candidate) - if m: - out[(m.group("old"), m.group("oldv"))] = (m.group("new"), m.group("newv")) - return out - - -def _is_local(path: str) -> bool: - return path.startswith(("./", "../", "/")) - - -_POM_DEP = re.compile(r"(.*?)", re.S) - - -def _tag(block: str, tag: str) -> str: - m = re.search(rf"<{tag}>\s*([^<]*?)\s*", block) - return m.group(1) if m else "" - - -def declared_scopes(filename: str, text: str) -> dict[str, str]: - """Names a manifest declares ONLY as test or dev dependencies. - - ``test``: Maven ``test`` (in ```` or - ````). ``dev``: npm ``devDependencies``, Cargo - ``[dev-dependencies]``, Poetry dev-dependencies and non-main groups. - A name also declared for runtime is not listed. The shared parsers merge - these sections, so scope is read here (owner decision 2026-10-21: tag - test and dev dependencies, never skip them). - """ - runtime: set[str] = set() - scoped: dict[str, str] = {} - try: - if filename == "package.json": - doc = json.loads(text) - runtime = {k.lower() for k in (doc.get("dependencies") or {})} - scoped = {k.lower(): "dev" for k in (doc.get("devDependencies") or {})} - elif filename == "pom.xml": - for block in _POM_DEP.findall(text): - name = f"{_tag(block, 'groupId')}:{_tag(block, 'artifactId')}".lower() - if _tag(block, "scope") == "test": - scoped[name] = "test" - else: - runtime.add(name) - elif filename in ("Cargo.toml", "pyproject.toml"): - import tomllib - - doc = tomllib.loads(text) - if filename == "Cargo.toml": - runtime = {k.lower() for k in doc.get("dependencies") or {}} - scoped = {k.lower(): "dev" for k in doc.get("dev-dependencies") or {}} - else: - poetry = (doc.get("tool") or {}).get("poetry") or {} - runtime = { - re.sub(r"[-_.]+", "-", k).lower() for k in poetry.get("dependencies") or {} - } - dev = dict(poetry.get("dev-dependencies") or {}) - for gname, group in (poetry.get("group") or {}).items(): - if gname != "main": - dev.update((group or {}).get("dependencies") or {}) - scoped = {re.sub(r"[-_.]+", "-", k).lower(): "dev" for k in dev} - except (ValueError, TypeError, AttributeError): - return {} - return {k: v for k, v in scoped.items() if k not in runtime} - - -def _scope_tag(scopes: set[str] | None) -> str | None: - """``test`` / ``dev`` when every path to a package is test or dev, else None.""" - if not scopes or "runtime" in scopes: - return None - return "test" if scopes == {"test"} else "dev" - - -def collect_packages(repo: Path) -> tuple[dict[tuple[str, str, str], dict], dict]: - """Every (OSV ecosystem, package, version) declared in the tree. - - Returns the packages (each with its ``sources`` and ``version_basis``) - and a stats block: manifests read, files that failed to parse, and - declarations skipped for lack of an exact version. - """ - manifests = discover_manifests(repo) - locked_dirs = { - (str(Path(rel).parent), eco) for rel, eco, _ in manifests if Path(rel).name in LOCKFILES - } - packages: dict[tuple[str, str, str], dict] = {} - stats = { - "manifests": 0, - "unparsed": [], - "unpinned_skipped": 0, - "superseded_by_lockfile": 0, - # manifests with no lockfile beside them: their direct declarations - # are queried at the declared floor, and the transitive tree is - # either resolved through deps.dev (resolve_transitive) or reported - # as a coverage gap (owner decisions 2026-10-20 option c, 2026-10-21) - "unresolved_manifests": [], - # per unresolved manifest: the direct declarations that were added - # (eco, name, version), and the declared names that carry no - # version, for resolve_transitive - "unresolved_direct": {}, - "unversioned_direct": {}, - # (manifest, (eco, name, version)) -> runtime / test / dev, so a - # resolved tree inherits its direct dependency's scope - "direct_scope": {}, - "unversioned_scope": {}, - # go.mod requires replaced with a directory in the checkout: the - # project's own modules, not published packages - "go_replaced_local": 0, - "go_replaced_module": 0, - } - - def add(eco: str, name: str, version: str, src: str, basis: str, scope: str = "runtime"): - version = version.strip() - if not name or not version or not _EXACT_VERSION.match(version) or "${" in version: - stats["unpinned_skipped"] += 1 - return None - if eco == "Go": - version = version.removeprefix("v") - entry = packages.setdefault((eco, name, version), {"sources": [], "version_basis": basis}) - if src not in entry["sources"]: - entry["sources"].append(src) - if basis == "locked": # a lockfile anywhere outranks a manifest floor - entry["version_basis"] = "locked" - elif entry["version_basis"] == "resolved": - entry["version_basis"] = basis # declared outranks resolved - entry.setdefault("scopes", set()).add(scope) - stats["direct_scope"][(src, (eco, name, version))] = scope - return (eco, name, version) - - for rel, eco, parser in manifests: - name = Path(rel).name - if name not in LOCKFILES and (str(Path(rel).parent), eco) in locked_dirs: - stats["superseded_by_lockfile"] += 1 - continue - try: - text = (repo / rel).read_text(encoding="utf-8", errors="replace") - except OSError: - stats["unparsed"].append(rel) - continue - stats["manifests"] += 1 - osv_eco = OSV_ECOSYSTEM[eco] - try: - if parser == "gomod": - module, deps = parse_gomod(text) - replaces = gomod_replaces(text) - for path, version, _indirect in deps: - if path == module: - continue - new = replaces.get((path, version)) or replaces.get((path, None)) - if new and (_is_local(new[0]) or not new[1]): - stats["go_replaced_local"] += 1 # built from the checkout - continue - if new: # another module is what gets built - stats["go_replaced_module"] += 1 - path, version = new - add(osv_eco, path, version, rel, "locked") - m = _GO_TOOLCHAIN.search(text) or _GO_DIRECTIVE.search(text) - if m: - ver = m.group(1) - ver = ver if ver.count(".") == 2 else ver + ".0" - add(osv_eco, "stdlib", ver, rel, "go-directive") - continue - _, deps = parser(text) - except Exception: # a malformed file is reported, never fatal - stats["unparsed"].append(rel) - continue - requirements = "requirements" in name.lower() - pinned_requirements = requirements and _requirements_all_pinned(text) - # a workflow's action refs are exactly what runs: nothing to resolve - exact = name in LOCKFILES or pinned_requirements or eco == "actions" - basis = "locked" if exact else "manifest-floor" - whole_tree = exact and not (pinned_requirements and not _COMPILED_REQUIREMENTS.search(text)) - scopes = declared_scopes(name, text) - - def scope_of(dep_name: str, scopes=scopes, osv_eco=osv_eco) -> str: - key = ( - dep_name.lower() if osv_eco != "PyPI" else re.sub(r"[-_.]+", "-", dep_name).lower() - ) - return scopes.get(key, "runtime") - - added = [ - add(osv_eco, dep_name, version or "", rel, basis, scope_of(dep_name)) - for dep_name, version, _i in deps - ] - for (dep_name, _v, _i), k in zip(deps, added, strict=True): - if k is None and dep_name: - stats["unversioned_scope"][(rel, osv_eco, dep_name)] = scope_of(dep_name) - bare = _bare_requirements(text) if requirements else [] - stats["unpinned_skipped"] += len(bare) - if not whole_tree: - stats["unresolved_manifests"].append( - {"path": rel, "ecosystem": osv_eco, "declarations": len(deps) + len(bare)} - ) - stats["unresolved_direct"][rel] = [k for k in added if k] - stats["unversioned_direct"][rel] = [ - (osv_eco, n) for (n, _v, _i), k in zip(deps, added, strict=True) if k is None and n - ] + [(osv_eco, n) for n in bare] - return packages, stats - - -def resolve_transitive(packages: dict[tuple[str, str, str], dict], stats: dict, resolver) -> dict: - """Add the transitive tree of every unresolved manifest through deps.dev. - - Each direct declaration of a manifest without a lockfile is resolved on - its own; the graph's packages join the lookup with ``version_basis: - resolved`` and ``resolved_via`` (the direct dependencies that brought - them in). A manifest moves to ``resolved_manifests`` only when every - declaration resolved; otherwise it stays unresolved with the reason. - Returns the ``resolution`` provenance block. - """ - from traust_engine.depsdev import ATTRIBUTION, LICENSE, SYSTEMS - - direct: dict[str, list] = stats.get("unresolved_direct") or {} - unversioned: dict[str, list] = stats.get("unversioned_direct") or {} - # A declared name overrides a resolved version of it where only one - # version can be installed. pip installs one per environment, and each - # requirements file is its own environment, so the rule is per file. - # Maven takes the version the project declares or manages, and parent - # poms manage it for every module, so the rule is repo-wide. npm and - # Cargo may install several versions side by side: no rule. - maven_declared = {n.lower() for (e, n, _v), ent in packages.items() if e == "Maven"} - pypi_declared: dict[str, set[str]] = {} - for (e, n, _v), ent in packages.items(): - if e == "PyPI": - for src in ent["sources"]: - pypi_declared.setdefault(src, set()).add(re.sub(r"[-_.]+", "-", n).lower()) - block: dict[str, Any] = { - "source": "deps.dev", - "base_url": resolver.base_url, - "resolved_at": utc_now(), - "license": LICENSE, - "attribution": ATTRIBUTION, - } - wanted = sorted({k for keys in direct.values() for k in keys if k[0] in SYSTEMS}) - bare = sorted({k for names in unversioned.values() for k in names if k[0] in SYSTEMS}) - probe_key = wanted[0] if wanted else ((*bare[0], "0") if bare else None) - failure = resolver.probe(probe_key) if probe_key else None - defaults: dict = {} - if failure: - results, counts = {}, Counter({failure[0]: len(wanted) + len(bare)}) - block.update(status="failed", failure={"status": failure[0], "detail": failure[1]}) - else: - # a declaration with no version: what an install gets today - defaults = resolver.defaults_many(bare) if bare else {} - for (eco, name), res in defaults.items(): - if not res.ok: - continue - version = res.nodes[0][1] - entry = packages.setdefault( - (eco, name, version), {"sources": [], "version_basis": "resolved-default"} - ) - for m, names in unversioned.items(): - if (eco, name) in names and m not in entry["sources"]: - entry["sources"].append(m) - scope = stats.get("unversioned_scope", {}).get((m, eco, name), "runtime") - entry.setdefault("scopes", set()).add(scope) - stats.setdefault("direct_scope", {})[(m, (eco, name, version))] = scope - todo = sorted( - set(wanted) | {(e, n, r.nodes[0][1]) for (e, n), r in defaults.items() if r.ok} - ) - results, counts = resolver.resolve_many(todo) - block["direct_dependencies"] = len(wanted) + len(bare) - block["defaulted"] = sum(1 for r in defaults.values() if r.ok) - added = 0 - for key, res in results.items(): - if not res.ok: - continue - via = f"{key[1]}@{key[2]}" - srcs = [m for m, keys in direct.items() if key in keys] - d = defaults.get((key[0], key[1])) - if d is not None and d.ok and d.nodes[0][1] == key[2]: # resolved at the default - srcs += [ - m for m, names in unversioned.items() if (key[0], key[1]) in names and m not in srcs - ] - for name, version, _relation in res.nodes: - node_srcs = srcs - if key[0] == "Maven" and name.lower() in maven_declared: - node_srcs = [] - elif key[0] == "PyPI": - norm = re.sub(r"[-_.]+", "-", name).lower() - node_srcs = [m for m in srcs if norm not in pypi_declared.get(m, set())] - if not node_srcs: - block["overridden_by_declaration"] = block.get("overridden_by_declaration", 0) + 1 - continue - entry = packages.get((key[0], name, version)) - if entry is None: - entry = packages[(key[0], name, version)] = { - "sources": [], - "version_basis": "resolved", - "resolved_via": [], - } - added += 1 - if entry["version_basis"] != "resolved": - continue # declared or locked: the declaration already covers it - for src in node_srcs: - if src not in entry["sources"]: - entry["sources"].append(src) - scope = (stats.get("direct_scope") or {}).get((src, key), "runtime") - entry.setdefault("scopes", set()).add(scope) - if via not in entry["resolved_via"]: - entry["resolved_via"].append(via) - still, resolved = [], [] - for m in stats["unresolved_manifests"]: - keys = list(direct.get(m["path"]) or []) - reasons = Counter() - if m["ecosystem"] not in SYSTEMS: - reasons[f"{m['ecosystem']} not resolvable through deps.dev"] += 1 - for eco, name in unversioned.get(m["path"]) or []: - d = defaults.get((eco, name)) - if d is not None and d.ok: - keys.append((eco, name, d.nodes[0][1])) - else: - reasons[f"no version, deps.dev default {d.status if d else 'not resolved'}"] += 1 - unaccounted = m["declarations"] - len(keys) - sum(reasons.values()) - if unaccounted > 0: - reasons["declarations without an exact version or floor"] += unaccounted - for k in keys: - res = results.get(k) - st = res.status if res else (failure[0] if failure else "not resolved") - if st != "ok": - reasons[f"deps.dev {st}"] += 1 - if reasons: - still.append({**m, "reason": dict(reasons)}) - else: - resolved.append({**m, "via": "deps.dev"}) - stats["unresolved_manifests"], stats["resolved_manifests"] = still, resolved - block.update( - resolved=counts.get("ok", 0), - not_found=counts.get("not-found", 0), - issues={k: v for k, v in sorted(counts.items()) if k not in ("ok", "not-found")}, - cache_hits=sum(1 for r in results.values() if r.cached), - packages_added=added, - manifests_resolved=len(resolved), - manifests_still_unresolved=len(still), - ) - block.setdefault("status", "degraded" if block["issues"] else "complete") - return block - - -def _severity_of(vuln: dict) -> str | None: - """A numeric v3 score when the record has a vector, else its declared word.""" - score = max_score(vuln) - if score is not None: - return f"{score:.1f}" - word = (vuln.get("database_specific") or {}).get("severity") - return str(word) if word else None - - -def lookup( - packages: dict[tuple[str, str, str], dict], client: OsvClient -) -> tuple[list[dict], dict]: - """Query OSV for every package; returns candidates and lookup provenance.""" - keys = sorted(packages) - outcome = client.query_batch([Query(*k) for k in keys]) - matched: dict[str, list[tuple[str, str, str]]] = defaultdict(list) - unanswered = 0 - for key, ids in zip(keys, outcome.ids, strict=True): - if ids is None: - unanswered += 1 - continue - for vid in ids: - matched[vid].append(key) - details, detail_issues = client.get_vulns(matched) - # Some databases (Go's, PyPA's) publish no severity; their GHSA or CVE - # alias usually does. Borrow the highest alias score, as osv-scanner's - # grouping did, and only fetch the aliases that are needed. Many CVE - # ids have no OSV record (404): that only leaves the advisory unscored. - needed = { - alias - for vid, vuln in details.items() - if _severity_of(vuln) is None - for alias in vuln.get("aliases") or [] - if alias.startswith(("GHSA-", "CVE-")) and alias not in details - } - alias_details, alias_issues = client.get_vulns(needed) if needed else ({}, Counter()) - known = {**alias_details, **details} - - def severity(vid: str, vuln: dict) -> tuple[str | None, str]: - """(value, basis). Malicious packages carry no CVSS: they are critical.""" - if vid.startswith("MAL-"): - return "CRITICAL", "malicious-package" - own = max_score(vuln) - if own is not None: - return f"{own:.1f}", "cvss-v3" - borrowed = [ - s - for a in vuln.get("aliases") or [] - if a in known - for s in [max_score(known[a])] - if s is not None - ] - if borrowed: - return f"{max(borrowed):.1f}", "alias-cvss-v3" - word = (vuln.get("database_specific") or {}).get("severity") - if word: - return str(word), "declared" - alias_words = [ - str((known[a].get("database_specific") or {}).get("severity")) - for a in vuln.get("aliases") or [] - if a in known and (known[a].get("database_specific") or {}).get("severity") - ] - if alias_words: - return alias_words[0], "alias-declared" - # no score, vector or severity anywhere: banded medium by the - # harness rule (dependency-watch), and labelled so it is never - # mistaken for a scored medium - return None, "unscored" - - candidates = [] - for vid in sorted(matched): - vuln = details.get(vid) or {} - sev, basis = severity(vid, vuln) if vuln else (None, "details-missing") - if vid.startswith("MAL-"): - sev, basis = "CRITICAL", "malicious-package" - for eco, name, version in matched[vid]: - entry = packages[(eco, name, version)] - candidates.append( - { - "osv_id": vid, - "aliases": vuln.get("aliases") or [], - # Red Hat records list their CVEs here, not in aliases - "upstream": vuln.get("upstream") or [], - "summary": vuln.get("summary") or (vuln.get("details") or "")[:200] or None, - "ecosystem": eco, - "package": name, - "found_version": version, - "fixed_version": fixed_version(vuln, name, eco) if vuln else None, - "max_severity": sev, - "severity_basis": basis, - "evidence": "dependency_declared", - "version_basis": entry["version_basis"], - "sources": list(entry["sources"]), - **( - {"resolved_via": list(entry["resolved_via"])} - if entry.get("resolved_via") - else {} - ), - **( - {"dependency_scope": tag} - if (tag := _scope_tag(entry.get("scopes"))) - else {} - ), - "details_fetched": bool(vuln), - } - ) - candidates.sort(key=lambda c: (c["ecosystem"], c["package"], c["osv_id"])) - # a missing alias only costs a severity, never an advisory, so it is - # reported but does not degrade the scan's coverage - issues = Counter(outcome.issues) + Counter(detail_issues) - provenance = { - "source": "osv-api", - "base_url": client.base_url, - "queried_at": utc_now(), - "packages_queried": len(keys), - "packages_by_ecosystem": dict(sorted(Counter(k[0] for k in keys).items())), - "packages_unanswered": unanswered, - "advisory_details_missing": sum(detail_issues.values()), - "alias_records_unavailable": sum(alias_issues.values()), - "lookup_issues": dict(sorted(issues.items())), - "status": "degraded" if issues else "complete", - } - if keys and unanswered == len(keys): - provenance["status"] = "failed" - return candidates, provenance - - -def _failed_lookup(keys: list, client: OsvClient, status: str, detail: str) -> dict: - """Provenance for a scan that never reached the API: every package unanswered.""" - return { - "source": "osv-api", - "base_url": client.base_url, - "queried_at": utc_now(), - "packages_queried": len(keys), - "packages_by_ecosystem": dict(sorted(Counter(k[0] for k in keys).items())), - "packages_unanswered": len(keys), - "advisory_details_missing": 0, - "alias_records_unavailable": 0, - "lookup_issues": {status: len(keys)}, - "status": "failed", - "failure": {"status": status, "detail": detail}, - } - - -def scan_api(repo: Path, client: OsvClient | None = None, resolver=None) -> dict: - """Collect packages and look them up. Returns the candidates document body. - - With a ``resolver`` (``traust_engine.depsdev.DepsDevClient``), manifests - without a lockfile have their transitive tree resolved first. - """ - repo = repo.resolve() - if not repo.is_dir(): - raise ValueError(f"not a directory: {repo}") - packages, stats = collect_packages(repo) - resolution = resolve_transitive(packages, stats, resolver) if resolver else None - client = client or OsvClient() - keys = sorted(packages) - failure = client.probe(Query(*keys[0])) if keys else None - if failure: - candidates, provenance = [], _failed_lookup(keys, client, *failure) - else: - candidates, provenance = lookup(packages, client) - provenance.update( - transitive_coverage="partial" if stats["unresolved_manifests"] else "complete", - unresolved_manifests=stats["unresolved_manifests"], - resolved_manifests=stats.get("resolved_manifests") or [], - manifests_read=stats["manifests"], - manifests_unparsed=stats["unparsed"], - declarations_without_exact_version=stats["unpinned_skipped"], - go_requires_replaced_locally=stats.get("go_replaced_local", 0), - go_requires_replaced_by_module=stats.get("go_replaced_module", 0), - manifests_superseded_by_lockfile=stats["superseded_by_lockfile"], - ) - if resolution is not None: - provenance["resolution"] = resolution - if stats["unparsed"] and provenance["status"] == "complete": - provenance["status"] = "degraded" - return {"provenance": provenance, "candidates": candidates} - - -def scan(repo: Path, timeout: int = 600, *, client: OsvClient | None = None) -> AdapterResult: - """Look up the checkout's declared packages through the OSV API. - - Raises ``RuntimeError`` when the API could not be used at all. - """ - doc = scan_api(repo, client or OsvClient(timeout=timeout)) - prov = doc["provenance"] - if prov["status"] == "failed": - detail = (prov.get("failure") or {}).get("detail") or "no package was answered" - raise RuntimeError(f"OSV API unavailable at {prov['base_url']}: {detail}") - return build_scan_result( - str(Path(repo).resolve()), - "osv-api", - _candidates_to_findings(doc["candidates"]), - scanned_at=doc["provenance"]["queried_at"], - scanner_version="osv-api", - ) - - -def _tool_version(binary: str): - try: - out = subprocess.run( - [binary, "--version"], capture_output=True, text=True, timeout=15 - ).stdout - return out.strip().splitlines()[0] if out.strip() else None - except (subprocess.SubprocessError, OSError): - return None diff --git a/src/traust_engine/adapters/osv_sbom.py b/src/traust_engine/adapters/osv_sbom.py deleted file mode 100644 index 12a3c7d..0000000 --- a/src/traust_engine/adapters/osv_sbom.py +++ /dev/null @@ -1,401 +0,0 @@ -"""Container-image language packages through OSV (secure-container-audit). - -Reads a **syft JSON** SBOM of an image and looks its *language* packages -(npm, PyPI, Maven, Go modules including the Go standard library, Cargo, -RubyGems, NuGet) up through the shared OSV client, reusing the dependency -pre-scan's ``lookup``. OS packages (rpm, deb, apk…) are never sent: grype, -with the distribution's own security data, owns them (owner decision -2026-10-20, option b). - -**A language package installed by an OS package is skipped too.** In a Red -Hat image most Python and npm packages come from RPMs (``python3-urllib3`` -puts urllib3 under ``/usr/lib/python3.9``). Red Hat backports fixes into -those RPMs without changing the upstream version, so an upstream lookup -would report CVEs the RPM already fixes; grype's Red Hat data covers them. -syft records that ownership as ``ownership-by-file-overlap`` relationships, -which CycloneDX output drops, hence the syft JSON input. On -``ubi9/python-311`` 216 of 218 language packages were RPM-installed -(measured 2026-10-20). - -``mark_grype_duplicates`` tags each candidate that grype also reported for -the same package (same purl identity) and the same advisory (by OSV id, -alias or upstream id), so a report adds only what OSV found beyond grype. -``grype_os_installed_matches`` lists grype's own matches on OS-installed -language packages, which grype reports even from syft JSON. - -Air-gapped deployments run osv-scanner offline instead: ``write_cyclonedx`` -writes the filtered packages as a CycloneDX file osv-scanner reads with -``-L`` (osv-scanner itself cannot see ownership), and ``parse_offline`` -maps its result back to the image paths. -""" - -from __future__ import annotations - -import json -import re -import subprocess -from collections import Counter -from pathlib import Path -from typing import Any -from urllib.parse import unquote - -from traust_engine.adapters.osv import _failed_lookup, fixed_version, lookup -from traust_engine.osv import OsvClient, Query - -# purl type -> OSV ecosystem -PURL_ECOSYSTEM = { - "npm": "npm", - "pypi": "PyPI", - "maven": "Maven", - "golang": "Go", - "cargo": "crates.io", - "gem": "RubyGems", - "nuget": "NuGet", -} -OS_PACKAGE_TYPES = frozenset({"rpm", "deb", "apk", "alpm", "portage"}) -OWNERSHIP = "ownership-by-file-overlap" -_VERSIONED = re.compile(r"^v?[0-9]") - - -class SbomFormatError(ValueError): - """The input is not a syft JSON SBOM.""" - - -def parse_purl(purl: str) -> tuple[str, str, str] | None: - """(OSV ecosystem, package name, version) for a supported language purl.""" - if not purl or not purl.startswith("pkg:"): - return None - body = purl[4:].split("?", 1)[0].split("#", 1)[0] - ptype, _, rest = body.partition("/") - eco = PURL_ECOSYSTEM.get(ptype.lower()) - if not eco: - return None - path, _, version = rest.rpartition("@") if "@" in rest else (rest, "", "") - parts = [unquote(p) for p in path.split("/") if p] - if not parts: - return None - version = unquote(version) - if eco == "Maven": - if len(parts) < 2: - return None - name = f"{parts[-2]}:{parts[-1]}" - elif eco == "PyPI": - name = re.sub(r"[-_.]+", "-", parts[-1]).lower() - else: - name = "/".join(parts) - if eco == "Go": - version = version.removeprefix("v") - if name == "stdlib": - version = version.removeprefix("go") - return eco, name, version - - -def _require_syft(doc: object, source: str) -> dict: - """Only syft JSON output qualifies: both arrays must be present.""" - if not isinstance(doc, dict) or "artifacts" not in doc or "artifactRelationships" not in doc: - raise SbomFormatError( - f"{source} is not a syft JSON SBOM (syft -o syft-json): CycloneDX and SPDX " - "drop the file-ownership relationships the scan needs" - ) - return doc - - -def load_syft(path: Path) -> dict: - return _require_syft(json.loads(Path(path).read_text(encoding="utf-8")), str(path)) - - -def collect_sbom_packages(doc: dict) -> tuple[dict[tuple[str, str, str], dict], dict]: - """Language packages to look up, and what was skipped and why.""" - artifacts = {a.get("id"): a for a in doc.get("artifacts") or [] if isinstance(a, dict)} - owned = { - r.get("child") - for r in doc.get("artifactRelationships") or [] - if r.get("type") == OWNERSHIP - and (artifacts.get(r.get("parent")) or {}).get("type") in OS_PACKAGE_TYPES - } - packages: dict[tuple[str, str, str], dict] = {} - stats: dict[str, Any] = { - "artifacts": len(artifacts), - "os_packages": 0, - "language_packages": 0, - "skipped_os_installed": 0, - "skipped_os_installed_by_type": Counter(), - "skipped_unversioned": 0, - "unsupported_types": Counter(), - } - for aid, art in artifacts.items(): - atype = art.get("type") or "" - if atype in OS_PACKAGE_TYPES: - stats["os_packages"] += 1 - continue - parsed = parse_purl(art.get("purl") or "") - if parsed is None: - stats["unsupported_types"][atype or "unknown"] += 1 - continue - stats["language_packages"] += 1 - if aid in owned: - stats["skipped_os_installed"] += 1 - stats["skipped_os_installed_by_type"][atype] += 1 - continue - eco, name, version = parsed - if not _VERSIONED.match(version): # UNKNOWN, (devel): nothing to look up - stats["skipped_unversioned"] += 1 - continue - entry = packages.setdefault((eco, name, version), {"sources": [], "version_basis": "sbom"}) - for loc in art.get("locations") or []: - p = (loc or {}).get("path") - if p and p not in entry["sources"]: - entry["sources"].append(p) - stats["skipped_os_installed_by_type"] = dict(stats["skipped_os_installed_by_type"]) - stats["unsupported_types"] = dict(stats["unsupported_types"]) - return packages, stats - - -def _with_stats(provenance: dict, stats: dict, doc: dict) -> dict: - provenance.update( - input="syft-json", - image=((doc.get("source") or {}).get("metadata") or {}).get("userInput") - or (doc.get("source") or {}).get("name"), - sbom_artifacts=stats["artifacts"], - os_packages_left_to_grype=stats["os_packages"], - language_packages=stats["language_packages"], - skipped_os_installed=stats["skipped_os_installed"], - skipped_os_installed_by_type=stats["skipped_os_installed_by_type"], - skipped_unversioned=stats["skipped_unversioned"], - unsupported_types=stats["unsupported_types"], - ) - return provenance - - -def scan_sbom(sbom: Path | dict, client: OsvClient | None = None) -> dict: - """Look an image's non-OS-installed language packages up through the OSV API.""" - doc = ( - _require_syft(sbom, "the dict passed to scan_sbom") - if isinstance(sbom, dict) - else load_syft(sbom) - ) - packages, stats = collect_sbom_packages(doc) - client = client or OsvClient() - keys = sorted(packages) - failure = client.probe(Query(*keys[0])) if keys else None - if failure: - candidates, provenance = [], _failed_lookup(keys, client, *failure) - else: - candidates, provenance = lookup(packages, client) - return {"provenance": _with_stats(provenance, stats, doc), "candidates": candidates} - - -def scan(sbom: Path, *, client: OsvClient) -> dict: - """`scan_sbom` as a tool run: RuntimeError when the OSV API answered no package.""" - doc = scan_sbom(sbom, client) - prov = doc["provenance"] - if prov["status"] == "failed": - detail = (prov.get("failure") or {}).get("detail") or "no package was answered" - raise RuntimeError(f"OSV API unavailable at {prov['base_url']}: {detail}") - return doc - - -# -- dedupe against grype ------------------------------------------------------ - - -def _ids(c: dict) -> set[str]: - return {c.get("osv_id") or "", *(c.get("aliases") or []), *(c.get("upstream") or [])} - {""} - - -def mark_grype_duplicates(candidates: list[dict], grype_doc: dict) -> dict: - """Tag ``also_in_grype`` on each candidate; return counts.""" - seen: dict[tuple[str, str, str], set[str]] = {} - for m in grype_doc.get("matches") or []: - art, vuln = m.get("artifact") or {}, m.get("vulnerability") or {} - key = parse_purl(art.get("purl") or "") - if key is None: - continue - ids = seen.setdefault(key, set()) - ids.add(vuln.get("id") or "") - ids.update(r.get("id") or "" for r in m.get("relatedVulnerabilities") or []) - dup = 0 - for c in candidates: - hit = bool(_ids(c) & seen.get((c["ecosystem"], c["package"], c["found_version"]), set())) - c["also_in_grype"] = hit - dup += hit - return {"also_in_grype": dup, "osv_only": len(candidates) - dup} - - -def _ownership( - doc: dict, -) -> tuple[ - # child artifact id -> (purl identity, owners, owned paths): grype carries - # syft's artifact ids, so a match's id names the exact copy - dict[str, tuple[tuple[str, str, str], set[str], set[str]]], - # purl identity -> (owners, owned paths), summed across same-purl copies: - # the fallback for a match that brings no usable id - dict[tuple[str, str, str], tuple[set[str], set[str]]], -]: - """Ownership by artifact id, and by purl identity as the fallback. - - The same name+version can be OS-installed more than once, by different OS - packages, and app-installed too; the artifact id (or, failing that, the - paths) says which copy is which. - """ - artifacts = {a.get("id"): a for a in doc.get("artifacts") or [] if isinstance(a, dict)} - by_id: dict[str, tuple[tuple[str, str, str], set[str], set[str]]] = {} - by_purl: dict[tuple[str, str, str], tuple[set[str], set[str]]] = {} - for r in doc.get("artifactRelationships") or []: - parent = artifacts.get(r.get("parent")) or {} - if r.get("type") != OWNERSHIP or parent.get("type") not in OS_PACKAGE_TYPES: - continue - child = artifacts.get(r.get("child")) or {} - key = parse_purl(child.get("purl") or "") - if not key: - continue - owner = parent.get("name") or "" - child_paths = {loc.get("path") for loc in child.get("locations") or [] if loc.get("path")} - cid = child.get("id") or "" - if cid: - _, id_owners, id_paths = by_id.setdefault(cid, (key, set(), set())) - id_owners.add(owner) - id_paths.update(child_paths) - purl_owners, purl_paths = by_purl.setdefault(key, (set(), set())) - purl_owners.add(owner) - purl_paths.update(child_paths) - return by_id, by_purl - - -def os_installed_owners(doc: dict) -> dict[tuple[str, str, str], list[str]]: - """Language package (purl identity) -> the OS packages that installed it.""" - return {key: sorted(owners - {""}) for key, (owners, _paths) in _ownership(doc)[1].items()} - - -def grype_os_installed_matches(grype_doc: dict, doc: dict) -> list[dict]: - """grype matches on language packages an OS package installed. - - grype applies file ownership only partly, even on syft JSON: on - ``ubi9/python-311`` 26 of its 32 language-package matches (1 critical, - 12 high: npm's bundled ``tar``, ``brace-expansion``…, owned by the - ``nodejs``/``npm`` RPMs) were against RPM-installed packages. Their fixes - arrive in the owning RPM, so the upstream advisory is not evidence on its - own: the owner's own advisory state is. - - A match is attributed through its artifact id — grype carries syft's ids, - which name the exact copy — so two OS packages that each installed the - same name+version are never conflated. Without a usable id the purl plus - the match's paths decide, and a match with neither id nor paths keeps the - purl attribution, since it cannot be told apart. - """ - by_id, by_purl = _ownership(doc) - out = [] - for m in grype_doc.get("matches") or []: - art, vuln = m.get("artifact") or {}, m.get("vulnerability") or {} - if art.get("type") in OS_PACKAGE_TYPES: - continue - paths = [loc.get("path") for loc in art.get("locations") or [] if loc.get("path")] - hit = by_id.get(art.get("id") or "") - if hit is not None: - key, owners, _owned = hit - else: - key = parse_purl(art.get("purl") or "") - fallback = by_purl.get(key) - if fallback is None: - continue - owners, owned_paths = fallback - # the same name+version can be OS-installed and app-installed: - # only a match inside the owned paths is the OS package's copy - if owned_paths and paths and not set(paths) & owned_paths: - continue - out.append( - { - "id": vuln.get("id"), - "severity": vuln.get("severity"), - "ecosystem": key[0], - "package": key[1], - "version": key[2], - "paths": paths, - "installed_by": sorted(owners - {""}), - } - ) - return sorted(out, key=lambda x: (x["package"], x["version"], x["id"] or "")) - - -# -- air-gapped: osv-scanner offline on the filtered packages ------------------ - -_PURL_TYPE = {v: k for k, v in PURL_ECOSYSTEM.items()} - - -def _purl(eco: str, name: str, version: str) -> str: - ptype = _PURL_TYPE[eco] - if eco == "Maven": - group, _, artifact = name.partition(":") - return f"pkg:maven/{group}/{artifact}@{version}" - if eco == "Go": - version = f"go{version}" if name == "stdlib" else f"v{version}" - if eco == "npm" and name.startswith("@"): - name = "%40" + name[1:] - return f"pkg:{ptype}/{name}@{version}" - - -def write_cyclonedx(packages: dict[tuple[str, str, str], dict], path: Path) -> Path: - """The filtered packages as a CycloneDX file (name must end ``.cdx.json``).""" - if not str(path).endswith(".cdx.json"): - raise ValueError("osv-scanner recognises a CycloneDX file only by a .cdx.json name") - components = [ - {"type": "library", "name": n, "version": v, "purl": _purl(e, n, v)} - for e, n, v in sorted(packages) - ] - doc = {"bomFormat": "CycloneDX", "specVersion": "1.5", "version": 1, "components": components} - Path(path).write_text(json.dumps(doc, indent=1) + "\n", encoding="utf-8") - return Path(path) - - -def run_offline(binary: str, cdx: Path, timeout: int) -> tuple[subprocess.CompletedProcess, str]: - """osv-scanner on the filtered CycloneDX, offline (local OSV database only).""" - from traust_engine.toolchain import freeze_flags - - flags = freeze_flags("osv-scanner") # --offline: never phones home - cmd = [binary, "scan", "source", "-L", str(cdx), "--format", "json", *flags] - proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) - return proc, " ".join(cmd) - - -def parse_offline(doc: dict, packages: dict[tuple[str, str, str], dict]) -> list[dict]: - """osv-scanner's result as candidates, with the image paths restored.""" - out = [] - for result in doc.get("results") or []: - for entry in result.get("packages") or []: - pkg = entry.get("package") or {} - eco, name, ver = ( - pkg.get("ecosystem") or "", - pkg.get("name") or "", - pkg.get("version") or "", - ) - if eco == "PyPI": - name = re.sub(r"[-_.]+", "-", name).lower() - if eco == "Go": - ver = ver.removeprefix("v").removeprefix("go") - src = (packages.get((eco, name, ver)) or {}).get("sources") or [] - groups = { - vid: g.get("max_severity") - for g in entry.get("groups") or [] - for vid in g.get("ids") or [] - } - for vuln in entry.get("vulnerabilities") or []: - vid = vuln.get("id") - if not vid: - continue - out.append( - { - "osv_id": vid, - "aliases": vuln.get("aliases") or [], - "upstream": vuln.get("upstream") or [], - "summary": vuln.get("summary") or (vuln.get("details") or "")[:200] or None, - "ecosystem": eco, - "package": name, - "found_version": ver, - "fixed_version": fixed_version(vuln, pkg.get("name") or name, eco), - "max_severity": groups.get(vid), - "severity_basis": "osv-scanner", - "evidence": "dependency_declared", - "version_basis": "sbom", - "sources": list(src), - } - ) - return sorted(out, key=lambda c: (c["ecosystem"], c["package"], c["osv_id"])) diff --git a/src/traust_engine/adapters/providers.py b/src/traust_engine/adapters/providers.py deleted file mode 100644 index 7e7c56f..0000000 --- a/src/traust_engine/adapters/providers.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Every traust-engine provider on a traust-core `Context`, registered once by name. The app's -engine (tool pins, safe-exec profiles, rule pack, allowlist) is loaded on first use, so a command -that never uses one runs without its config; one that does gets the load's error naming the file. -""" - -from __future__ import annotations - -import os -from collections.abc import Callable -from functools import cache -from pathlib import Path -from typing import Any - -from traust_core.v1.context import Context -from traust_core.v1.errors import ConfigError -from traust_core.v1.models.cloud_config import CHECKOV - -from traust_engine.adapters.cloud_config import register_checkov -from traust_engine.adapters.compliance_sources import register_compliance_sources -from traust_engine.adapters.detection_quality import register_detection_quality -from traust_engine.adapters.fleet_fix import register_fleet_fix -from traust_engine.adapters.opengrep import load_supplemental_packs -from traust_engine.adapters.tools import TOOL_NAMES, register_tools -from traust_engine.engine import HarnessEngine - - -def _resolved_feeds_cache(feeds_cache: Callable[[], Path] | None) -> Path | None: - if feeds_cache is None: - return None - try: - return feeds_cache() - except (ConfigError, FileNotFoundError): - # Opt-in: a deployment that hasn't configured a feeds cache (traust's - # `feeds_cache_dir` raises DeploymentConfigMissing when `progress_tracker` is also - # unset) keeps scanning its tools' native caches exactly as before the image split, - # rather than failing every scan on a setting only the deployed runtime needs. - return None - - -class _OnFirstUse: - """A provider built on its first attribute read.""" - - def __init__(self, build: Callable[[], Any]) -> None: - self._build = cache(build) - - def __getattr__(self, name: str) -> Any: - return getattr(self._build(), name) - - -def register_providers( - ctx: Context, - engine: Callable[[], HarnessEngine], - *, - canaries: str = "canaries", - feeds_cache: Callable[[], Path] | None = None, -) -> None: - """ADR and docs sources, fleet-fix applier and golden runner, rule-pack source and canary - materialiser (`recall-benchmark/`), and every scanner tool with its floors. - - `feeds_cache`, when given, resolves the feeds-cache root on first tool use (same timing as - `engine`); grype and govulncheck then read their databases from - `/tool-dbs/` instead of their native caches. Unresolvable (e.g. no - `progress_tracker` configured on a laptop install) falls back to native caches rather than - blocking every tool on a setting only the deployed runtime needs. - """ - - @cache - def configured() -> Context: - # PENDING(engine Policy): the tools' config still comes from the HarnessContext. - loaded = engine() - try: - pack = loaded.adapters.rule_pack_dir() - except ValueError as e: - raise ConfigError(str(e)) from e - cache_dir = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "recall-bench" - staged = Context(ctx.config, clock=ctx.clock) - register_detection_quality( - staged, - pack=pack, - supplements=load_supplemental_packs(), - canary_sources=loaded.ctx.config_home / "recall-benchmark" / canaries, - canary_repos=cache_dir / canaries, - ) - register_tools( - staged, - loc=loaded.ctx.locations, - profile_map=loaded.adapters.safe_exec_profile_map(), - pins=loaded.toolchain.pins(), - rule_pack=pack, - allowlist=loaded.adapters.allowlist_for_opengrep(), - feeds_cache=_resolved_feeds_cache(feeds_cache), - ) - ctx.tool_floors.update(staged.tool_floors) - return staged - - # NEED(config: product -> docs slug/version): no DocsTarget map yet, so every product's - # docs lookup is refused by name until one has a home. - register_compliance_sources(ctx, {}) - register_fleet_fix(ctx) - register_checkov(ctx) # needs none of the engine's config - ctx.rule_packs = _OnFirstUse(lambda: configured().rule_packs) - ctx.canaries = _OnFirstUse(lambda: configured().canaries) - for name in set(TOOL_NAMES) - {CHECKOV}: - ctx.tools.register(name, lambda name=name: configured().tools.get(name)) diff --git a/src/traust_engine/adapters/tools.py b/src/traust_engine/adapters/tools.py deleted file mode 100644 index f63a3e1..0000000 --- a/src/traust_engine/adapters/tools.py +++ /dev/null @@ -1,309 +0,0 @@ -"""traust-engine's scanner adapters as traust-core tool providers. - -Each provider runs its adapter's own `scan()` over one target (the adapter keeps its argv, -safe_exec profile, freeze env, rule-pack pinning and secret redaction) and reports the -binary's version and database through the engine toolchain probes; this module only adapts -the call to core's `ToolProvider`. Nothing here starts a process or opens a connection the -adapter or toolchain didn't already. - -Not registered (see `UNREGISTERED`): the crypto probes core already implements natively. -""" - -from __future__ import annotations - -import re -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, field -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any, Generic, TypeVar - -from traust_core.v1.context import Context -from traust_core.v1.errors import ServiceError, TraustError, ValidationError -from traust_core.v1.models.common import ToolTarget -from traust_core.v1.providers.base import Acquisition, Readiness, Result, argv_digest, on_path -from traust_core.v1.providers.clock import Clock -from traust_core.v1.providers.tools import DatabaseStatus, NoDatabase, ToolRequest - -from traust_engine import toolchain -from traust_engine.adapters import ( - cloud_config, - gitleaks, - govulncheck, - grype, - joern, - opengrep, - osv, - osv_sbom, - yara, -) -from traust_engine.adapters._contract_bridge import AdapterResult # contract types via adapters -from traust_engine.osv import OsvClient - -# grype refuses a database older than this itself (traust check_drift GRYPE_DB_MAX_AGE_DAYS). -GRYPE_DB_MAX_AGE = timedelta(days=5) - -# Adapters with no provider here, and why; `traust admin toolchain` can name them. -UNREGISTERED: Mapping[str, str] = { - "crypto_probe": "core's CryptoProbe (FactEnumerator) is the source-tree crypto census", - "crypto_audit": "core's CryptoProbe, ImageCryptoProbe and ClusterCryptoProbe cover its " - "source, image and cluster modes", -} - -# What `register_tools` registers, so a lazy registration knows the names before the config. -TOOL_NAMES = ( - "checkov", - "gitleaks", - "gitleaks-history", - "govulncheck", - "grype", - "joern", - "opengrep", - "osv", - "osv_sbom", - "yara", -) - -Res = TypeVar("Res") - - -@dataclass(frozen=True, slots=True) -class AdapterTool(Generic[Res]): - """One engine adapter over one kind of target, with its binary's version and database.""" - - name: str - binaries: tuple[str, ...] - scan: Callable[[Any], Res] - clock: Clock - precondition: Callable[[], Readiness] | None = None - target: ToolTarget = ToolTarget.CHECKOUT - pin: Mapping[str, Any] = field(default_factory=dict) # the binary's external-tools.yaml row - # The argv `scan` runs, for adapters that build it from the target alone. - argv: Callable[[Any], Sequence[str]] | None = None - # The feeds-cache mount this tool's database reads from (grype, govulncheck); None keeps the - # tool's native cache location, unchanged from before the image split. - db_dir: Path | None = None - - def check(self) -> Readiness: - if self.binaries and not (found := on_path(*self.binaries, missing=" not on PATH")).ready: - return found - return self.precondition() if self.precondition else Readiness(ready=True) - - def acquire(self, request: ToolRequest) -> Result[Res]: - given, where = self._target(request) - try: - result = self.scan(given) - except TraustError: - raise - except ValueError as e: - raise ValidationError(f"{self.name} over {where}: {e}") from e - except (OSError, RuntimeError) as e: # missing binary, timeout, tool failure - raise ServiceError(f"{self.name} over {where}: {e}") from e - db = self.database() - return Result( - result, - Acquisition( - provider=self.name, - acquired_at=self.clock.now(), - version=_scanner_version(result), - source=str(where.resolve()), - argv_sha256=argv_digest(self.argv(given)) if self.argv else None, - db_built_at=db.built_at if db else None, - ), - ) - - def version(self) -> str | None: - if not self._binary: - return None # an API, not a binary (osv) - return toolchain.tool_version( - self._binary, - version_cmd=self.pin.get("version_cmd"), - version_regex=self.pin.get("version_regex"), - ) - - def database(self) -> DatabaseStatus | None: - binary = self._binary - # Gated on POPULATABLE_TOOLS (grype only), matching ProviderCatalog.fetch_databases' - # assumption in traust-core that `database() is not None` means `update_database()` works - # (it tries every such tool and has no populatable-only filter of its own). Exposing - # govulncheck's now-real database status through this protocol needs that core-side - # distinction first — NEED(core: ProviderCatalog.fetch_databases to filter populatable - # tools itself, not infer it from database() being set). Until then, govulncheck's - # database presence is still checked where it matters: the scan-time preflight in - # `adapters.govulncheck._run_govulncheck`, via `toolchain.db_status(..., db_dir=...)` - # directly. - if ( - binary not in toolchain.POPULATABLE_TOOLS - or (info := toolchain.db_status(binary, db_dir=self.db_dir)) is None - ): - return None - return DatabaseStatus( - tool=self.name, - exists=info.exists, - built_at=_built_at(info.built), - schema_version=info.schema_version, - max_age=GRYPE_DB_MAX_AGE if binary == "grype" else None, - path=info.path, - ) - - def update_database(self) -> DatabaseStatus: - binary = self._binary - if binary not in toolchain.POPULATABLE_TOOLS: - raise NoDatabase(f"{self.name} keeps no vulnerability database to update") - ok, message = toolchain.populate(binary) - if not ok: - raise ServiceError(message) - status = self.database() - if status is None: - raise ServiceError(f"{self.name}: database updated but its status is unreadable") - return status - - @property - def _binary(self) -> str: - return self.binaries[0] if self.binaries else "" - - def _target(self, request: ToolRequest) -> tuple[Any, Path]: - """The scan's input, and the path its errors and provenance name.""" - if request.target is not self.target: - raise ValidationError( - f"{self.name} runs over a {self.target.value}, not a {request.target.value}" - ) - if (sbom := request.sbom) is not None: - if not sbom.is_file(): - raise ValidationError(f"{self.name}: SBOM is not a file: {sbom}") - return sbom, sbom - query = request.symbols - workspace = query.workspace if query else request.checkout() - if not workspace.is_dir(): - raise ValidationError(f"{self.name}: workspace is not a directory: {workspace}") - return query or workspace, workspace - - -def register_tools( - ctx: Context, - *, - loc: opengrep.Locations | None, - profile_map: Mapping[str, Any], - pins: Mapping[str, Mapping[str, Any]], - rule_pack: Path, - allowlist: opengrep.RulePackAllowlist | None = None, - yara_rules: Sequence[str] = (), - osv_client: Callable[[], OsvClient] = OsvClient, - feeds_cache: Path | None = None, -) -> None: - """Register every engine scanner in `ctx.tools` (Checkov included, through - `register_checkov`) and the pins' `expected` versions as `ctx.tool_floors`. - - `profile_map` is the deployment's safe_exec profiles (`AdaptersOps.safe_exec_profile_map()`), - `pins` external-tools.yaml (`ToolchainOps.pins()`), `rule_pack` / `allowlist` the opengrep - pack and supplemental-pack allowlist, `yara_rules` pinned `url@sha` sources (empty: the - adapter's pinned default). `feeds_cache`, when given, points grype and govulncheck at - `/tool-dbs/` instead of their native caches; a missing or unusable database - there raises before the scan starts (ScannerDatabaseUnavailable), never a silent zero. - """ - clock = ctx.clock - profiles = dict(profile_map) - rules = list(yara_rules) - - def tool_db_dir(name: str) -> Path | None: - return None if feeds_cache is None else feeds_cache / "tool-dbs" / name - - def tool( - name: str, binaries: tuple[str, ...], scan: Callable[[Any], Any], **kw: Any - ) -> AdapterTool[Any]: - return AdapterTool( - name, binaries, scan, clock, pin=pins.get(next(iter(binaries), ""), {}), **kw - ) - - def gitleaks_ready() -> Readiness: - return _resolves("gitleaks", lambda: gitleaks.resolve_gitleaks_config(None, loc)) - - tools = [ - tool( - "gitleaks", - ("gitleaks",), - lambda w: gitleaks.scan(w, mode="dir", loc=loc), - precondition=gitleaks_ready, - ), - tool( - "gitleaks-history", - ("gitleaks", "git"), - lambda w: gitleaks.scan(w, mode="history", loc=loc), - precondition=gitleaks_ready, - ), - tool( - "govulncheck", - ("govulncheck", "go"), - lambda w: govulncheck.scan(w, profile_map=profiles, db_dir=tool_db_dir("govulncheck")), - argv=lambda _: govulncheck.command(db_dir=tool_db_dir("govulncheck")), - db_dir=tool_db_dir("govulncheck"), - ), - tool( # DB frozen by default - "grype", - ("grype",), - lambda w: grype.scan(w, db_dir=tool_db_dir("grype")), - argv=lambda w: grype.command(str(w)), - db_dir=tool_db_dir("grype"), - ), - tool( - "joern", - ("joern", "joern-parse"), - lambda q: joern.scan(q.workspace, symbols=list(q.names)), - target=ToolTarget.SYMBOLS, - ), - tool( - "opengrep", - ("opengrep",), - lambda w: opengrep.scan(w, rule_pack=rule_pack, allowlist=allowlist, loc=loc), - precondition=lambda: _resolves( - "opengrep", lambda: opengrep.resolve_rule_pack_dir(rule_pack, loc) - ), - ), - tool("osv", (), lambda w: osv.scan(w, client=osv_client())), - tool( - "osv_sbom", - (), - lambda p: osv_sbom.scan(p, client=osv_client()), - target=ToolTarget.SBOM, - ), - tool("yara", ("yara", "git"), lambda w: yara.scan(w, rules=rules)), - ] - for t in tools: - ctx.tools.register(t.name, lambda t=t: t) - cloud_config.register_checkov(ctx) - binaries = {t.name: t._binary for t in tools} - ctx.tool_floors.update( - { - name: str(spec["expected"]) - for name, binary in {**binaries, cloud_config.CHECKOV: cloud_config.CHECKOV}.items() - if binary and (spec := pins.get(binary)) and spec.get("expected") - } - ) - - -def _scanner_version(result: object) -> str | None: - return (result.metadata.scanner_version or None) if isinstance(result, AdapterResult) else None - - -def _resolves(tool: str, resolve: Callable[[], object]) -> Readiness: - try: - resolve() - except ValueError as e: - return Readiness(ready=False, detail=f"{tool}: {e}") - return Readiness(ready=True) - - -def _built_at(raw: str | None) -> datetime | None: - """`grype db status`'s `Built:` value (`2026-08-24T01:31:46Z`, or with a trailing zone - name), as an aware datetime; None when it reads as neither.""" - if not raw: - return None - text = raw.strip().replace("Z", "+00:00") - try: - found = datetime.fromisoformat(text.split(" ")[0] if "T" in text else text) - except ValueError: - m = re.match(r"(\d{4}-\d{2}-\d{2})", text) - if not m: - return None - found = datetime.fromisoformat(m.group(1)) - return found if found.tzinfo else found.replace(tzinfo=UTC) diff --git a/src/traust_engine/adapters/yara.py b/src/traust_engine/adapters/yara.py deleted file mode 100644 index c27586f..0000000 --- a/src/traust_engine/adapters/yara.py +++ /dev/null @@ -1,398 +0,0 @@ -"""Known-malware-family signature candidates via YARA (deterministic pre-scan). - -Runs the YARA engine (BSD-3-Clause, invoked as a subprocess) over an -already-materialized blob tree — an exported container rootfs -(secure-container-audit) or an RPM prepared source tree -(secure-rpm-audit) — with an explicitly pinned rule pack, and reduces -its output to normalized FACTS for the audit skills to judge in context. - -This is a CANDIDATE GENERATOR, never a finder or verdict of record -(docs/deterministic-inferential-mix.md: deterministic tools route, -gate, tag, or index — never conclude). A YARA match is a true statement -that a byte pattern for a known malware family is present; it is NOT a -confirmed compromise. The audit must judge whether the match is a real -implant vs. a benign carrier (a security tool's own signature corpus, a -test fixture, an EICAR-style sample, a detection rule shipped on purpose) -before promoting a fact to a finding, and must record dismissed facts in -`scanner_correlation`. - -Because these are KNOWN-FAMILY signatures, recall is bounded: a clean scan -is not proof of absence, only absence of known families. High precision, -not high recall — a supply-chain tripwire, not a general malware detector. - -RULE PACKS ARE A SWAPPABLE INPUT, NEVER A BAKED-IN ASSET — the same -contract as run_opengrep.py. Rules are fetched or referenced at run time -and are deliberately NOT vendored. Sources accepted by --rules: - - /path/to/rules(dir|.yar|.yara) local rules, used as-is - https://github.com/...@ git repo, shallow-cloned to the cache - at the pinned 40-hex commit SHA - -Default (no --rules): the ReversingLabs YARA rules pack pinned below -(MIT — permissive, redistributable; license note carried into the report -for metadata.tools). It ships known-malware-family signatures organized -by family (ransomware/backdoor/trojan/…). Its freshness is watched by -/drift-watch's `yara-rules-pin` row: a frozen pin stops seeing new -families, so the pin is a deliberate, reviewed bump (never auto-advanced). - -Usage: - python3 run_yara.py [--rules SRC ...] [--out FILE] - [--timeout SECONDS] [--yara BIN] [--max-file-mb N] - -Output defaults to -yara.json in the CWD. -Exit 0 on a completed scan (with or without facts); 1 on tool failure; -2 on usage errors. -""" - -from __future__ import annotations - -import os -import re -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path -from typing import Any - -from traust_contracts.models import AdapterResult, Location - -from traust_engine.adapters._contract_bridge import ( - build_scan_result, - map_severity, - utc_now, -) -from traust_engine.assets import harness_version - -CACHE_ROOT = Path.home() / ".cache" / "traust-engine" - -# Default pack: ReversingLabs YARA rules (known-malware families). -RL_RULES_REPO = "https://github.com/reversinglabs/reversinglabs-yara-rules" -RL_RULES_SHA = "e0a0be54aa1e11ccfd6854e4f19e9476f328fd84" # develop @ 2026-08 -RL_RULES_LICENSE = ( - "ReversingLabs YARA rules: MIT (permissive, " - "redistributable) — see docs/external-dependencies.md" -) - -# A YARA match names a known-malware family — inherently high-signal, but -# still a candidate the audit judges (benign carrier vs. real implant). -DEFAULT_SEVERITY_HINT = "high" - -# Rule-declaration line: optional `global`/`private` modifiers, `rule`, -# then the identifier. The opening `{` may sit on the same line or the -# next, and tags (`: tag1 tag2`) are irrelevant to the index, so anchor -# only on the keyword + identifier at line start. Comments (`//`, `/*`) -# never match — their first non-space char isn't `rule`/`global`/`private`. -RULE_DECL_RE = re.compile(r"^\s*(?:global\s+|private\s+)*rule\s+([A-Za-z_][A-Za-z0-9_]*)\b") - -# Carrier trees whose YARA hits are almost always the tool's own signature -# corpus / test fixtures rather than a live implant — tagged, not dropped, -# so the audit can weight them (a detection rule IS supposed to contain the -# byte pattern it detects; see the doctrine note above). -CARRIER_PATH_RE = re.compile( - r"(^|/)(testdata|test|tests|fixtures?|samples?|examples?|" - r"yara[-_]?rules?|signatures?|clamav|rules)(/|$)", - re.IGNORECASE, -) - - -def discover_rule_files(root: Path) -> list[Path]: - """Every .yar/.yara file under a resolved rules path (or the file - itself if a single rule file was given).""" - if root.is_file(): - return [root] if root.suffix in (".yar", ".yara") else [] - return sorted(p for p in root.rglob("*") if p.is_file() and p.suffix in (".yar", ".yara")) - - -def build_rule_index(rule_files: list[Path]) -> dict[str, dict]: - """Map rule_id -> {source_file, category, family} by parsing rule - declarations. YARA match output gives only the rule id and the scanned - path, so this pre-index is how a fact recovers which family/category it - belongs to. `category` = the immediate parent directory (ReversingLabs - groups by family class: backdoor/ransomware/trojan/…); `family` is - derived from the rule id's trailing segment.""" - index: dict[str, dict] = {} - for f in rule_files: - category = f.parent.name - try: - text = f.read_text(encoding="utf-8", errors="replace") - except OSError: - continue - for line in text.splitlines(): - m = RULE_DECL_RE.match(line) - if not m: - continue - rid = m.group(1) - family = rid.split("_")[-1] if "_" in rid else rid - index[rid] = { - "source_file": str(f), - "category": category, - "family": family, - } - return index - - -def generate_include_index(rule_files: list[Path], workdir: Path) -> Path: - """Write a single index .yar that `include`s every rule file, so one - `yara` invocation compiles the whole pack. Absolute include paths keep - it independent of cwd.""" - index_path = workdir / "index.yar" - lines = [f'include "{f.as_posix()}"' for f in rule_files] - index_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - return index_path - - -def resolve_rules(sources: list[str]) -> list[dict]: - """Resolve each --rules source to {source, root, sha, license_note}. - Mirrors run_opengrep.resolve_rules: https-only, immutable 40-hex SHA, - cache trusted only when HEAD==pin and the worktree is pristine.""" - resolved: list[dict] = [] - if not sources: - sources = [f"{RL_RULES_REPO}@{RL_RULES_SHA}"] - for src in sources: - if src.startswith(("http://", "https://", "git@")): - m = re.match( - r"^(?P.+?)@(?P[A-Za-z0-9._/-]+)$", - src.replace("://", "\x00", 1).replace("git@", "\x01", 1), - ) - if m and "/" not in m.group("ref"): - url = m.group("url").replace("\x00", "://", 1).replace("\x01", "git@", 1) - ref = m.group("ref") - else: - url, ref = src, "HEAD" - # A movable branch/tag/HEAD silently changes what the scan - # reports despite a "pinned" claim — require an immutable - # commit (same rule as run_opengrep; audit E6). - if not re.fullmatch(r"[0-9a-f]{40}", ref): - raise SystemExit( - f"rules source {src!r}: ref must be a 40-hex commit SHA " - "(movable refs change scan results silently)" - ) - if not url.startswith("https://"): - raise SystemExit(f"rules source {src!r}: https:// URLs only") - dest = CACHE_ROOT / f"{Path(url).name}-{ref[:12]}" - - def _clone(_dest=dest, _url=url, _ref=ref) -> None: - _dest.parent.mkdir(parents=True, exist_ok=True) - subprocess.run( - ["git", "clone", "--quiet", "--", _url, str(_dest)], - check=True, - env={**os.environ, "GIT_ALLOW_PROTOCOL": "https"}, - ) - subprocess.run(["git", "-C", str(_dest), "checkout", "--quiet", _ref], check=True) - - def _cache_ok(_dest=dest, _ref=ref) -> bool: - head = subprocess.run( - ["git", "-C", str(_dest), "rev-parse", "HEAD"], capture_output=True, text=True - ) - if head.returncode != 0 or head.stdout.strip() != _ref: - return False - dirty = subprocess.run( - ["git", "-C", str(_dest), "status", "--porcelain"], - capture_output=True, - text=True, - ) - return dirty.returncode == 0 and not dirty.stdout.strip() - - if not dest.is_dir(): - _clone() - elif not _cache_ok(): - print( - f"[run_yara] rule cache {dest} fails pin/cleanliness " - "verification — discarding and re-cloning", - file=sys.stderr, - ) - shutil.rmtree(dest) - _clone() - if not _cache_ok(): - raise SystemExit( - f"rules source {src!r}: freshly cloned cache still fails " - "pin/cleanliness verification — refusing to scan with " - "unverifiable rules" - ) - license_note = ( - RL_RULES_LICENSE - if url.rstrip("/") == RL_RULES_REPO - else "verify upstream license before redistribution" - ) - resolved.append( - {"source": src, "root": str(dest), "sha": ref, "license_note": license_note} - ) - continue - path = Path(src).expanduser().resolve() - if not path.exists(): - print(f"rules source not found: {src}", file=sys.stderr) - sys.exit(2) - resolved.append( - { - "source": src, - "root": str(path), - "sha": None, - "license_note": "local rules — license per author", - } - ) - return resolved - - -def parse_matches(stdout: str, target: Path, index: dict[str, dict]) -> list[dict]: - """Turn `yara -r` output (` ` per match line) into - normalized facts. Scanned with the default namespace and without - -m/-g/-s, so each line is exactly the rule id, one space, then the - absolute path — the id has no spaces, so split(' ', 1) is unambiguous.""" - facts: list[dict] = [] - for line in stdout.splitlines(): - if " " not in line.strip(): - continue - rule_id, path = line.split(" ", 1) - if not rule_id or not path.strip(): - continue - try: - rel = str(Path(path).resolve().relative_to(target)) - except ValueError: - rel = path - meta = index.get(rule_id, {}) - facts.append( - { - "rule_id": rule_id, - "family": meta.get("family"), - "category": meta.get("category"), - "source_rule_file": meta.get("source_file"), - "file": rel, - "severity_hint": DEFAULT_SEVERITY_HINT, - "match_type": "yara_signature", - "carrier_path": bool(CARRIER_PATH_RE.search(rel)), - } - ) - return facts - - -def count_regular_files(target: Path) -> int: - """Regular files under target, not following symlinks (the same tree - yara -N walks). Bounded best-effort for the stats block.""" - n = 0 - for root, _dirs, files in os.walk(target, followlinks=False): - n += sum(1 for f in files if not (Path(root) / f).is_symlink()) - return n - - -def _execute_yara_scan( - target: Path, - rule_sources: list[str], - *, - timeout: int, - yara_bin: str, - max_file_mb: int, -) -> dict[str, Any]: - """Run yara subprocess and return report dict.""" - yara_version = subprocess.run( - [yara_bin, "--version"], capture_output=True, text=True - ).stdout.strip() - - rules = resolve_rules(rule_sources) - rule_files: list[Path] = [] - for r in rules: - rule_files.extend(discover_rule_files(Path(r["root"]))) - if not rule_files: - raise RuntimeError("no .yar/.yara rule files found") - - index = build_rule_index(rule_files) - with tempfile.TemporaryDirectory(prefix="run_yara_") as tmp: - index_file = generate_include_index(rule_files, Path(tmp)) - cmd = [ - yara_bin, - "-r", - "-w", - "-N", - "-z", - str(max_file_mb * 1024 * 1024), - str(index_file), - str(target), - ] - proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) - - errors = [ln for ln in proc.stderr.splitlines() if ln.strip()] - if proc.returncode != 0 and not proc.stdout.strip(): - raise RuntimeError(f"yara failed (exit {proc.returncode}): {proc.stderr[-500:]}") - - facts = parse_matches(proc.stdout, target, index) - return { - "tool": "run_yara", - "version": harness_version(), - "yara_version": yara_version, - "target": str(target), - "rules": rules, - "stats": { - "facts": len(facts), - "rules_compiled": len(index), - "rule_files": len(rule_files), - "files_considered": count_regular_files(target), - "carrier_facts": sum(1 for f in facts if f["carrier_path"]), - "engine_errors": len(errors), - }, - "facts": facts, - "errors": errors[:100], - } - - -def _facts_to_findings(facts: list[dict]) -> list[dict[str, Any]]: - findings: list[dict[str, Any]] = [] - for fact in facts: - loc = ( - Location(path=fact["file"]) - if Location is not None - else { - "path": fact["file"], - } - ) - title = f"YARA match: {fact['rule_id']}" - if fact.get("family"): - title += f" ({fact['family']})" - findings.append( - { - "id": f"yara/{fact['rule_id']}", - "title": title, - "severity": map_severity(fact.get("severity_hint", "high")), - "locations": [loc], - "description": ( - f"category={fact.get('category')} " - f"family={fact.get('family')} " - f"carrier_path={fact.get('carrier_path')}" - ), - "category": fact.get("category", ""), - } - ) - return findings - - -def scan( - target: Path, - rules: list[str] | None = None, - timeout: int = 600, -) -> AdapterResult: - """Run yara scan and return typed scan result.""" - yara_bin = "yara" - max_file_mb = 64 - target = target.resolve() - if not target.is_dir(): - raise ValueError(f"not a directory: {target}") - if not shutil.which(yara_bin): - raise FileNotFoundError(f"yara not found on PATH ({yara_bin})") - - try: - report = _execute_yara_scan( - target, - rules or [], - timeout=timeout, - yara_bin=yara_bin, - max_file_mb=max_file_mb, - ) - except subprocess.TimeoutExpired as e: - raise TimeoutError(f"yara timed out after {timeout}s") from e - - return build_scan_result( - str(target), - "yara", - _facts_to_findings(report["facts"]), - scanned_at=utc_now(), - scanner_version=report.get("yara_version", ""), - ) diff --git a/src/traust_engine/assets.py b/src/traust_engine/assets.py index d24f4a9..48b8d70 100644 --- a/src/traust_engine/assets.py +++ b/src/traust_engine/assets.py @@ -1,15 +1,16 @@ -"""Engine-owned assets: the engine's own version and its bundled scanner data. +"""Engine-owned assets: the engine's own version and its bundled opengrep rule pack. NOT configuration and NOT injected: these are intrinsic to the engine package — -its version (from package metadata), the opengrep rule pack + gitleaks config, -and the compliance reference data (NIST 800-53 spine and profiles, framework -catalogs, the control-to-check mapping) that ship *inside* the wheel, -versioned with the code. An operator never -configures them; they do not vary per estate. +its version (from package metadata) and the opengrep rule pack that ships +*inside* the wheel, versioned with the code. An operator never configures them; +they do not vary per estate. Located via ``importlib`` (metadata + resources), never ``Path(__file__)`` arithmetic or a repo-layout assumption — the latter breaks the moment the package is installed as a wheel (the original HARNESS_ROOT bug). + +PENDING(decision: sweep stages | metrics journal) — kept for traust; +deleted when traust retires or ports it. """ from __future__ import annotations @@ -36,21 +37,3 @@ def harness_version() -> str: def default_rule_pack_dir() -> Path: """Bundled opengrep rule pack directory (ships inside the package).""" return _package_root() / "data" / "secure-code-audit" / "opengrep-rules" - - -def default_gitleaks_config() -> Path: - """Bundled gitleaks config file (ships inside the package).""" - return ( - _package_root() / "data" / "secure-code-audit" / "gitleaks-rules" / "gitleaks-default.toml" - ) - - -def compliance_data_dir() -> Path: - """Bundled compliance reference data (ships inside the package). - - Public framework data with no estate content: the NIST 800-53 rev5 spine, - the 800-53B profiles, the PCI DSS / SOC 2 / GDPR catalogs, their - provenance, and the harness's control-to-check mapping - (``compliance-mapping.yaml``). - """ - return _package_root() / "data" / "compliance" diff --git a/src/traust_engine/compliance/__init__.py b/src/traust_engine/compliance/__init__.py deleted file mode 100644 index 068dc5d..0000000 --- a/src/traust_engine/compliance/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compliance scope + boundary resolution.""" diff --git a/src/traust_engine/compliance/dashboard.py b/src/traust_engine/compliance/dashboard.py deleted file mode 100644 index 8ea8a47..0000000 --- a/src/traust_engine/compliance/dashboard.py +++ /dev/null @@ -1,423 +0,0 @@ -"""Compliance posture dashboard (Phase 5). - -Aggregates every compliance-assessment artifact under -analysis-results/compliance/ (the assessment-output tree of record) into one posture view -per (target, framework) — newest artifact per pair wins — and renders -`metrics/dashboards/compliance/compliance-dashboard.{json,md}` with the -campaign's reporting discipline intact: - - - **population block** (corpus.py — never a hand-rolled denominator): - what the corpus is, and how many of its targets have any compliance - assessment at all ("compliance-assessed" coverage — the census-cut - line, computed on census's own resolver) - - **coverage honesty**: per-framework tables carry the transparency - rows (not_assessed, out-of-scope, skipped-with-reason) verbatim; - there is NO blended compliance percentage anywhere - - **per-team not_satisfied table** where attributable: findings - cross-filed with `control_refs` join to owner teams through - findings.db + the repo-graph owned-by edges; environment-target - results without a repo owner report "(unattributed — environment - target)" rather than a guess - - **trend**: each run appends a summary snapshot to - compliance-history.jsonl; the trend section renders once >=2 - snapshots exist (never from a single point) - -Usage: - traust compliance dashboard - [--assessments ] [--out-dir ] - [--findings-db ] [--results-root ] -""" - -from __future__ import annotations - -import datetime -import json -import sqlite3 -from pathlib import Path - -from traust_contracts import CorpusConfig - - -def collect_assessments(root: Path) -> list[dict]: - from traust_contracts.models import ComplianceAssessment - - docs = [] - for p in sorted(root.rglob("*compliance-assessment.json")) if root.is_dir() else []: - try: - raw = json.loads(p.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - continue - if (raw.get("metadata") or {}).get("artifact") != "compliance-assessment": - continue - doc = ComplianceAssessment.from_dict(raw).to_dict() - skipped = (raw.get("metadata") or {}).get("skipped_frameworks") - if skipped: - doc["metadata"]["skipped_frameworks"] = skipped - doc["_path"] = str(p) - docs.append(doc) - return docs - - -def _tid(d: dict) -> str: - t = d["metadata"].get("target") or {} - return t.get("environment") or t.get("product") or ",".join(t.get("repos") or []) or "(unnamed)" - - -def latest_per_target_framework(docs: list[dict]) -> dict[tuple, dict]: - """(target-id, framework) -> newest assessment covering it. - Per-framework selection so a newer single-framework run (e.g. - SOC 2 only) NEVER hides older coverage of other frameworks for the - same target — each framework row shows its own newest assessment, - dated.""" - sel: dict[tuple, dict] = {} - for d in docs: - tid = _tid(d) - fws = set(d.get("coverage") or {}) | set(d["metadata"].get("skipped_frameworks") or {}) - for fw in fws: - cur = sel.get((tid, fw)) - if cur is None or d["metadata"].get("generated_at", "") > cur["metadata"].get( - "generated_at", "" - ): - sel[(tid, fw)] = d - return sel - - -def population_block(results_root: Path, cfg: CorpusConfig) -> list[str]: - try: - from traust_engine.corpus import resolver as corpus - - res = corpus.resolve(results_root, cfg) - block = corpus.render_population_block( - res, - tool="build_compliance_dashboard.py", - unit="assessment targets (products/environments)", - filters=( - "newest assessment artifact per target; skipped " - "frameworks shown with reasons, never dropped" - ), - denominator=( - "corpus resolver (census authority); " - "compliance-assessed targets are the cut, " - "corpus repos are the base" - ), - ) - return block.splitlines()[2:] # our md supplies the heading - except Exception as e: - return [f"_population block unavailable: {e}_"] # say so loudly - - -def team_table(findings_db: Path) -> list[tuple[str, int, int]]: - """(team, open control_refs findings, distinct controls) via - findings.db + owned-by edges.""" - if not findings_db.is_file(): - return [] - con = sqlite3.connect(findings_db) - con.row_factory = sqlite3.Row - try: - rows = con.execute( - "SELECT f.control_refs, COALESCE(" - " (SELECT REPLACE(ge.from_id,'owner-team:','') " - " FROM graph_edges ge WHERE ge.rel='owned-by' AND " - " ge.to_id='repo:'||REPLACE(REPLACE(" - " COALESCE(r.repo_url,''),'https://',''),'.git','')" - " LIMIT 1), '(unassigned)') AS team " - "FROM v_open f JOIN repos r USING (repo_key) " - "WHERE f.control_refs IS NOT NULL" - ).fetchall() - except sqlite3.OperationalError: - con.close() - return [] - con.close() - agg: dict[str, dict] = {} - for r in rows: - t = agg.setdefault(r["team"], {"n": 0, "controls": set()}) - t["n"] += 1 - for ref in json.loads(r["control_refs"]): - t["controls"].add(ref) - return sorted( - ((team, v["n"], len(v["controls"])) for team, v in agg.items()), - key=lambda x: -x[1], - ) - - -def boundary_coverage( - scope_path: Path, graph_path: Path, findings_db: Path, docs: list[dict] -) -> list[dict]: - """Per-boundary in-scope coverage (Phase 6) — the auditor's - question: of the repos DECLARED in scope, which have assessment - evidence at all? Three cuts per boundary: resolved size, repos with - findings-db presence (findings evidence exists), and the newest - assessment run that cited this boundary. Resolution failures are - reported as rows, never silently dropped.""" - if not scope_path or not scope_path.is_file(): - return [] - from traust_engine.compliance.scope import ScopeError, load_scope, resolve - - try: - scope_doc = load_scope(scope_path) - except ScopeError as e: - return [{"boundary": "(registry unreadable)", "error": str(e)}] - - # newest assessment per boundary id - newest: dict[str, str] = {} - for d in docs: - sc = ((d.get("metadata") or {}).get("target") or {}).get("scope") - if sc and sc.get("boundary"): - gen = (d.get("metadata") or {}).get("generated_at", "") - if gen > newest.get(sc["boundary"], ""): - newest[sc["boundary"]] = gen - - evidence_urls: set[str] = set() - if findings_db.is_file(): - try: - con = sqlite3.connect(f"file:{findings_db}?mode=ro", uri=True) - evidence_urls = { - (r[0] or "").lower().rstrip("/") - for r in con.execute("SELECT DISTINCT repo_url FROM repos") - } - con.close() - except sqlite3.Error: - pass - - rows = [] - for bid in sorted(scope_doc.get("boundaries") or {}): - try: - res = resolve(scope_doc, bid, graph_path) - except ScopeError as e: - rows.append({"boundary": bid, "error": str(e)}) - continue - with_evidence = sum( - 1 - for r in res["repos"] - if any(u.endswith(f"github.com/{r}".lower()) for u in evidence_urls) - ) - rows.append( - { - "boundary": bid, - "frameworks": res["frameworks"], - "draft": res["draft"], - "declared_by": res["declared_by"], - "declared_at": res["declared_at"], - "in_scope_repos": len(res["repos"]), - "with_findings_evidence": with_evidence, - "excluded": len(res["excluded"]), - "last_assessed": newest.get(bid), - } - ) - return rows - - -def build( - assessments_dir: Path, - findings_db: Path, - results_root: Path, - cfg: CorpusConfig, - scope_path: Path | None = None, - graph_path: Path | None = None, -) -> dict: - from traust_engine.compliance.scope import default_scope_path - - docs = collect_assessments(assessments_dir) - sel = latest_per_target_framework(docs) - by_tid: dict[str, dict] = {} - for (tid, fw), d in sorted(sel.items()): - m = d["metadata"] - entry = by_tid.setdefault( - tid, - { - "target": tid, - "kind": (m.get("target") or {}).get("kind"), - "generated_at": "", - "harness_version": m.get("harness_version"), - "registry_hash": (m.get("registry_hash") or "")[:12], - "snapshot_id": ((m.get("target") or {}).get("snapshot_id") or "")[:12], - "frameworks": {}, - "skipped_frameworks": {}, - "coverage": {}, - "assessed_at": {}, - "not_satisfied": [], - }, - ) - gen = m.get("generated_at", "") - if gen > entry["generated_at"]: - entry.update( - generated_at=gen, - harness_version=m.get("harness_version"), - registry_hash=(m.get("registry_hash") or "")[:12], - snapshot_id=((m.get("target") or {}).get("snapshot_id") or "")[:12], - ) - if fw in (d.get("coverage") or {}): - entry["coverage"][fw] = d["coverage"][fw] - entry["assessed_at"][fw] = gen - for f in m.get("frameworks") or []: - if f["id"] == fw and f.get("caveat"): - entry["frameworks"][fw] = {"caveat": f["caveat"]} - entry["not_satisfied"].extend( - { - "framework": r["framework"], - "control_id": r["control_id"], - "check_id": r.get("check_id"), - "evidence_items": len(r.get("evidence") or []), - } - for r in d.get("results") or [] - if r.get("verdict") == "not_satisfied" and r.get("framework") == fw - ) - elif fw in (m.get("skipped_frameworks") or {}): - entry["skipped_frameworks"][fw] = m["skipped_frameworks"][fw] - # one winner per (target, framework), so covered-vs-skipped is - # already the newest state — no reconciliation needed here - targets = [by_tid[k] for k in sorted(by_tid)] - return { - "metadata": { - "artifact": "compliance-dashboard", - "role": ( - "posture aggregation — per-framework coverage with " - "transparency rows; no blended compliance " - "percentage exists; census remains the denominator " - "authority" - ), - "assessments_found": len(docs), - "targets": len(by_tid), - "generated_at": datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), - }, - "population_block": population_block(results_root, cfg), - "compliance_assessed": { - "targets_with_assessments": len(by_tid), - "note": ( - "census-cut line: corpus denominators in the " - "population block above; a target counts once " - "regardless of framework count" - ), - }, - "targets": targets, - "boundary_coverage": boundary_coverage( - scope_path or default_scope_path(), - graph_path, - findings_db, - docs, - ), - "team_table": [ - {"team": t, "open_findings": n, "distinct_controls": c} - for t, n, c in team_table(findings_db) - ], - } - - -def render_md(doc: dict, history: list[dict]) -> str: - L = ["# Compliance Posture Dashboard", ""] - A = L.append - m = doc["metadata"] - A( - f"_Generated {m['generated_at']} · {m['targets']} target(s) from " - f"{m['assessments_found']} assessment artifact(s). Per-framework " - f"numbers are never blended; the census remains the denominator " - f"authority._" - ) - A("") - A("## Population") - A("") - L.extend(doc["population_block"]) - A("") - A( - f"**Compliance-assessed targets:** " - f"{doc['compliance_assessed']['targets_with_assessments']} — " - + doc["compliance_assessed"]["note"] - ) - A("") - for t in doc["targets"]: - A(f"## Target: {t['target']} ({t['kind']})") - A("") - A( - f"_Assessed {t['generated_at']} · harness " - f"{t['harness_version']} · registry {t['registry_hash']}… · " - f"snapshot {t['snapshot_id']}…_" - ) - A("") - A( - "| Framework | Assessed | In scope | Satisfied | " - "Not satisfied | Not applicable | Not assessed |" - ) - A("|---|---|---:|---:|---:|---:|---:|") - for fw, cov in sorted(t["coverage"].items()): - when = (t.get("assessed_at", {}).get(fw) or "")[:10] - A( - f"| {fw} | {when} | {cov['total_in_scope']} | " - f"{cov['satisfied']} | **{cov['not_satisfied']}** | " - f"{cov['not_applicable']} | {cov['not_assessed']} |" - ) - for fw, reason in sorted(t["skipped_frameworks"].items()): - A(f"| {fw} | — | — | — | — | — | _{reason}_ |") - A("") - caveats = {fw: v["caveat"] for fw, v in t["frameworks"].items() if v.get("caveat")} - for fw, c in sorted(caveats.items()): - A(f"- **{fw} caveat:** {c}") - if t["not_satisfied"]: - A("") - A("**Not satisfied:**") - for r in t["not_satisfied"]: - A( - f"- `{r['framework']}:{r['control_id']}` via " - f"`{r['check_id']}` ({r['evidence_items']} evidence " - f"item(s))" - ) - A("") - bc = doc.get("boundary_coverage") or [] - if bc: - A("## Declared boundaries — in-scope coverage (scope registry)") - A("") - A( - "_The auditor's cut: of the repos DECLARED in each compliance " - "boundary, how many have findings evidence, and when was the " - "boundary last assessed. DRAFT = unsigned declaration._" - ) - A("") - A( - "| Boundary | Frameworks | In-scope repos | With findings " - "evidence | Excluded | Last assessed | Declared |" - ) - A("|---|---|---:|---:|---:|---|---|") - for b in bc: - if b.get("error"): - A(f"| {b['boundary']} | — | — | — | — | — | RESOLUTION FAILED: {b['error'][:80]} |") - continue - draft = " **DRAFT**" if b["draft"] else "" - A( - f"| {b['boundary']}{draft} | " - f"{', '.join(b['frameworks'])} | {b['in_scope_repos']} | " - f"{b['with_findings_evidence']} | {b['excluded']} | " - f"{b.get('last_assessed') or 'never'} | " - f"{b['declared_by']} {b['declared_at']} |" - ) - A("") - A("## Per-team open control-linked findings") - A("") - if doc["team_table"]: - A("| Team | Open findings with control_refs | Distinct controls |") - A("|---|---:|---:|") - for row in doc["team_table"]: - A(f"| {row['team']} | {row['open_findings']} | {row['distinct_controls']} |") - else: - A( - "_None yet — populated as not_satisfied controls cross-file " - "findings with `control_refs` (environment-target results " - "without a repo owner stay unattributed rather than " - "guessed)._" - ) - A("") - A("## Trend") - A("") - if len(history) >= 2: - A("| Run | Targets | Not satisfied (total) | Not assessed (total) |") - A("|---|---:|---:|---:|") - for h in history[-6:]: - A( - f"| {h['generated_at'][:10]} | {h['targets']} | " - f"{h['not_satisfied_total']} | {h['not_assessed_total']} |" - ) - else: - A( - "_Trend renders once ≥2 snapshots exist " - f"({len(history)} so far) — never from a single point._" - ) - A("") - return "\n".join(L) + "\n" diff --git a/src/traust_engine/compliance/ops.py b/src/traust_engine/compliance/ops.py deleted file mode 100644 index 3d39d2d..0000000 --- a/src/traust_engine/compliance/ops.py +++ /dev/null @@ -1,64 +0,0 @@ -"""``HarnessEngine.compliance`` — posture dashboard, bound to context. - -Paths come from the injected ``HarnessContext``; callers use ``h.compliance.build()`` -where they used to pass ``compliance_dir()``, ``findings_db()``, etc. -""" - -from __future__ import annotations - -from pathlib import Path - -from traust_contracts import DeploymentConfigMissing - -from traust_engine._ops_base import ContextOps -from traust_engine.compliance import dashboard -from traust_engine.locations import COMPLIANCE_REL, FINDINGS_DB_REL, REPO_GRAPH_REL - -SCOPE_REGISTRY = "compliance-scope.yaml" -DASHBOARD_OUT_REL = Path("metrics") / "dashboards" / "compliance" - - -class ComplianceOps(ContextOps): - def _compliance_dir(self) -> Path: - return self._analysis_results() / COMPLIANCE_REL - - def _findings_db(self) -> Path: - return self._analysis_results() / FINDINGS_DB_REL - - def _repo_graph(self) -> Path | None: - try: - return self._analysis_results() / REPO_GRAPH_REL - except DeploymentConfigMissing: - return None - - def scope_registry(self) -> Path: - """Where the scope registry lives: the config home. - - The file may not exist yet; the scope intake writes it there. - """ - return self._ctx.config_home / SCOPE_REGISTRY - - def dashboard_out_dir(self) -> Path: - return self._progress_tracker() / DASHBOARD_OUT_REL - - def build( - self, - *, - assessments_dir: Path | None = None, - findings_db: Path | None = None, - results_root: Path | None = None, - scope_path: Path | None = None, - graph_path: Path | None = None, - ) -> dict: - resolved_scope = scope_path if scope_path is not None else self.scope_registry() - return dashboard.build( - assessments_dir or self._compliance_dir(), - findings_db or self._findings_db(), - results_root or self._analysis_results(), - self._ctx.corpus, - scope_path=resolved_scope, - graph_path=graph_path if graph_path is not None else self._repo_graph(), - ) - - def render_md(self, doc: dict, history: list[dict]) -> str: - return dashboard.render_md(doc, history) diff --git a/src/traust_engine/compliance/scope.py b/src/traust_engine/compliance/scope.py deleted file mode 100644 index c47435b..0000000 --- a/src/traust_engine/compliance/scope.py +++ /dev/null @@ -1,224 +0,0 @@ -"""Compliance-boundary scope resolver (Phase 6). - -Resolves a declared compliance boundary (compliance-scope.yaml in the -deployment config home, schema-gated) into the concrete -repo set an assessment run scopes to. Frameworks scope SYSTEMS, not -repos: the registry declares boundaries at service/product level and -this resolver derives membership from the repo-graph product mapping -(`product:* --ships--> repo:*` edges — the same resolution -/isolation-review uses), applies the declared include/exclude edges, -and returns `org/name` repos in the exact shape -`run_compliance_check.py --repos` consumes. - -Resolution modes: - repo-graph product-mapping edges (organizational assertion) - deployment-iac a declared deployment inventory — repos a - service-declaration IaC checkout declares as deployed - for the service, every row citing its IaC source - (adopted after a boundary was withdrawn on the strength - of graph edges alone: graph edges are what a product - CLAIMS to ship; deployment IaC is evidence of what - actually deploys — prefer it whenever the evidence - inventory exists) - explicit include[] only, for boundaries with neither - -Doctrine: -- The graph RESOLVES scope; the registry DECLARES it. Neither stores - the other's job: a product gaining a repo flows into scope on the - next resolution; only genuine boundary decisions need registry edits. -- Fail-loud contract: an unknown boundary, an unresolvable product, or - an exclude that matches nothing in the resolved set (a stale boundary - claim) is an error, never a silent shrink. -- Draft boundaries (`declared_by: draft:...`) resolve normally but are - flagged `draft: true` so consumers label their outputs. - -Consumers: run_compliance_check.py (--boundary), the compliance -dashboard's in-scope coverage cut, and /compliance-check interview -(Phase 7) which writes the entries this resolves. - -Usage: - python3 resolve_compliance_scope.py --boundary - [--scope ] [--graph ] - [--list] [--json] -Exit 0 with the resolved set; 1 on any resolution failure. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -from traust_contracts.config import deployment_config_dir - -from traust_engine.locations import configured_locations, repo_graph - - -def default_scope_path() -> Path | None: - """compliance-scope.yaml in the deployment config home (it may not exist yet).""" - home = deployment_config_dir() - return home / "compliance-scope.yaml" if home is not None else None - - -def default_graph() -> Path | None: - """The repo-graph path under the config-owned analysis-results, or None.""" - return repo_graph(configured_locations()) - - -try: - import yaml -except ImportError: # pragma: no cover - yaml = None - - -class ScopeError(RuntimeError): - pass - - -def load_scope(path: Path) -> dict: - if yaml is None: - raise ScopeError("PyYAML required") - if not path.is_file(): - raise ScopeError(f"scope registry not found: {path}") - doc = yaml.safe_load(path.read_text(encoding="utf-8")) - if not isinstance(doc, dict) or "boundaries" not in doc: - raise ScopeError(f"not a scope registry (no 'boundaries'): {path}") - doc["_scope_dir"] = path.parent # relative deployment inventories - return doc - - -def _repo_from_node(node_id: str) -> str | None: - """repo:github.com/org/name -> org/name (non-github hosts keep - host prefix stripped only when the org/name shape survives).""" - if not node_id.startswith("repo:"): - return None - rest = node_id[len("repo:") :] - parts = rest.split("/", 1) - if len(parts) == 2 and "/" in parts[1]: - # host/org/name -> org/name - return parts[1] - return rest - - -def graph_product_repos(graph_path: Path, product: str) -> list[str]: - if not graph_path.is_file(): - raise ScopeError( - f"repo-graph not found: {graph_path} — build it with /repo-graph before resolving scope" - ) - g = json.loads(graph_path.read_text(encoding="utf-8")) - nodes = g.get("nodes") or [] - products = [n for n in nodes if n.get("type") == "product"] - matches = [n for n in products if n.get("id") == product] - if not matches: - by_label = [n for n in products if n.get("label") == product] - if len(by_label) > 1: - ids = ", ".join(n["id"] for n in by_label) - raise ScopeError( - f"product label {product!r} is ambiguous ({ids}) — " - f"declare the node id in the registry" - ) - matches = by_label - if not matches: - raise ScopeError( - f"product {product!r} not found in the repo-graph " - f"(node id or exact label) — nothing resolved, refusing " - f"to return an empty boundary" - ) - pid = matches[0]["id"] - repos = sorted( - { - r - for e in (g.get("edges") or []) - if e.get("from") == pid and (r := _repo_from_node(str(e.get("to") or ""))) is not None - } - ) - if not repos: - raise ScopeError( - f"product {pid!r} has no repo edges — an empty " - f"boundary is a declaration error, not a result" - ) - return repos - - -def resolve(scope_doc: dict, boundary_id: str, graph_path: Path | None = None) -> dict: - graph_path = graph_path if graph_path is not None else default_graph() - boundaries = scope_doc.get("boundaries") or {} - if boundary_id not in boundaries: - known = ", ".join(sorted(boundaries)) or "(none)" - raise ScopeError(f"unknown boundary {boundary_id!r} (declared: {known})") - b = boundaries[boundary_id] - include = [i["repo"] for i in b.get("include") or []] - exclude = {i["repo"]: i["reason"] for i in b.get("exclude") or []} - - if b["resolves_via"] == "deployment-iac": - ev = b.get("deployment_evidence") or {} - inv_path = Path(ev.get("inventory", "")) - if not inv_path.is_absolute(): - inv_path = (scope_doc.get("_scope_dir") or Path()) / inv_path - if not ev.get("service") or not str(ev.get("inventory", "")): - raise ScopeError( - f"{boundary_id}: deployment-iac requires deployment_evidence.inventory + .service" - ) - if not inv_path.is_file(): - raise ScopeError(f"{boundary_id}: deployment inventory not found: {inv_path}") - inv = json.loads(inv_path.read_text(encoding="utf-8")) - svc = (inv.get("services") or {}).get(ev["service"]) - if svc is None: - known = ", ".join(sorted(inv.get("services") or {})) or "(none)" - raise ScopeError( - f"{boundary_id}: service {ev['service']!r} not in inventory (declared: {known})" - ) - rows = svc.get("repos") or [] - base, citations = [], {} - for r in rows: - url = str(r.get("url") or "") - m = url.rstrip("/").split("/") - if len(m) < 2 or not r.get("source"): - raise ScopeError( - f"{boundary_id}: malformed inventory row " - f"{r!r} — every repo needs url + source " - f"(the IaC citation is the point)" - ) - repo = "/".join(m[-2:]) - base.append(repo) - citations[repo] = r["source"] - if not base: - raise ScopeError(f"{boundary_id}: inventory declares no repos for {ev['service']!r}") - elif b["resolves_via"] == "repo-graph": - if not b.get("product"): - raise ScopeError(f"{boundary_id}: resolves_via=repo-graph requires 'product'") - base = graph_product_repos(graph_path, b["product"]) - else: # explicit - if not include: - raise ScopeError( - f"{boundary_id}: resolves_via=explicit requires a non-empty include list" - ) - base = [] - - resolved = sorted(set(base) | set(include)) - stale = [r for r in exclude if r not in resolved] - if stale: - raise ScopeError( - f"{boundary_id}: exclude entries match nothing in the " - f"resolved set ({', '.join(stale)}) — stale boundary claim; " - f"update the registry, don't let it rot" - ) - final = [r for r in resolved if r not in exclude] - if not final: - raise ScopeError(f"{boundary_id}: resolution produced an empty repo set") - result_extra = {} - if b["resolves_via"] == "deployment-iac": - result_extra["evidence"] = [ - {"repo": r, "source": citations[r]} for r in final if r in citations - ] - return { - **result_extra, - "boundary": boundary_id, - "frameworks": list(b["frameworks"]), - "repos": final, - "excluded": [{"repo": r, "reason": exclude[r]} for r in sorted(exclude)], - "resolves_via": b["resolves_via"], - "product": b.get("product"), - "declared_by": b["declared_by"], - "declared_at": str(b["declared_at"]), - "draft": str(b["declared_by"]).startswith("draft:"), - } diff --git a/src/traust_engine/corpus/__init__.py b/src/traust_engine/corpus/__init__.py index c2ddf93..2c918b8 100644 --- a/src/traust_engine/corpus/__init__.py +++ b/src/traust_engine/corpus/__init__.py @@ -1 +1,5 @@ -"""Corpus management — discovery, identity, dedup of the report population.""" +"""Corpus management — discovery, identity, dedup of the report population. + +PENDING(decision: sweep stages) — kept for traust; +deleted when traust retires or ports it. +""" diff --git a/src/traust_engine/data/compliance/README.md b/src/traust_engine/data/compliance/README.md deleted file mode 100644 index 337a31c..0000000 --- a/src/traust_engine/data/compliance/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Compliance framework data (canonical spine + baselines) - -Vendored, license-verified framework data for the `/compliance-check` -skill (plan: `../../plans/compliance-check-plan.md`, epic XWING-397). -Everything here is **derived from public-domain OSCAL sources** — -`provenance.json` records the upstream URL, upstream sha256, retrieval -date, and derivation rule for every file. Nothing in this directory may -contain PCI DSS standard text (harness content guard rule 4) or any -restrictively-licensed framework prose. - -| File | Contents | Source | -|---|---|---| -| `spine-800-53-rev5.json` | canonical control spine: 1,196 controls+enhancements (id, title, family, withdrawn flag) | NIST SP 800-53 rev 5 OSCAL catalog (usnistgov/oscal-content, CC0/US-Gov public domain) | -| `profile-800-53b-moderate.json` | 287 control selections | NIST SP 800-53B Moderate baseline OSCAL profile | -| `profile-800-53b-high.json` | 370 control selections | NIST SP 800-53B High baseline OSCAL profile | -| `provenance.json` | upstream URLs, sha256s, retrieval dates, derivation rules | — | - -## FedRAMP overlay gap (open item, Phase 0 finding 2026-07-19) - -The authoritative FedRAMP rev 5 OSCAL baselines (GSA/fedramp-automation) -were **decommissioned during the FedRAMP 20x transition** — the repo and -automate.fedramp.gov are gone; only community mirrors remain -(e.g. 18farchive/fedramp-automation), which do not meet the provenance -bar for compliance data. Interim posture: - -- FedRAMP Moderate/High assessments run against the **NIST 800-53B - Moderate/High selections** (what the FedRAMP baselines are built on), - labeled as such in every report — never presented as the FedRAMP - baselines themselves. -- The FedRAMP-specific overlays (parameter values, selected additions) - should be acquired through an authoritative channel: Red Hat's own - FedRAMP program teams (ROSA-GC / ARO ATOs hold the official baseline - documents). Until then, FedRAMP-only deltas report `not_assessed - (fedramp overlay unavailable)`. - -## Refresh procedure - -Re-download from the URLs in `provenance.json`, verify the upstream -sha256 changed intentionally (NIST revision), re-derive with the same -rules, update `provenance.json` retrieval date. Frameworks are data: -consumers (the Phase 1 mapping registry, the Phase 3 skill) never -hardcode control lists. diff --git a/src/traust_engine/data/compliance/catalog-gdpr-technical.yaml b/src/traust_engine/data/compliance/catalog-gdpr-technical.yaml deleted file mode 100644 index 4ff3151..0000000 --- a/src/traust_engine/data/compliance/catalog-gdpr-technical.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# GDPR — technical-slice catalog (digital sovereignty). -# -# Article references are facts; the slice definitions below are -# harness-authored (EUR-Lex reuse verified 2026-07-19 — but this file -# deliberately cites article numbers only). Scope statement of record: -# compliance-check-plan.md "GDPR technical slice". The skill produces -# technical evidence; legal compliance judgment stays with counsel. -# Which stores hold personal data is DECLARED input, never inferred. - -reviewed: "2026-07-19" # last review against the living framework (drift-watch 180d window) -framework: gdpr-technical -slices: - - id: gdpr-residency - articles: ["44", "45", "46", "47", "48", "49"] - slice: where personal data physically resides and flows — - region-allowlist assertions over the inventory; cross-region - replication and transfer paths flagged. The skill reports where - data flows; whether the transfer mechanism suffices is counsel's - judgment. - classification_hint: deterministic - - id: gdpr-security-of-processing - articles: ["32"] - slice: encryption at rest and in transit on declared - personal-data stores; resilience and access controls on them. - classification_hint: deterministic-or-evidence_review - - id: gdpr-storage-limitation - articles: ["5(1)(e)"] - slice: retention configuration — lifecycle rules and TTLs on - declared personal-data stores. - classification_hint: evidence_review - - id: gdpr-erasure-capability - articles: ["17"] - slice: technical capability to delete — deletion APIs/procedures - exist for declared personal-data stores (capability, not - obligation-compliance). - classification_hint: evidence_review - - id: gdpr-access-audit - articles: ["30", "32"] - slice: access logging and audit trails on declared personal-data - stores. - classification_hint: deterministic-or-evidence_review -out_of_scope_note: lawful basis, consent, DPIA obligations, - controller/processor agreements, transfer-adequacy judgments, and any - overall compliance claim — organizational/legal, never narrated. diff --git a/src/traust_engine/data/compliance/catalog-pci-dss-v4.yaml b/src/traust_engine/data/compliance/catalog-pci-dss-v4.yaml deleted file mode 100644 index 74aedde..0000000 --- a/src/traust_engine/data/compliance/catalog-pci-dss-v4.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# PCI DSS v4.x — requirement-ID catalog. -# -# Requirement IDs and principal-requirement area labels only (facts, -# citable). PCI DSS text is PCI SSC copyright — it never appears here -# or anywhere in the workspace (harness content-guard rule 4). Area -# labels below are HARNESS-AUTHORED shorthand, not the standard's -# headings. A declared CDE boundary is mandatory input for every PCI -# run; requirements are assessed only against systems inside it. -# -# The technical starter set lists sub-requirements the mapping registry -# seeds with checks or crosswalks; full-requirement enumeration grows -# with registry coverage, not ahead of it. - -reviewed: "2026-07-19" # last review against the living framework (drift-watch 180d window) -framework: pci-dss-v4 -principal_requirements: - - {id: "1", area: network security controls} - - {id: "2", area: secure component configuration} - - {id: "3", area: stored account data protection} - - {id: "4", area: transmission cryptography} - - {id: "5", area: anti-malware} - - {id: "6", area: secure systems and software} - - {id: "7", area: access restriction / need-to-know} - - {id: "8", area: identification and authentication} - - {id: "9", area: physical access} - - {id: "10", area: logging and monitoring} - - {id: "11", area: security testing} - - {id: "12", area: organizational security policy} -technical_starter_set: - - {id: "3.5.1", crosswalk: ["nist-800-53-rev5:sc-28"]} - - {id: "4.2.1", crosswalk: ["nist-800-53-rev5:sc-8"]} - - {id: "6.3.3", crosswalk: ["nist-800-53-rev5:si-2"]} - - {id: "8.3.6", crosswalk: ["nist-800-53-rev5:ia-5"]} - - {id: "10.2.1", crosswalk: ["nist-800-53-rev5:au-2"]} - - {id: "11.3.1", crosswalk: ["nist-800-53-rev5:ra-5"]} -organizational_examples: ["12.1.1", "12.6.1", "9.2.1"] diff --git a/src/traust_engine/data/compliance/catalog-soc2-tsc.yaml b/src/traust_engine/data/compliance/catalog-soc2-tsc.yaml deleted file mode 100644 index f840dbd..0000000 --- a/src/traust_engine/data/compliance/catalog-soc2-tsc.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# SOC 2 — AICPA Trust Services Criteria: criterion-ID catalog. -# -# IDs and series/category labels only (facts, citable). TSC criteria -# text is AICPA copyright — it never appears here or anywhere in the -# workspace (harness content-guard rule 5). Category labels and -# classification hints below are HARNESS-AUTHORED words informed by the -# criterion subject areas, not reproductions. -# -# classification_hint seeds the mapping registry; the registry is the -# decision of record. Trust-service category selection is a declared -# engagement input; Security (CC series) is always in scope. - -reviewed: "2026-07-19" # last review against the living framework (drift-watch 180d window) -framework: soc2-tsc -version_note: "TSC 2017, 2022 points-of-focus revision — ids stable" -series: - - id: CC1 - area: control environment - criteria: [CC1.1, CC1.2, CC1.3, CC1.4, CC1.5] - classification_hint: organizational - - id: CC2 - area: information and communication - criteria: [CC2.1, CC2.2, CC2.3] - classification_hint: organizational - - id: CC3 - area: risk assessment - criteria: [CC3.1, CC3.2, CC3.3, CC3.4] - classification_hint: organizational - - id: CC4 - area: monitoring of controls - criteria: [CC4.1, CC4.2] - classification_hint: evidence_review - - id: CC5 - area: control activities - criteria: [CC5.1, CC5.2, CC5.3] - classification_hint: organizational - - id: CC6 - area: logical and physical access - criteria: [CC6.1, CC6.2, CC6.3, CC6.4, CC6.5, CC6.6, CC6.7, CC6.8] - classification_hint: deterministic-or-evidence_review - - id: CC7 - area: system operations - criteria: [CC7.1, CC7.2, CC7.3, CC7.4, CC7.5] - classification_hint: deterministic-or-evidence_review - - id: CC8 - area: change management - criteria: [CC8.1] - classification_hint: evidence_review - - id: CC9 - area: risk mitigation - criteria: [CC9.1, CC9.2] - classification_hint: evidence_review - - id: A1 - area: availability (optional category) - criteria: [A1.1, A1.2, A1.3] - classification_hint: deterministic-or-evidence_review - - id: C1 - area: confidentiality (optional category) - criteria: [C1.1, C1.2] - classification_hint: deterministic-or-evidence_review - - id: PI1 - area: processing integrity (optional category) - criteria: [PI1.1, PI1.2, PI1.3, PI1.4, PI1.5] - classification_hint: evidence_review - - id: P - area: privacy (optional category) - criteria: [P1.1, P2.1, P3.1, P3.2, P4.1, P4.2, P4.3, P5.1, P5.2, - P6.1, P6.2, P6.3, P6.4, P6.5, P6.6, P6.7, P7.1, P8.1] - classification_hint: organizational diff --git a/src/traust_engine/data/compliance/compliance-mapping.yaml b/src/traust_engine/data/compliance/compliance-mapping.yaml deleted file mode 100644 index 8a542b5..0000000 --- a/src/traust_engine/data/compliance/compliance-mapping.yaml +++ /dev/null @@ -1,275 +0,0 @@ -# Control-mapping registry — POLICY DATA, schema-gated by -# ai-security-harness/schema/compliance-mapping.schema.json. -# -# Every in-scope control is pre-classified here (deterministic / -# evidence_review / organizational); the LLM never classifies at run -# time. Deterministic checks are declarative assertion records evaluated -# by scripts/compliance_assert.py — one engine, no per-check code. -# Changing a classification or an assertion is a reviewed config change. -# Ships with traust-engine (data/compliance/); a deployment that needs a -# different mapping passes --registry. Each check's `fixtures:` path is -# relative to traust's tests/fixtures/compliance/. -# -# Snapshot contracts (what each collector materializes; Phase 2/3): -# findings_db pre-shaped views over analysis-results/graph/ -# findings.db for the target's repos: -# open_confirmed_hardcoded_credential_findings[], -# open_critical_dependency_findings[] -# (view definitions live with the Phase 3 runner — -# pre-shaping keeps the assertion DSL filter-free) -# cloud_inventory scripts/collect_cloud_inventory.py canonical -# snapshot: resources[] each carrying region, -# encryption_at_rest, public_exposure, logging … -# scan_k8s_hardening the KHS fact scanner's JSON output verbatim -# (results[].check, summary. counts) -# crypto_audit crypto-audit/v1 from scripts/crypto_audit.py -# (facts[].probe_id — provider census / image / -# runtime tiers; tier recorded in the snapshot) -# -# Multi-check controls: verdict = satisfied only when EVERY listed -# check is satisfied (runner semantic; a single not_satisfied wins). -# -# Coverage grows control-by-control, never by narration. - -version: 2 -note: 23 controls across four frameworks, 9 checks (findings_db, - scan_k8s_hardening, cloud_inventory, crypto_audit collectors). - -controls: - # ---- NIST 800-53 rev 5 (spine; FedRAMP M/H select from it) -------- - - framework: nist-800-53-rev5 - control_id: si-2 - classification: deterministic - checks: [chk-no-open-critical-dependency-vulns] - note: flaw remediation — open confirmed critical dependency CVEs - in the target's repos contradict it - - framework: nist-800-53-rev5 - control_id: ia-5.7 - classification: deterministic - checks: [chk-no-confirmed-hardcoded-credentials] - note: no embedded static authenticators — triage-confirmed - hardcoded-credential findings are the counter-evidence (raw - gitleaks candidates are never a verdict; the ledger concludes) - - framework: nist-800-53-rev5 - control_id: sc-28 - classification: deterministic - checks: [chk-encryption-at-rest] - - framework: nist-800-53-rev5 - control_id: sc-8 - classification: evidence_review - evidence_request: TLS posture for in-scope service endpoints - (termination points, minimum versions) — deterministic check - lands with the Phase 2 cloud collector - - framework: nist-800-53-rev5 - control_id: ac-2 - classification: evidence_review - evidence_request: account-review records for the target environment - parameters: [account_review_frequency_days] - - framework: nist-800-53-rev5 - control_id: ra-5 - classification: evidence_review - evidence_request: vulnerability-scanning cadence and coverage - evidence for the target (the campaign's own audit trail qualifies) - - framework: nist-800-53-rev5 - control_id: at-2 - classification: organizational - note: literacy training — out of technical scope - - framework: nist-800-53-rev5 - control_id: ac-6 - classification: deterministic - checks: [chk-no-privileged-workloads, chk-no-rbac-wildcards] - note: least privilege — privileged containers (KHS-W01) and RBAC - wildcards (KHS-R01) in the target's manifests contradict it - - framework: nist-800-53-rev5 - control_id: cm-6 - classification: deterministic - checks: [chk-no-privilege-escalation] - note: configuration settings — allowPrivilegeEscalation left - enabled (KHS-W02) is a baseline-configuration deviation - - - framework: nist-800-53-rev5 - control_id: sc-13 - classification: deterministic - checks: [chk-no-pqc-tls-killswitch] - note: cryptographic protection — a GODEBUG PQC-TLS killswitch - (tlsmlkem=0 / tlskyber=0) declared in build config disables - approved post-quantum key exchange; crypto-analysis governance - chains resolve ownership when it fires - - # ---- more PCI / SOC 2 rows on the new KHS checks ------------------ - - framework: pci-dss-v4 - control_id: "2.2.1" - classification: deterministic - checks: [chk-no-privilege-escalation, chk-no-privileged-workloads] - crosswalk: ["nist-800-53-rev5:cm-6"] - - framework: pci-dss-v4 - control_id: "7.2.1" - classification: deterministic - checks: [chk-no-rbac-wildcards] - crosswalk: ["nist-800-53-rev5:ac-6"] - - framework: soc2-tsc - control_id: CC6.3 - classification: deterministic - checks: [chk-no-rbac-wildcards, chk-no-privileged-workloads] - crosswalk: ["nist-800-53-rev5:ac-6"] - - framework: soc2-tsc - control_id: CC6.7 - classification: deterministic - checks: [chk-no-pqc-tls-killswitch] - crosswalk: ["nist-800-53-rev5:sc-13"] - - framework: soc2-tsc - control_id: C1.1 - classification: deterministic - checks: [chk-no-secrets-in-env] - note: confidential-information protection — secrets exposed via - container environment variables (KHS-W13) contradict it - - # ---- PCI DSS v4 (ids only; CDE boundary is declared input) -------- - - framework: pci-dss-v4 - control_id: "3.5.1" - classification: deterministic - checks: [chk-encryption-at-rest] - crosswalk: ["nist-800-53-rev5:sc-28"] - - framework: pci-dss-v4 - control_id: "6.3.3" - classification: deterministic - checks: [chk-no-open-critical-dependency-vulns] - crosswalk: ["nist-800-53-rev5:si-2"] - - framework: pci-dss-v4 - control_id: "12.1.1" - classification: organizational - note: security-policy maintenance — out of technical scope - - # ---- SOC 2 TSC (ids only; categories are declared input) ---------- - - framework: soc2-tsc - control_id: CC6.1 - classification: deterministic - checks: [chk-encryption-at-rest] - crosswalk: ["nist-800-53-rev5:sc-28"] - - framework: soc2-tsc - control_id: CC7.1 - classification: deterministic - checks: [chk-no-open-critical-dependency-vulns] - crosswalk: ["nist-800-53-rev5:si-2"] - - framework: soc2-tsc - control_id: CC1.1 - classification: organizational - note: control-environment / conduct — out of technical scope - - framework: soc2-tsc - control_id: A1.2 - classification: evidence_review - evidence_request: backup and recovery evidence (schedules, restore - tests) for the target environment - - # ---- GDPR technical slice (personal-data stores declared) --------- - - framework: gdpr-technical - control_id: gdpr-residency - classification: deterministic - checks: [chk-region-allowlist] - parameters: [allowed_data_regions] - - framework: gdpr-technical - control_id: gdpr-security-of-processing - classification: deterministic - checks: [chk-encryption-at-rest] - -checks: - - id: chk-no-open-critical-dependency-vulns - description: the findings-db view of the target's repos contains no - open, triage-confirmed critical dependency-CVE finding - collector: findings_db - assertion: - path: open_critical_dependency_findings[*] - operator: absent - fixtures: fixtures/chk-no-open-critical-dependency-vulns/ - - id: chk-no-confirmed-hardcoded-credentials - description: no open, triage-confirmed hardcoded-credential finding - (CWE-798/321 class) in the target's repos - collector: findings_db - assertion: - path: open_confirmed_hardcoded_credential_findings[*] - operator: absent - fixtures: fixtures/chk-no-confirmed-hardcoded-credentials/ - - id: chk-encryption-at-rest - description: every in-scope inventory resource that persists data - reports encryption at rest enabled - collector: cloud_inventory - assertion: - path: resources[*].encryption_at_rest - operator: equals - expected: true - applies_when: - path: resources[*] - operator: exists - fixtures: fixtures/chk-encryption-at-rest/ - - id: chk-region-allowlist - description: every in-scope inventory resource sits in a declared - allowed region (sovereignty/residency) - collector: cloud_inventory - assertion: - path: resources[*].region - operator: in - expected: "param:allowed_data_regions" - applies_when: - path: resources[*] - operator: exists - fixtures: fixtures/chk-region-allowlist/ - - id: chk-no-pqc-tls-killswitch - description: no GODEBUG PQC-TLS killswitch (tlsmlkem=0/tlskyber=0) - declared anywhere the crypto census scanned - collector: crypto_audit - assertion: - path: facts[*].probe_id - operator: not_equals - expected: CRYPTO_GODEBUG_TLS_KILLSWITCH - applies_when: - path: facts[*] - operator: exists - fixtures: fixtures/chk-no-pqc-tls-killswitch/ - - id: chk-no-privileged-workloads - description: no privileged container in the target's manifests - (KHS-W01 fires zero times) - collector: scan_k8s_hardening - assertion: - path: summary.KHS-W01 - operator: absent - applies_when: - path: stats.k8s_docs - operator: gte - expected: 1 - fixtures: fixtures/chk-no-privileged-workloads/ - - id: chk-no-rbac-wildcards - description: no wildcard verbs/resources in RBAC rules (KHS-R01 - fires zero times) - collector: scan_k8s_hardening - assertion: - path: summary.KHS-R01 - operator: absent - applies_when: - path: stats.k8s_docs - operator: gte - expected: 1 - fixtures: fixtures/chk-no-rbac-wildcards/ - - id: chk-no-privilege-escalation - description: allowPrivilegeEscalation disabled everywhere - (KHS-W02 fires zero times) - collector: scan_k8s_hardening - assertion: - path: summary.KHS-W02 - operator: absent - applies_when: - path: stats.k8s_docs - operator: gte - expected: 1 - fixtures: fixtures/chk-no-privilege-escalation/ - - id: chk-no-secrets-in-env - description: no secret exposed via container environment variable - (KHS-W13 fires zero times) - collector: scan_k8s_hardening - assertion: - path: summary.KHS-W13 - operator: absent - applies_when: - path: stats.k8s_docs - operator: gte - expected: 1 - fixtures: fixtures/chk-no-secrets-in-env/ diff --git a/src/traust_engine/data/compliance/profile-800-53b-high.json b/src/traust_engine/data/compliance/profile-800-53b-high.json deleted file mode 100644 index 6256942..0000000 --- a/src/traust_engine/data/compliance/profile-800-53b-high.json +++ /dev/null @@ -1,377 +0,0 @@ -{ - "source": "NIST SP 800-53B (Electronic (OSCAL) Version of NIST Special Publication 800-53 Revision 5.1.1 HIGH IMPACT BASELINE)", - "profile_uuid": "b5c9c74d-b24d-4e80-815a-80936528fb6d", - "note": "FedRAMP Moderate/High baselines are built on these NIST 800-53B selections; FedRAMP-specific overlays pending an authoritative source \u2014 see README", - "control_ids": [ - "ac-1", - "ac-10", - "ac-11", - "ac-11.1", - "ac-12", - "ac-14", - "ac-17", - "ac-17.1", - "ac-17.2", - "ac-17.3", - "ac-17.4", - "ac-18", - "ac-18.1", - "ac-18.3", - "ac-18.4", - "ac-18.5", - "ac-19", - "ac-19.5", - "ac-2", - "ac-2.1", - "ac-2.11", - "ac-2.12", - "ac-2.13", - "ac-2.2", - "ac-2.3", - "ac-2.4", - "ac-2.5", - "ac-20", - "ac-20.1", - "ac-20.2", - "ac-21", - "ac-22", - "ac-3", - "ac-4", - "ac-4.4", - "ac-5", - "ac-6", - "ac-6.1", - "ac-6.10", - "ac-6.2", - "ac-6.3", - "ac-6.5", - "ac-6.7", - "ac-6.9", - "ac-7", - "ac-8", - "at-1", - "at-2", - "at-2.2", - "at-2.3", - "at-3", - "at-4", - "au-1", - "au-10", - "au-11", - "au-12", - "au-12.1", - "au-12.3", - "au-2", - "au-3", - "au-3.1", - "au-4", - "au-5", - "au-5.1", - "au-5.2", - "au-6", - "au-6.1", - "au-6.3", - "au-6.5", - "au-6.6", - "au-7", - "au-7.1", - "au-8", - "au-9", - "au-9.2", - "au-9.3", - "au-9.4", - "ca-1", - "ca-2", - "ca-2.1", - "ca-2.2", - "ca-3", - "ca-3.6", - "ca-5", - "ca-6", - "ca-7", - "ca-7.1", - "ca-7.4", - "ca-8", - "ca-8.1", - "ca-9", - "cm-1", - "cm-10", - "cm-11", - "cm-12", - "cm-12.1", - "cm-2", - "cm-2.2", - "cm-2.3", - "cm-2.7", - "cm-3", - "cm-3.1", - "cm-3.2", - "cm-3.4", - "cm-3.6", - "cm-4", - "cm-4.1", - "cm-4.2", - "cm-5", - "cm-5.1", - "cm-6", - "cm-6.1", - "cm-6.2", - "cm-7", - "cm-7.1", - "cm-7.2", - "cm-7.5", - "cm-8", - "cm-8.1", - "cm-8.2", - "cm-8.3", - "cm-8.4", - "cm-9", - "cp-1", - "cp-10", - "cp-10.2", - "cp-10.4", - "cp-2", - "cp-2.1", - "cp-2.2", - "cp-2.3", - "cp-2.5", - "cp-2.8", - "cp-3", - "cp-3.1", - "cp-4", - "cp-4.1", - "cp-4.2", - "cp-6", - "cp-6.1", - "cp-6.2", - "cp-6.3", - "cp-7", - "cp-7.1", - "cp-7.2", - "cp-7.3", - "cp-7.4", - "cp-8", - "cp-8.1", - "cp-8.2", - "cp-8.3", - "cp-8.4", - "cp-9", - "cp-9.1", - "cp-9.2", - "cp-9.3", - "cp-9.5", - "cp-9.8", - "ia-1", - "ia-11", - "ia-12", - "ia-12.2", - "ia-12.3", - "ia-12.4", - "ia-12.5", - "ia-2", - "ia-2.1", - "ia-2.12", - "ia-2.2", - "ia-2.5", - "ia-2.8", - "ia-3", - "ia-4", - "ia-4.4", - "ia-5", - "ia-5.1", - "ia-5.2", - "ia-5.6", - "ia-6", - "ia-7", - "ia-8", - "ia-8.1", - "ia-8.2", - "ia-8.4", - "ir-1", - "ir-2", - "ir-2.1", - "ir-2.2", - "ir-3", - "ir-3.2", - "ir-4", - "ir-4.1", - "ir-4.11", - "ir-4.4", - "ir-5", - "ir-5.1", - "ir-6", - "ir-6.1", - "ir-6.3", - "ir-7", - "ir-7.1", - "ir-8", - "ma-1", - "ma-2", - "ma-2.2", - "ma-3", - "ma-3.1", - "ma-3.2", - "ma-3.3", - "ma-4", - "ma-4.3", - "ma-5", - "ma-5.1", - "ma-6", - "mp-1", - "mp-2", - "mp-3", - "mp-4", - "mp-5", - "mp-6", - "mp-6.1", - "mp-6.2", - "mp-6.3", - "mp-7", - "pe-1", - "pe-10", - "pe-11", - "pe-11.1", - "pe-12", - "pe-13", - "pe-13.1", - "pe-13.2", - "pe-14", - "pe-15", - "pe-15.1", - "pe-16", - "pe-17", - "pe-18", - "pe-2", - "pe-3", - "pe-3.1", - "pe-4", - "pe-5", - "pe-6", - "pe-6.1", - "pe-6.4", - "pe-8", - "pe-8.1", - "pe-9", - "pl-1", - "pl-10", - "pl-11", - "pl-2", - "pl-4", - "pl-4.1", - "pl-8", - "ps-1", - "ps-2", - "ps-3", - "ps-4", - "ps-4.2", - "ps-5", - "ps-6", - "ps-7", - "ps-8", - "ps-9", - "ra-1", - "ra-2", - "ra-3", - "ra-3.1", - "ra-5", - "ra-5.11", - "ra-5.2", - "ra-5.4", - "ra-5.5", - "ra-7", - "ra-9", - "sa-1", - "sa-10", - "sa-11", - "sa-15", - "sa-15.3", - "sa-16", - "sa-17", - "sa-2", - "sa-21", - "sa-22", - "sa-3", - "sa-4", - "sa-4.1", - "sa-4.10", - "sa-4.2", - "sa-4.5", - "sa-4.9", - "sa-5", - "sa-8", - "sa-9", - "sa-9.2", - "sc-1", - "sc-10", - "sc-12", - "sc-12.1", - "sc-13", - "sc-15", - "sc-17", - "sc-18", - "sc-2", - "sc-20", - "sc-21", - "sc-22", - "sc-23", - "sc-24", - "sc-28", - "sc-28.1", - "sc-3", - "sc-39", - "sc-4", - "sc-5", - "sc-7", - "sc-7.18", - "sc-7.21", - "sc-7.3", - "sc-7.4", - "sc-7.5", - "sc-7.7", - "sc-7.8", - "sc-8", - "sc-8.1", - "si-1", - "si-10", - "si-11", - "si-12", - "si-16", - "si-2", - "si-2.2", - "si-3", - "si-4", - "si-4.10", - "si-4.12", - "si-4.14", - "si-4.2", - "si-4.20", - "si-4.22", - "si-4.4", - "si-4.5", - "si-5", - "si-5.1", - "si-6", - "si-7", - "si-7.1", - "si-7.15", - "si-7.2", - "si-7.5", - "si-7.7", - "si-8", - "si-8.2", - "sr-1", - "sr-10", - "sr-11", - "sr-11.1", - "sr-11.2", - "sr-12", - "sr-2", - "sr-2.1", - "sr-3", - "sr-5", - "sr-6", - "sr-8", - "sr-9", - "sr-9.1" - ] -} diff --git a/src/traust_engine/data/compliance/profile-800-53b-moderate.json b/src/traust_engine/data/compliance/profile-800-53b-moderate.json deleted file mode 100644 index ec5aaa1..0000000 --- a/src/traust_engine/data/compliance/profile-800-53b-moderate.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "source": "NIST SP 800-53B (Electronic (OSCAL) Version of NIST Special Publication 800-53 Revision 5.2.0 MODERATE IMPACT BASELINE)", - "profile_uuid": "b07979a6-1b98-42dc-a776-60ee575b061e", - "note": "FedRAMP Moderate/High baselines are built on these NIST 800-53B selections; FedRAMP-specific overlays pending an authoritative source \u2014 see README", - "control_ids": [ - "ac-1", - "ac-11", - "ac-11.1", - "ac-12", - "ac-14", - "ac-17", - "ac-17.1", - "ac-17.2", - "ac-17.3", - "ac-17.4", - "ac-18", - "ac-18.1", - "ac-18.3", - "ac-19", - "ac-19.5", - "ac-2", - "ac-2.1", - "ac-2.13", - "ac-2.2", - "ac-2.3", - "ac-2.4", - "ac-2.5", - "ac-20", - "ac-20.1", - "ac-20.2", - "ac-21", - "ac-22", - "ac-3", - "ac-4", - "ac-5", - "ac-6", - "ac-6.1", - "ac-6.10", - "ac-6.2", - "ac-6.5", - "ac-6.7", - "ac-6.9", - "ac-7", - "ac-8", - "at-1", - "at-2", - "at-2.2", - "at-2.3", - "at-3", - "at-4", - "au-1", - "au-11", - "au-12", - "au-2", - "au-3", - "au-3.1", - "au-4", - "au-5", - "au-6", - "au-6.1", - "au-6.3", - "au-7", - "au-7.1", - "au-8", - "au-9", - "au-9.4", - "ca-1", - "ca-2", - "ca-2.1", - "ca-3", - "ca-5", - "ca-6", - "ca-7", - "ca-7.1", - "ca-7.4", - "ca-9", - "cm-1", - "cm-10", - "cm-11", - "cm-12", - "cm-12.1", - "cm-2", - "cm-2.2", - "cm-2.3", - "cm-2.7", - "cm-3", - "cm-3.2", - "cm-3.4", - "cm-4", - "cm-4.2", - "cm-5", - "cm-6", - "cm-7", - "cm-7.1", - "cm-7.2", - "cm-7.5", - "cm-8", - "cm-8.1", - "cm-8.3", - "cm-9", - "cp-1", - "cp-10", - "cp-10.2", - "cp-2", - "cp-2.1", - "cp-2.3", - "cp-2.8", - "cp-3", - "cp-4", - "cp-4.1", - "cp-6", - "cp-6.1", - "cp-6.3", - "cp-7", - "cp-7.1", - "cp-7.2", - "cp-7.3", - "cp-8", - "cp-8.1", - "cp-8.2", - "cp-9", - "cp-9.1", - "cp-9.8", - "ia-1", - "ia-11", - "ia-12", - "ia-12.2", - "ia-12.3", - "ia-12.5", - "ia-2", - "ia-2.1", - "ia-2.12", - "ia-2.2", - "ia-2.8", - "ia-3", - "ia-4", - "ia-4.4", - "ia-5", - "ia-5.1", - "ia-5.2", - "ia-5.6", - "ia-6", - "ia-7", - "ia-8", - "ia-8.1", - "ia-8.2", - "ia-8.4", - "ir-1", - "ir-2", - "ir-3", - "ir-3.2", - "ir-4", - "ir-4.1", - "ir-5", - "ir-6", - "ir-6.1", - "ir-6.3", - "ir-7", - "ir-7.1", - "ir-8", - "ma-1", - "ma-2", - "ma-3", - "ma-3.1", - "ma-3.2", - "ma-3.3", - "ma-4", - "ma-5", - "ma-6", - "mp-1", - "mp-2", - "mp-3", - "mp-4", - "mp-5", - "mp-6", - "mp-7", - "pe-1", - "pe-10", - "pe-11", - "pe-12", - "pe-13", - "pe-13.1", - "pe-14", - "pe-15", - "pe-16", - "pe-17", - "pe-2", - "pe-3", - "pe-4", - "pe-5", - "pe-6", - "pe-6.1", - "pe-8", - "pe-9", - "pl-1", - "pl-10", - "pl-11", - "pl-2", - "pl-4", - "pl-4.1", - "pl-8", - "ps-1", - "ps-2", - "ps-3", - "ps-4", - "ps-5", - "ps-6", - "ps-7", - "ps-8", - "ps-9", - "ra-1", - "ra-2", - "ra-3", - "ra-3.1", - "ra-5", - "ra-5.11", - "ra-5.2", - "ra-5.5", - "ra-7", - "ra-9", - "sa-1", - "sa-10", - "sa-11", - "sa-15", - "sa-15.3", - "sa-2", - "sa-22", - "sa-3", - "sa-4", - "sa-4.1", - "sa-4.10", - "sa-4.2", - "sa-4.9", - "sa-5", - "sa-8", - "sa-9", - "sa-9.2", - "sc-1", - "sc-10", - "sc-12", - "sc-13", - "sc-15", - "sc-17", - "sc-18", - "sc-2", - "sc-20", - "sc-21", - "sc-22", - "sc-23", - "sc-28", - "sc-28.1", - "sc-39", - "sc-4", - "sc-5", - "sc-7", - "sc-7.3", - "sc-7.4", - "sc-7.5", - "sc-7.7", - "sc-7.8", - "sc-8", - "sc-8.1", - "si-1", - "si-10", - "si-11", - "si-12", - "si-16", - "si-2", - "si-2.2", - "si-3", - "si-4", - "si-4.2", - "si-4.4", - "si-4.5", - "si-5", - "si-7", - "si-7.1", - "si-7.7", - "si-8", - "si-8.2", - "sr-1", - "sr-10", - "sr-11", - "sr-11.1", - "sr-11.2", - "sr-12", - "sr-2", - "sr-2.1", - "sr-3", - "sr-5", - "sr-6", - "sr-8" - ] -} diff --git a/src/traust_engine/data/compliance/provenance.json b/src/traust_engine/data/compliance/provenance.json deleted file mode 100644 index d3cc3de..0000000 --- a/src/traust_engine/data/compliance/provenance.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "retrieved": "2026-07-19", - "license": "CC0-1.0 / US-Gov public domain (LICENSE.md verified 2026-07-19)", - "upstream": "https://github.com/usnistgov/oscal-content", - "upstream_head": "78650f02ad93", - "files": { - "spine-800-53-rev5.json": { - "derived_from": "https://raw.githubusercontent.com/usnistgov/oscal-content/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json", - "upstream_sha256": "01f37cf90ea99d92242c936cbfbdebcc338eef1f71454e2acac36cc56e9bc062", - "derivation": "id/title/family/withdrawn per control+enhancement" - }, - "profile-800-53b-moderate.json": { - "derived_from": "https://raw.githubusercontent.com/usnistgov/oscal-content/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_MODERATE-baseline_profile.json", - "upstream_sha256": "9030dbf1f13169947eb97eb101b4bd2f00d3c151b100455a923ac75803f00ea1", - "derivation": "flattened include-controls ids" - }, - "profile-800-53b-high.json": { - "derived_from": "https://raw.githubusercontent.com/usnistgov/oscal-content/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json", - "upstream_sha256": "60576970caef91b2cba56d46e5948b72e3b879434fc2a4840bc4179e5e75cfd7", - "derivation": "flattened include-controls ids" - } - } -} diff --git a/src/traust_engine/data/compliance/spine-800-53-rev5.json b/src/traust_engine/data/compliance/spine-800-53-rev5.json deleted file mode 100644 index b0468ff..0000000 --- a/src/traust_engine/data/compliance/spine-800-53-rev5.json +++ /dev/null @@ -1,6169 +0,0 @@ -{ - "source": "NIST SP 800-53 rev 5 (OSCAL catalog)", - "catalog_uuid": "ea7c7688-79c5-463b-a91b-0650f2d98623", - "catalog_version": "5.2.0", - "controls": [ - { - "id": "ac-1", - "title": "Policy and Procedures", - "family": "AC" - }, - { - "id": "ac-2", - "title": "Account Management", - "family": "AC" - }, - { - "id": "ac-2.1", - "title": "Automated System Account Management", - "family": "AC" - }, - { - "id": "ac-2.2", - "title": "Automated Temporary and Emergency Account Management", - "family": "AC" - }, - { - "id": "ac-2.3", - "title": "Disable Accounts", - "family": "AC" - }, - { - "id": "ac-2.4", - "title": "Automated Audit Actions", - "family": "AC" - }, - { - "id": "ac-2.5", - "title": "Inactivity Logout", - "family": "AC" - }, - { - "id": "ac-2.6", - "title": "Dynamic Privilege Management", - "family": "AC" - }, - { - "id": "ac-2.7", - "title": "Privileged User Accounts", - "family": "AC" - }, - { - "id": "ac-2.8", - "title": "Dynamic Account Management", - "family": "AC" - }, - { - "id": "ac-2.9", - "title": "Restrictions on Use of Shared and Group Accounts", - "family": "AC" - }, - { - "id": "ac-2.10", - "title": "Shared and Group Account Credential Change", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-2.11", - "title": "Usage Conditions", - "family": "AC" - }, - { - "id": "ac-2.12", - "title": "Account Monitoring for Atypical Usage", - "family": "AC" - }, - { - "id": "ac-2.13", - "title": "Disable Accounts for High-risk Individuals", - "family": "AC" - }, - { - "id": "ac-3", - "title": "Access Enforcement", - "family": "AC" - }, - { - "id": "ac-3.1", - "title": "Restricted Access to Privileged Functions", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-3.2", - "title": "Dual Authorization", - "family": "AC" - }, - { - "id": "ac-3.3", - "title": "Mandatory Access Control", - "family": "AC" - }, - { - "id": "ac-3.4", - "title": "Discretionary Access Control", - "family": "AC" - }, - { - "id": "ac-3.5", - "title": "Security-relevant Information", - "family": "AC" - }, - { - "id": "ac-3.6", - "title": "Protection of User and System Information", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-3.7", - "title": "Role-based Access Control", - "family": "AC" - }, - { - "id": "ac-3.8", - "title": "Revocation of Access Authorizations", - "family": "AC" - }, - { - "id": "ac-3.9", - "title": "Controlled Release", - "family": "AC" - }, - { - "id": "ac-3.10", - "title": "Audited Override of Access Control Mechanisms", - "family": "AC" - }, - { - "id": "ac-3.11", - "title": "Restrict Access to Specific Information Types", - "family": "AC" - }, - { - "id": "ac-3.12", - "title": "Assert and Enforce Application Access", - "family": "AC" - }, - { - "id": "ac-3.13", - "title": "Attribute-based Access Control", - "family": "AC" - }, - { - "id": "ac-3.14", - "title": "Individual Access", - "family": "AC" - }, - { - "id": "ac-3.15", - "title": "Discretionary and Mandatory Access Control", - "family": "AC" - }, - { - "id": "ac-4", - "title": "Information Flow Enforcement", - "family": "AC" - }, - { - "id": "ac-4.1", - "title": "Object Security and Privacy Attributes", - "family": "AC" - }, - { - "id": "ac-4.2", - "title": "Processing Domains", - "family": "AC" - }, - { - "id": "ac-4.3", - "title": "Dynamic Information Flow Control", - "family": "AC" - }, - { - "id": "ac-4.4", - "title": "Flow Control of Encrypted Information", - "family": "AC" - }, - { - "id": "ac-4.5", - "title": "Embedded Data Types", - "family": "AC" - }, - { - "id": "ac-4.6", - "title": "Metadata", - "family": "AC" - }, - { - "id": "ac-4.7", - "title": "One-way Flow Mechanisms", - "family": "AC" - }, - { - "id": "ac-4.8", - "title": "Security and Privacy Policy Filters", - "family": "AC" - }, - { - "id": "ac-4.9", - "title": "Human Reviews", - "family": "AC" - }, - { - "id": "ac-4.10", - "title": "Enable and Disable Security or Privacy Policy Filters", - "family": "AC" - }, - { - "id": "ac-4.11", - "title": "Configuration of Security or Privacy Policy Filters", - "family": "AC" - }, - { - "id": "ac-4.12", - "title": "Data Type Identifiers", - "family": "AC" - }, - { - "id": "ac-4.13", - "title": "Decomposition into Policy-relevant Subcomponents", - "family": "AC" - }, - { - "id": "ac-4.14", - "title": "Security or Privacy Policy Filter Constraints", - "family": "AC" - }, - { - "id": "ac-4.15", - "title": "Detection of Unsanctioned Information", - "family": "AC" - }, - { - "id": "ac-4.16", - "title": "Information Transfers on Interconnected Systems", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-4.17", - "title": "Domain Authentication", - "family": "AC" - }, - { - "id": "ac-4.18", - "title": "Security Attribute Binding", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-4.19", - "title": "Validation of Metadata", - "family": "AC" - }, - { - "id": "ac-4.20", - "title": "Approved Solutions", - "family": "AC" - }, - { - "id": "ac-4.21", - "title": "Physical or Logical Separation of Information Flows", - "family": "AC" - }, - { - "id": "ac-4.22", - "title": "Access Only", - "family": "AC" - }, - { - "id": "ac-4.23", - "title": "Modify Non-releasable Information", - "family": "AC" - }, - { - "id": "ac-4.24", - "title": "Internal Normalized Format", - "family": "AC" - }, - { - "id": "ac-4.25", - "title": "Data Sanitization", - "family": "AC" - }, - { - "id": "ac-4.26", - "title": "Audit Filtering Actions", - "family": "AC" - }, - { - "id": "ac-4.27", - "title": "Redundant/Independent Filtering Mechanisms", - "family": "AC" - }, - { - "id": "ac-4.28", - "title": "Linear Filter Pipelines", - "family": "AC" - }, - { - "id": "ac-4.29", - "title": "Filter Orchestration Engines", - "family": "AC" - }, - { - "id": "ac-4.30", - "title": "Filter Mechanisms Using Multiple Processes", - "family": "AC" - }, - { - "id": "ac-4.31", - "title": "Failed Content Transfer Prevention", - "family": "AC" - }, - { - "id": "ac-4.32", - "title": "Process Requirements for Information Transfer", - "family": "AC" - }, - { - "id": "ac-5", - "title": "Separation of Duties", - "family": "AC" - }, - { - "id": "ac-6", - "title": "Least Privilege", - "family": "AC" - }, - { - "id": "ac-6.1", - "title": "Authorize Access to Security Functions", - "family": "AC" - }, - { - "id": "ac-6.2", - "title": "Non-privileged Access for Nonsecurity Functions", - "family": "AC" - }, - { - "id": "ac-6.3", - "title": "Network Access to Privileged Commands", - "family": "AC" - }, - { - "id": "ac-6.4", - "title": "Separate Processing Domains", - "family": "AC" - }, - { - "id": "ac-6.5", - "title": "Privileged Accounts", - "family": "AC" - }, - { - "id": "ac-6.6", - "title": "Privileged Access by Non-organizational Users", - "family": "AC" - }, - { - "id": "ac-6.7", - "title": "Review of User Privileges", - "family": "AC" - }, - { - "id": "ac-6.8", - "title": "Privilege Levels for Code Execution", - "family": "AC" - }, - { - "id": "ac-6.9", - "title": "Log Use of Privileged Functions", - "family": "AC" - }, - { - "id": "ac-6.10", - "title": "Prohibit Non-privileged Users from Executing Privileged Functions", - "family": "AC" - }, - { - "id": "ac-7", - "title": "Unsuccessful Logon Attempts", - "family": "AC" - }, - { - "id": "ac-7.1", - "title": "Automatic Account Lock", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-7.2", - "title": "Purge or Wipe Mobile Device", - "family": "AC" - }, - { - "id": "ac-7.3", - "title": "Biometric Attempt Limiting", - "family": "AC" - }, - { - "id": "ac-7.4", - "title": "Use of Alternate Authentication Factor", - "family": "AC" - }, - { - "id": "ac-8", - "title": "System Use Notification", - "family": "AC" - }, - { - "id": "ac-9", - "title": "Previous Logon Notification", - "family": "AC" - }, - { - "id": "ac-9.1", - "title": "Unsuccessful Logons", - "family": "AC" - }, - { - "id": "ac-9.2", - "title": "Successful and Unsuccessful Logons", - "family": "AC" - }, - { - "id": "ac-9.3", - "title": "Notification of Account Changes", - "family": "AC" - }, - { - "id": "ac-9.4", - "title": "Additional Logon Information", - "family": "AC" - }, - { - "id": "ac-10", - "title": "Concurrent Session Control", - "family": "AC" - }, - { - "id": "ac-11", - "title": "Device Lock", - "family": "AC" - }, - { - "id": "ac-11.1", - "title": "Pattern-hiding Displays", - "family": "AC" - }, - { - "id": "ac-12", - "title": "Session Termination", - "family": "AC" - }, - { - "id": "ac-12.1", - "title": "User-initiated Logouts", - "family": "AC" - }, - { - "id": "ac-12.2", - "title": "Termination Message", - "family": "AC" - }, - { - "id": "ac-12.3", - "title": "Timeout Warning Message", - "family": "AC" - }, - { - "id": "ac-13", - "title": "Supervision and Review \u2014 Access Control", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-14", - "title": "Permitted Actions Without Identification or Authentication", - "family": "AC" - }, - { - "id": "ac-14.1", - "title": "Necessary Uses", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-15", - "title": "Automated Marking", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-16", - "title": "Security and Privacy Attributes", - "family": "AC" - }, - { - "id": "ac-16.1", - "title": "Dynamic Attribute Association", - "family": "AC" - }, - { - "id": "ac-16.2", - "title": "Attribute Value Changes by Authorized Individuals", - "family": "AC" - }, - { - "id": "ac-16.3", - "title": "Maintenance of Attribute Associations by System", - "family": "AC" - }, - { - "id": "ac-16.4", - "title": "Association of Attributes by Authorized Individuals", - "family": "AC" - }, - { - "id": "ac-16.5", - "title": "Attribute Displays on Objects to Be Output", - "family": "AC" - }, - { - "id": "ac-16.6", - "title": "Maintenance of Attribute Association", - "family": "AC" - }, - { - "id": "ac-16.7", - "title": "Consistent Attribute Interpretation", - "family": "AC" - }, - { - "id": "ac-16.8", - "title": "Association Techniques and Technologies", - "family": "AC" - }, - { - "id": "ac-16.9", - "title": "Attribute Reassignment \u2014 Regrading Mechanisms", - "family": "AC" - }, - { - "id": "ac-16.10", - "title": "Attribute Configuration by Authorized Individuals", - "family": "AC" - }, - { - "id": "ac-17", - "title": "Remote Access", - "family": "AC" - }, - { - "id": "ac-17.1", - "title": "Monitoring and Control", - "family": "AC" - }, - { - "id": "ac-17.2", - "title": "Protection of Confidentiality and Integrity Using Encryption", - "family": "AC" - }, - { - "id": "ac-17.3", - "title": "Managed Access Control Points", - "family": "AC" - }, - { - "id": "ac-17.4", - "title": "Privileged Commands and Access", - "family": "AC" - }, - { - "id": "ac-17.5", - "title": "Monitoring for Unauthorized Connections", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-17.6", - "title": "Protection of Mechanism Information", - "family": "AC" - }, - { - "id": "ac-17.7", - "title": "Additional Protection for Security Function Access", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-17.8", - "title": "Disable Nonsecure Network Protocols", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-17.9", - "title": "Disconnect or Disable Access", - "family": "AC" - }, - { - "id": "ac-17.10", - "title": "Authenticate Remote Commands", - "family": "AC" - }, - { - "id": "ac-18", - "title": "Wireless Access", - "family": "AC" - }, - { - "id": "ac-18.1", - "title": "Authentication and Encryption", - "family": "AC" - }, - { - "id": "ac-18.2", - "title": "Monitoring Unauthorized Connections", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-18.3", - "title": "Disable Wireless Networking", - "family": "AC" - }, - { - "id": "ac-18.4", - "title": "Restrict Configurations by Users", - "family": "AC" - }, - { - "id": "ac-18.5", - "title": "Antennas and Transmission Power Levels", - "family": "AC" - }, - { - "id": "ac-19", - "title": "Access Control for Mobile Devices", - "family": "AC" - }, - { - "id": "ac-19.1", - "title": "Use of Writable and Portable Storage Devices", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-19.2", - "title": "Use of Personally Owned Portable Storage Devices", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-19.3", - "title": "Use of Portable Storage Devices with No Identifiable Owner", - "family": "AC", - "withdrawn": true - }, - { - "id": "ac-19.4", - "title": "Restrictions for Classified Information", - "family": "AC" - }, - { - "id": "ac-19.5", - "title": "Full Device or Container-based Encryption", - "family": "AC" - }, - { - "id": "ac-20", - "title": "Use of External Systems", - "family": "AC" - }, - { - "id": "ac-20.1", - "title": "Limits on Authorized Use", - "family": "AC" - }, - { - "id": "ac-20.2", - "title": "Portable Storage Devices \u2014 Restricted Use", - "family": "AC" - }, - { - "id": "ac-20.3", - "title": "Non-organizationally Owned Systems \u2014 Restricted Use", - "family": "AC" - }, - { - "id": "ac-20.4", - "title": "Network Accessible Storage Devices \u2014 Prohibited Use", - "family": "AC" - }, - { - "id": "ac-20.5", - "title": "Portable Storage Devices \u2014 Prohibited Use", - "family": "AC" - }, - { - "id": "ac-21", - "title": "Information Sharing", - "family": "AC" - }, - { - "id": "ac-21.1", - "title": "Automated Decision Support", - "family": "AC" - }, - { - "id": "ac-21.2", - "title": "Information Search and Retrieval", - "family": "AC" - }, - { - "id": "ac-22", - "title": "Publicly Accessible Content", - "family": "AC" - }, - { - "id": "ac-23", - "title": "Data Mining Protection", - "family": "AC" - }, - { - "id": "ac-24", - "title": "Access Control Decisions", - "family": "AC" - }, - { - "id": "ac-24.1", - "title": "Transmit Access Authorization Information", - "family": "AC" - }, - { - "id": "ac-24.2", - "title": "No User or Process Identity", - "family": "AC" - }, - { - "id": "ac-25", - "title": "Reference Monitor", - "family": "AC" - }, - { - "id": "at-1", - "title": "Policy and Procedures", - "family": "AT" - }, - { - "id": "at-2", - "title": "Literacy Training and Awareness", - "family": "AT" - }, - { - "id": "at-2.1", - "title": "Practical Exercises", - "family": "AT" - }, - { - "id": "at-2.2", - "title": "Insider Threat", - "family": "AT" - }, - { - "id": "at-2.3", - "title": "Social Engineering and Mining", - "family": "AT" - }, - { - "id": "at-2.4", - "title": "Suspicious Communications and Anomalous System Behavior", - "family": "AT" - }, - { - "id": "at-2.5", - "title": "Advanced Persistent Threat", - "family": "AT" - }, - { - "id": "at-2.6", - "title": "Cyber Threat Environment", - "family": "AT" - }, - { - "id": "at-3", - "title": "Role-based Training", - "family": "AT" - }, - { - "id": "at-3.1", - "title": "Environmental Controls", - "family": "AT" - }, - { - "id": "at-3.2", - "title": "Physical Security Controls", - "family": "AT" - }, - { - "id": "at-3.3", - "title": "Practical Exercises", - "family": "AT" - }, - { - "id": "at-3.4", - "title": "Suspicious Communications and Anomalous System Behavior", - "family": "AT", - "withdrawn": true - }, - { - "id": "at-3.5", - "title": "Processing Personally Identifiable Information", - "family": "AT" - }, - { - "id": "at-4", - "title": "Training Records", - "family": "AT" - }, - { - "id": "at-5", - "title": "Contacts with Security Groups and Associations", - "family": "AT", - "withdrawn": true - }, - { - "id": "at-6", - "title": "Training Feedback", - "family": "AT" - }, - { - "id": "au-1", - "title": "Policy and Procedures", - "family": "AU" - }, - { - "id": "au-2", - "title": "Event Logging", - "family": "AU" - }, - { - "id": "au-2.1", - "title": "Compilation of Audit Records from Multiple Sources", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-2.2", - "title": "Selection of Audit Events by Component", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-2.3", - "title": "Reviews and Updates", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-2.4", - "title": "Privileged Functions", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-3", - "title": "Content of Audit Records", - "family": "AU" - }, - { - "id": "au-3.1", - "title": "Additional Audit Information", - "family": "AU" - }, - { - "id": "au-3.2", - "title": "Centralized Management of Planned Audit Record Content", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-3.3", - "title": "Limit Personally Identifiable Information Elements", - "family": "AU" - }, - { - "id": "au-4", - "title": "Audit Log Storage Capacity", - "family": "AU" - }, - { - "id": "au-4.1", - "title": "Transfer to Alternate Storage", - "family": "AU" - }, - { - "id": "au-5", - "title": "Response to Audit Logging Process Failures", - "family": "AU" - }, - { - "id": "au-5.1", - "title": "Storage Capacity Warning", - "family": "AU" - }, - { - "id": "au-5.2", - "title": "Real-time Alerts", - "family": "AU" - }, - { - "id": "au-5.3", - "title": "Configurable Traffic Volume Thresholds", - "family": "AU" - }, - { - "id": "au-5.4", - "title": "Shutdown on Failure", - "family": "AU" - }, - { - "id": "au-5.5", - "title": "Alternate Audit Logging Capability", - "family": "AU" - }, - { - "id": "au-6", - "title": "Audit Record Review, Analysis, and Reporting", - "family": "AU" - }, - { - "id": "au-6.1", - "title": "Automated Process Integration", - "family": "AU" - }, - { - "id": "au-6.2", - "title": "Automated Security Alerts", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-6.3", - "title": "Correlate Audit Record Repositories", - "family": "AU" - }, - { - "id": "au-6.4", - "title": "Central Review and Analysis", - "family": "AU" - }, - { - "id": "au-6.5", - "title": "Integrated Analysis of Audit Records", - "family": "AU" - }, - { - "id": "au-6.6", - "title": "Correlation with Physical Monitoring", - "family": "AU" - }, - { - "id": "au-6.7", - "title": "Permitted Actions", - "family": "AU" - }, - { - "id": "au-6.8", - "title": "Full Text Analysis of Privileged Commands", - "family": "AU" - }, - { - "id": "au-6.9", - "title": "Correlation with Information from Nontechnical Sources", - "family": "AU" - }, - { - "id": "au-6.10", - "title": "Audit Level Adjustment", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-7", - "title": "Audit Record Reduction and Report Generation", - "family": "AU" - }, - { - "id": "au-7.1", - "title": "Automatic Processing", - "family": "AU" - }, - { - "id": "au-7.2", - "title": "Automatic Sort and Search", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-8", - "title": "Time Stamps", - "family": "AU" - }, - { - "id": "au-8.1", - "title": "Synchronization with Authoritative Time Source", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-8.2", - "title": "Secondary Authoritative Time Source", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-9", - "title": "Protection of Audit Information", - "family": "AU" - }, - { - "id": "au-9.1", - "title": "Hardware Write-once Media", - "family": "AU" - }, - { - "id": "au-9.2", - "title": "Store on Separate Physical Systems or Components", - "family": "AU" - }, - { - "id": "au-9.3", - "title": "Cryptographic Protection", - "family": "AU" - }, - { - "id": "au-9.4", - "title": "Access by Subset of Privileged Users", - "family": "AU" - }, - { - "id": "au-9.5", - "title": "Dual Authorization", - "family": "AU" - }, - { - "id": "au-9.6", - "title": "Read-only Access", - "family": "AU" - }, - { - "id": "au-9.7", - "title": "Store on Component with Different Operating System", - "family": "AU" - }, - { - "id": "au-10", - "title": "Non-repudiation", - "family": "AU" - }, - { - "id": "au-10.1", - "title": "Association of Identities", - "family": "AU" - }, - { - "id": "au-10.2", - "title": "Validate Binding of Information Producer Identity", - "family": "AU" - }, - { - "id": "au-10.3", - "title": "Chain of Custody", - "family": "AU" - }, - { - "id": "au-10.4", - "title": "Validate Binding of Information Reviewer Identity", - "family": "AU" - }, - { - "id": "au-10.5", - "title": "Digital Signatures", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-11", - "title": "Audit Record Retention", - "family": "AU" - }, - { - "id": "au-11.1", - "title": "Long-term Retrieval Capability", - "family": "AU" - }, - { - "id": "au-12", - "title": "Audit Record Generation", - "family": "AU" - }, - { - "id": "au-12.1", - "title": "System-wide and Time-correlated Audit Trail", - "family": "AU" - }, - { - "id": "au-12.2", - "title": "Standardized Formats", - "family": "AU" - }, - { - "id": "au-12.3", - "title": "Changes by Authorized Individuals", - "family": "AU" - }, - { - "id": "au-12.4", - "title": "Query Parameter Audits of Personally Identifiable Information", - "family": "AU" - }, - { - "id": "au-13", - "title": "Monitoring for Information Disclosure", - "family": "AU" - }, - { - "id": "au-13.1", - "title": "Use of Automated Tools", - "family": "AU" - }, - { - "id": "au-13.2", - "title": "Review of Monitored Sites", - "family": "AU" - }, - { - "id": "au-13.3", - "title": "Unauthorized Replication of Information", - "family": "AU" - }, - { - "id": "au-14", - "title": "Session Audit", - "family": "AU" - }, - { - "id": "au-14.1", - "title": "System Start-up", - "family": "AU" - }, - { - "id": "au-14.2", - "title": "Capture and Record Content", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-14.3", - "title": "Remote Viewing and Listening", - "family": "AU" - }, - { - "id": "au-15", - "title": "Alternate Audit Logging Capability", - "family": "AU", - "withdrawn": true - }, - { - "id": "au-16", - "title": "Cross-organizational Audit Logging", - "family": "AU" - }, - { - "id": "au-16.1", - "title": "Identity Preservation", - "family": "AU" - }, - { - "id": "au-16.2", - "title": "Sharing of Audit Information", - "family": "AU" - }, - { - "id": "au-16.3", - "title": "Disassociability", - "family": "AU" - }, - { - "id": "ca-1", - "title": "Policy and Procedures", - "family": "CA" - }, - { - "id": "ca-2", - "title": "Control Assessments", - "family": "CA" - }, - { - "id": "ca-2.1", - "title": "Independent Assessors", - "family": "CA" - }, - { - "id": "ca-2.2", - "title": "Specialized Assessments", - "family": "CA" - }, - { - "id": "ca-2.3", - "title": "Leveraging Results from External Organizations", - "family": "CA" - }, - { - "id": "ca-3", - "title": "Information Exchange", - "family": "CA" - }, - { - "id": "ca-3.1", - "title": "Unclassified National Security System Connections", - "family": "CA", - "withdrawn": true - }, - { - "id": "ca-3.2", - "title": "Classified National Security System Connections", - "family": "CA", - "withdrawn": true - }, - { - "id": "ca-3.3", - "title": "Unclassified Non-national Security System Connections", - "family": "CA", - "withdrawn": true - }, - { - "id": "ca-3.4", - "title": "Connections to Public Networks", - "family": "CA", - "withdrawn": true - }, - { - "id": "ca-3.5", - "title": "Restrictions on External System Connections", - "family": "CA", - "withdrawn": true - }, - { - "id": "ca-3.6", - "title": "Transfer Authorizations", - "family": "CA" - }, - { - "id": "ca-3.7", - "title": "Transitive Information Exchanges", - "family": "CA" - }, - { - "id": "ca-4", - "title": "Security Certification", - "family": "CA", - "withdrawn": true - }, - { - "id": "ca-5", - "title": "Plan of Action and Milestones", - "family": "CA" - }, - { - "id": "ca-5.1", - "title": "Automation Support for Accuracy and Currency", - "family": "CA" - }, - { - "id": "ca-6", - "title": "Authorization", - "family": "CA" - }, - { - "id": "ca-6.1", - "title": "Joint Authorization \u2014 Intra-organization", - "family": "CA" - }, - { - "id": "ca-6.2", - "title": "Joint Authorization \u2014 Inter-organization", - "family": "CA" - }, - { - "id": "ca-7", - "title": "Continuous Monitoring", - "family": "CA" - }, - { - "id": "ca-7.1", - "title": "Independent Assessment", - "family": "CA" - }, - { - "id": "ca-7.2", - "title": "Types of Assessments", - "family": "CA", - "withdrawn": true - }, - { - "id": "ca-7.3", - "title": "Trend Analyses", - "family": "CA" - }, - { - "id": "ca-7.4", - "title": "Risk Monitoring", - "family": "CA" - }, - { - "id": "ca-7.5", - "title": "Consistency Analysis", - "family": "CA" - }, - { - "id": "ca-7.6", - "title": "Automation Support for Monitoring", - "family": "CA" - }, - { - "id": "ca-8", - "title": "Penetration Testing", - "family": "CA" - }, - { - "id": "ca-8.1", - "title": "Independent Penetration Testing Agent or Team", - "family": "CA" - }, - { - "id": "ca-8.2", - "title": "Red Team Exercises", - "family": "CA" - }, - { - "id": "ca-8.3", - "title": "Facility Penetration Testing", - "family": "CA" - }, - { - "id": "ca-9", - "title": "Internal System Connections", - "family": "CA" - }, - { - "id": "ca-9.1", - "title": "Compliance Checks", - "family": "CA" - }, - { - "id": "cm-1", - "title": "Policy and Procedures", - "family": "CM" - }, - { - "id": "cm-2", - "title": "Baseline Configuration", - "family": "CM" - }, - { - "id": "cm-2.1", - "title": "Reviews and Updates", - "family": "CM", - "withdrawn": true - }, - { - "id": "cm-2.2", - "title": "Automation Support for Accuracy and Currency", - "family": "CM" - }, - { - "id": "cm-2.3", - "title": "Retention of Previous Configurations", - "family": "CM" - }, - { - "id": "cm-2.4", - "title": "Unauthorized Software", - "family": "CM", - "withdrawn": true - }, - { - "id": "cm-2.5", - "title": "Authorized Software", - "family": "CM", - "withdrawn": true - }, - { - "id": "cm-2.6", - "title": "Development and Test Environments", - "family": "CM" - }, - { - "id": "cm-2.7", - "title": "Configure Systems and Components for High-risk Areas", - "family": "CM" - }, - { - "id": "cm-3", - "title": "Configuration Change Control", - "family": "CM" - }, - { - "id": "cm-3.1", - "title": "Automated Documentation, Notification, and Prohibition of Changes", - "family": "CM" - }, - { - "id": "cm-3.2", - "title": "Testing, Validation, and Documentation of Changes", - "family": "CM" - }, - { - "id": "cm-3.3", - "title": "Automated Change Implementation", - "family": "CM" - }, - { - "id": "cm-3.4", - "title": "Security and Privacy Representatives", - "family": "CM" - }, - { - "id": "cm-3.5", - "title": "Automated Security Response", - "family": "CM" - }, - { - "id": "cm-3.6", - "title": "Cryptography Management", - "family": "CM" - }, - { - "id": "cm-3.7", - "title": "Review System Changes", - "family": "CM" - }, - { - "id": "cm-3.8", - "title": "Prevent or Restrict Configuration Changes", - "family": "CM" - }, - { - "id": "cm-4", - "title": "Impact Analyses", - "family": "CM" - }, - { - "id": "cm-4.1", - "title": "Separate Test Environments", - "family": "CM" - }, - { - "id": "cm-4.2", - "title": "Verification of Controls", - "family": "CM" - }, - { - "id": "cm-5", - "title": "Access Restrictions for Change", - "family": "CM" - }, - { - "id": "cm-5.1", - "title": "Automated Access Enforcement and Audit Records", - "family": "CM" - }, - { - "id": "cm-5.2", - "title": "Review System Changes", - "family": "CM", - "withdrawn": true - }, - { - "id": "cm-5.3", - "title": "Signed Components", - "family": "CM", - "withdrawn": true - }, - { - "id": "cm-5.4", - "title": "Dual Authorization", - "family": "CM" - }, - { - "id": "cm-5.5", - "title": "Privilege Limitation for Production and Operation", - "family": "CM" - }, - { - "id": "cm-5.6", - "title": "Limit Library Privileges", - "family": "CM" - }, - { - "id": "cm-5.7", - "title": "Automatic Implementation of Security Safeguards", - "family": "CM", - "withdrawn": true - }, - { - "id": "cm-6", - "title": "Configuration Settings", - "family": "CM" - }, - { - "id": "cm-6.1", - "title": "Automated Management, Application, and Verification", - "family": "CM" - }, - { - "id": "cm-6.2", - "title": "Respond to Unauthorized Changes", - "family": "CM" - }, - { - "id": "cm-6.3", - "title": "Unauthorized Change Detection", - "family": "CM", - "withdrawn": true - }, - { - "id": "cm-6.4", - "title": "Conformance Demonstration", - "family": "CM", - "withdrawn": true - }, - { - "id": "cm-7", - "title": "Least Functionality", - "family": "CM" - }, - { - "id": "cm-7.1", - "title": "Periodic Review", - "family": "CM" - }, - { - "id": "cm-7.2", - "title": "Prevent Program Execution", - "family": "CM" - }, - { - "id": "cm-7.3", - "title": "Registration Compliance", - "family": "CM" - }, - { - "id": "cm-7.4", - "title": "Unauthorized Software \u2014 Deny-by-exception", - "family": "CM" - }, - { - "id": "cm-7.5", - "title": "Authorized Software \u2014 Allow-by-exception", - "family": "CM" - }, - { - "id": "cm-7.6", - "title": "Confined Environments with Limited Privileges", - "family": "CM" - }, - { - "id": "cm-7.7", - "title": "Code Execution in Protected Environments", - "family": "CM" - }, - { - "id": "cm-7.8", - "title": "Binary or Machine Executable Code", - "family": "CM" - }, - { - "id": "cm-7.9", - "title": "Prohibiting The Use of Unauthorized Hardware", - "family": "CM" - }, - { - "id": "cm-8", - "title": "System Component Inventory", - "family": "CM" - }, - { - "id": "cm-8.1", - "title": "Updates During Installation and Removal", - "family": "CM" - }, - { - "id": "cm-8.2", - "title": "Automated Maintenance", - "family": "CM" - }, - { - "id": "cm-8.3", - "title": "Automated Unauthorized Component Detection", - "family": "CM" - }, - { - "id": "cm-8.4", - "title": "Accountability Information", - "family": "CM" - }, - { - "id": "cm-8.5", - "title": "No Duplicate Accounting of Components", - "family": "CM", - "withdrawn": true - }, - { - "id": "cm-8.6", - "title": "Assessed Configurations and Approved Deviations", - "family": "CM" - }, - { - "id": "cm-8.7", - "title": "Centralized Repository", - "family": "CM" - }, - { - "id": "cm-8.8", - "title": "Automated Location Tracking", - "family": "CM" - }, - { - "id": "cm-8.9", - "title": "Assignment of Components to Systems", - "family": "CM" - }, - { - "id": "cm-9", - "title": "Configuration Management Plan", - "family": "CM" - }, - { - "id": "cm-9.1", - "title": "Assignment of Responsibility", - "family": "CM" - }, - { - "id": "cm-10", - "title": "Software Usage Restrictions", - "family": "CM" - }, - { - "id": "cm-10.1", - "title": "Open-source Software", - "family": "CM" - }, - { - "id": "cm-11", - "title": "User-installed Software", - "family": "CM" - }, - { - "id": "cm-11.1", - "title": "Alerts for Unauthorized Installations", - "family": "CM", - "withdrawn": true - }, - { - "id": "cm-11.2", - "title": "Software Installation with Privileged Status", - "family": "CM" - }, - { - "id": "cm-11.3", - "title": "Automated Enforcement and Monitoring", - "family": "CM" - }, - { - "id": "cm-12", - "title": "Information Location", - "family": "CM" - }, - { - "id": "cm-12.1", - "title": "Automated Tools to Support Information Location", - "family": "CM" - }, - { - "id": "cm-13", - "title": "Data Action Mapping", - "family": "CM" - }, - { - "id": "cm-14", - "title": "Signed Components", - "family": "CM" - }, - { - "id": "cp-1", - "title": "Policy and Procedures", - "family": "CP" - }, - { - "id": "cp-2", - "title": "Contingency Plan", - "family": "CP" - }, - { - "id": "cp-2.1", - "title": "Coordinate with Related Plans", - "family": "CP" - }, - { - "id": "cp-2.2", - "title": "Capacity Planning", - "family": "CP" - }, - { - "id": "cp-2.3", - "title": "Resume Mission and Business Functions", - "family": "CP" - }, - { - "id": "cp-2.4", - "title": "Resume All Mission and Business Functions", - "family": "CP", - "withdrawn": true - }, - { - "id": "cp-2.5", - "title": "Continue Mission and Business Functions", - "family": "CP" - }, - { - "id": "cp-2.6", - "title": "Alternate Processing and Storage Sites", - "family": "CP" - }, - { - "id": "cp-2.7", - "title": "Coordinate with External Service Providers", - "family": "CP" - }, - { - "id": "cp-2.8", - "title": "Identify Critical Assets", - "family": "CP" - }, - { - "id": "cp-3", - "title": "Contingency Training", - "family": "CP" - }, - { - "id": "cp-3.1", - "title": "Simulated Events", - "family": "CP" - }, - { - "id": "cp-3.2", - "title": "Mechanisms Used in Training Environments", - "family": "CP" - }, - { - "id": "cp-4", - "title": "Contingency Plan Testing", - "family": "CP" - }, - { - "id": "cp-4.1", - "title": "Coordinate with Related Plans", - "family": "CP" - }, - { - "id": "cp-4.2", - "title": "Alternate Processing Site", - "family": "CP" - }, - { - "id": "cp-4.3", - "title": "Automated Testing", - "family": "CP" - }, - { - "id": "cp-4.4", - "title": "Full Recovery and Reconstitution", - "family": "CP" - }, - { - "id": "cp-4.5", - "title": "Self-challenge", - "family": "CP" - }, - { - "id": "cp-5", - "title": "Contingency Plan Update", - "family": "CP", - "withdrawn": true - }, - { - "id": "cp-6", - "title": "Alternate Storage Site", - "family": "CP" - }, - { - "id": "cp-6.1", - "title": "Separation from Primary Site", - "family": "CP" - }, - { - "id": "cp-6.2", - "title": "Recovery Time and Recovery Point Objectives", - "family": "CP" - }, - { - "id": "cp-6.3", - "title": "Accessibility", - "family": "CP" - }, - { - "id": "cp-7", - "title": "Alternate Processing Site", - "family": "CP" - }, - { - "id": "cp-7.1", - "title": "Separation from Primary Site", - "family": "CP" - }, - { - "id": "cp-7.2", - "title": "Accessibility", - "family": "CP" - }, - { - "id": "cp-7.3", - "title": "Priority of Service", - "family": "CP" - }, - { - "id": "cp-7.4", - "title": "Preparation for Use", - "family": "CP" - }, - { - "id": "cp-7.5", - "title": "Equivalent Information Security Safeguards", - "family": "CP", - "withdrawn": true - }, - { - "id": "cp-7.6", - "title": "Inability to Return to Primary Site", - "family": "CP" - }, - { - "id": "cp-8", - "title": "Telecommunications Services", - "family": "CP" - }, - { - "id": "cp-8.1", - "title": "Priority of Service Provisions", - "family": "CP" - }, - { - "id": "cp-8.2", - "title": "Single Points of Failure", - "family": "CP" - }, - { - "id": "cp-8.3", - "title": "Separation of Primary and Alternate Providers", - "family": "CP" - }, - { - "id": "cp-8.4", - "title": "Provider Contingency Plan", - "family": "CP" - }, - { - "id": "cp-8.5", - "title": "Alternate Telecommunication Service Testing", - "family": "CP" - }, - { - "id": "cp-9", - "title": "System Backup", - "family": "CP" - }, - { - "id": "cp-9.1", - "title": "Testing for Reliability and Integrity", - "family": "CP" - }, - { - "id": "cp-9.2", - "title": "Test Restoration Using Sampling", - "family": "CP" - }, - { - "id": "cp-9.3", - "title": "Separate Storage for Critical Information", - "family": "CP" - }, - { - "id": "cp-9.4", - "title": "Protection from Unauthorized Modification", - "family": "CP", - "withdrawn": true - }, - { - "id": "cp-9.5", - "title": "Transfer to Alternate Storage Site", - "family": "CP" - }, - { - "id": "cp-9.6", - "title": "Redundant Secondary System", - "family": "CP" - }, - { - "id": "cp-9.7", - "title": "Dual Authorization for Deletion or Destruction", - "family": "CP" - }, - { - "id": "cp-9.8", - "title": "Cryptographic Protection", - "family": "CP" - }, - { - "id": "cp-10", - "title": "System Recovery and Reconstitution", - "family": "CP" - }, - { - "id": "cp-10.1", - "title": "Contingency Plan Testing", - "family": "CP", - "withdrawn": true - }, - { - "id": "cp-10.2", - "title": "Transaction Recovery", - "family": "CP" - }, - { - "id": "cp-10.3", - "title": "Compensating Security Controls", - "family": "CP", - "withdrawn": true - }, - { - "id": "cp-10.4", - "title": "Restore Within Time Period", - "family": "CP" - }, - { - "id": "cp-10.5", - "title": "Failover Capability", - "family": "CP", - "withdrawn": true - }, - { - "id": "cp-10.6", - "title": "Component Protection", - "family": "CP" - }, - { - "id": "cp-11", - "title": "Alternate Communications Protocols", - "family": "CP" - }, - { - "id": "cp-12", - "title": "Safe Mode", - "family": "CP" - }, - { - "id": "cp-13", - "title": "Alternative Security Mechanisms", - "family": "CP" - }, - { - "id": "ia-1", - "title": "Policy and Procedures", - "family": "IA" - }, - { - "id": "ia-2", - "title": "Identification and Authentication (Organizational Users)", - "family": "IA" - }, - { - "id": "ia-2.1", - "title": "Multi-factor Authentication to Privileged Accounts", - "family": "IA" - }, - { - "id": "ia-2.2", - "title": "Multi-factor Authentication to Non-privileged Accounts", - "family": "IA" - }, - { - "id": "ia-2.3", - "title": "Local Access to Privileged Accounts", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-2.4", - "title": "Local Access to Non-privileged Accounts", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-2.5", - "title": "Individual Authentication with Group Authentication", - "family": "IA" - }, - { - "id": "ia-2.6", - "title": "Access to Accounts \u2014separate Device", - "family": "IA" - }, - { - "id": "ia-2.7", - "title": "Network Access to Non-privileged Accounts \u2014 Separate Device", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-2.8", - "title": "Access to Accounts \u2014 Replay Resistant", - "family": "IA" - }, - { - "id": "ia-2.9", - "title": "Network Access to Non-privileged Accounts \u2014 Replay Resistant", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-2.10", - "title": "Single Sign-on", - "family": "IA" - }, - { - "id": "ia-2.11", - "title": "Remote Access \u2014 Separate Device", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-2.12", - "title": "Acceptance of PIV Credentials", - "family": "IA" - }, - { - "id": "ia-2.13", - "title": "Out-of-band Authentication", - "family": "IA" - }, - { - "id": "ia-3", - "title": "Device Identification and Authentication", - "family": "IA" - }, - { - "id": "ia-3.1", - "title": "Cryptographic Bidirectional Authentication", - "family": "IA" - }, - { - "id": "ia-3.2", - "title": "Cryptographic Bidirectional Network Authentication", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-3.3", - "title": "Dynamic Address Allocation", - "family": "IA" - }, - { - "id": "ia-3.4", - "title": "Device Attestation", - "family": "IA" - }, - { - "id": "ia-4", - "title": "Identifier Management", - "family": "IA" - }, - { - "id": "ia-4.1", - "title": "Prohibit Account Identifiers as Public Identifiers", - "family": "IA" - }, - { - "id": "ia-4.2", - "title": "Supervisor Authorization", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-4.3", - "title": "Multiple Forms of Certification", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-4.4", - "title": "Identify User Status", - "family": "IA" - }, - { - "id": "ia-4.5", - "title": "Dynamic Management", - "family": "IA" - }, - { - "id": "ia-4.6", - "title": "Cross-organization Management", - "family": "IA" - }, - { - "id": "ia-4.7", - "title": "In-person Registration", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-4.8", - "title": "Pairwise Pseudonymous Identifiers", - "family": "IA" - }, - { - "id": "ia-4.9", - "title": "Attribute Maintenance and Protection", - "family": "IA" - }, - { - "id": "ia-5", - "title": "Authenticator Management", - "family": "IA" - }, - { - "id": "ia-5.1", - "title": "Password-based Authentication", - "family": "IA" - }, - { - "id": "ia-5.2", - "title": "Public Key-based Authentication", - "family": "IA" - }, - { - "id": "ia-5.3", - "title": "In-person or Trusted External Party Registration", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-5.4", - "title": "Automated Support for Password Strength Determination", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-5.5", - "title": "Change Authenticators Prior to Delivery", - "family": "IA" - }, - { - "id": "ia-5.6", - "title": "Protection of Authenticators", - "family": "IA" - }, - { - "id": "ia-5.7", - "title": "No Embedded Unencrypted Static Authenticators", - "family": "IA" - }, - { - "id": "ia-5.8", - "title": "Multiple System Accounts", - "family": "IA" - }, - { - "id": "ia-5.9", - "title": "Federated Credential Management", - "family": "IA" - }, - { - "id": "ia-5.10", - "title": "Dynamic Credential Binding", - "family": "IA" - }, - { - "id": "ia-5.11", - "title": "Hardware Token-based Authentication", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-5.12", - "title": "Biometric Authentication Performance", - "family": "IA" - }, - { - "id": "ia-5.13", - "title": "Expiration of Cached Authenticators", - "family": "IA" - }, - { - "id": "ia-5.14", - "title": "Managing Content of PKI Trust Stores", - "family": "IA" - }, - { - "id": "ia-5.15", - "title": "GSA-approved Products and Services", - "family": "IA" - }, - { - "id": "ia-5.16", - "title": "In-person or Trusted External Party Authenticator Issuance", - "family": "IA" - }, - { - "id": "ia-5.17", - "title": "Presentation Attack Detection for Biometric Authenticators", - "family": "IA" - }, - { - "id": "ia-5.18", - "title": "Password Managers", - "family": "IA" - }, - { - "id": "ia-6", - "title": "Authentication Feedback", - "family": "IA" - }, - { - "id": "ia-7", - "title": "Cryptographic Module Authentication", - "family": "IA" - }, - { - "id": "ia-8", - "title": "Identification and Authentication (Non-organizational Users)", - "family": "IA" - }, - { - "id": "ia-8.1", - "title": "Acceptance of PIV Credentials from Other Agencies", - "family": "IA" - }, - { - "id": "ia-8.2", - "title": "Acceptance of External Authenticators", - "family": "IA" - }, - { - "id": "ia-8.3", - "title": "Use of FICAM-approved Products", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-8.4", - "title": "Use of Defined Profiles", - "family": "IA" - }, - { - "id": "ia-8.5", - "title": "Acceptance of PIV-I Credentials", - "family": "IA" - }, - { - "id": "ia-8.6", - "title": "Disassociability", - "family": "IA" - }, - { - "id": "ia-9", - "title": "Service Identification and Authentication", - "family": "IA" - }, - { - "id": "ia-9.1", - "title": "Information Exchange", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-9.2", - "title": "Transmission of Decisions", - "family": "IA", - "withdrawn": true - }, - { - "id": "ia-10", - "title": "Adaptive Authentication", - "family": "IA" - }, - { - "id": "ia-11", - "title": "Re-authentication", - "family": "IA" - }, - { - "id": "ia-12", - "title": "Identity Proofing", - "family": "IA" - }, - { - "id": "ia-12.1", - "title": "Supervisor Authorization", - "family": "IA" - }, - { - "id": "ia-12.2", - "title": "Identity Evidence", - "family": "IA" - }, - { - "id": "ia-12.3", - "title": "Identity Evidence Validation and Verification", - "family": "IA" - }, - { - "id": "ia-12.4", - "title": "In-person Validation and Verification", - "family": "IA" - }, - { - "id": "ia-12.5", - "title": "Address Confirmation", - "family": "IA" - }, - { - "id": "ia-12.6", - "title": "Accept Externally-proofed Identities", - "family": "IA" - }, - { - "id": "ia-13", - "title": "Identity Providers and Authorization Servers", - "family": "IA" - }, - { - "id": "ia-13.1", - "title": "Protection of Cryptographic Keys", - "family": "IA" - }, - { - "id": "ia-13.2", - "title": "Verification of Identity Assertions and Access Tokens", - "family": "IA" - }, - { - "id": "ia-13.3", - "title": "Token Management", - "family": "IA" - }, - { - "id": "ir-1", - "title": "Policy and Procedures", - "family": "IR" - }, - { - "id": "ir-2", - "title": "Incident Response Training", - "family": "IR" - }, - { - "id": "ir-2.1", - "title": "Simulated Events", - "family": "IR" - }, - { - "id": "ir-2.2", - "title": "Automated Training Environments", - "family": "IR" - }, - { - "id": "ir-2.3", - "title": "Breach", - "family": "IR" - }, - { - "id": "ir-3", - "title": "Incident Response Testing", - "family": "IR" - }, - { - "id": "ir-3.1", - "title": "Automated Testing", - "family": "IR" - }, - { - "id": "ir-3.2", - "title": "Coordination with Related Plans", - "family": "IR" - }, - { - "id": "ir-3.3", - "title": "Continuous Improvement", - "family": "IR" - }, - { - "id": "ir-4", - "title": "Incident Handling", - "family": "IR" - }, - { - "id": "ir-4.1", - "title": "Automated Incident Handling Processes", - "family": "IR" - }, - { - "id": "ir-4.2", - "title": "Dynamic Reconfiguration", - "family": "IR" - }, - { - "id": "ir-4.3", - "title": "Continuity of Operations", - "family": "IR" - }, - { - "id": "ir-4.4", - "title": "Information Correlation", - "family": "IR" - }, - { - "id": "ir-4.5", - "title": "Automatic Disabling of System", - "family": "IR" - }, - { - "id": "ir-4.6", - "title": "Insider Threats", - "family": "IR" - }, - { - "id": "ir-4.7", - "title": "Insider Threats \u2014 Intra-organization Coordination", - "family": "IR" - }, - { - "id": "ir-4.8", - "title": "Correlation with External Organizations", - "family": "IR" - }, - { - "id": "ir-4.9", - "title": "Dynamic Response Capability", - "family": "IR" - }, - { - "id": "ir-4.10", - "title": "Supply Chain Coordination", - "family": "IR" - }, - { - "id": "ir-4.11", - "title": "Integrated Incident Response Team", - "family": "IR" - }, - { - "id": "ir-4.12", - "title": "Malicious Code and Forensic Analysis", - "family": "IR" - }, - { - "id": "ir-4.13", - "title": "Behavior Analysis", - "family": "IR" - }, - { - "id": "ir-4.14", - "title": "Security Operations Center", - "family": "IR" - }, - { - "id": "ir-4.15", - "title": "Public Relations and Reputation Repair", - "family": "IR" - }, - { - "id": "ir-5", - "title": "Incident Monitoring", - "family": "IR" - }, - { - "id": "ir-5.1", - "title": "Automated Tracking, Data Collection, and Analysis", - "family": "IR" - }, - { - "id": "ir-6", - "title": "Incident Reporting", - "family": "IR" - }, - { - "id": "ir-6.1", - "title": "Automated Reporting", - "family": "IR" - }, - { - "id": "ir-6.2", - "title": "Vulnerabilities Related to Incidents", - "family": "IR" - }, - { - "id": "ir-6.3", - "title": "Supply Chain Coordination", - "family": "IR" - }, - { - "id": "ir-7", - "title": "Incident Response Assistance", - "family": "IR" - }, - { - "id": "ir-7.1", - "title": "Automation Support for Availability of Information and Support", - "family": "IR" - }, - { - "id": "ir-7.2", - "title": "Coordination with External Providers", - "family": "IR" - }, - { - "id": "ir-8", - "title": "Incident Response Plan", - "family": "IR" - }, - { - "id": "ir-8.1", - "title": "Breaches", - "family": "IR" - }, - { - "id": "ir-9", - "title": "Information Spillage Response", - "family": "IR" - }, - { - "id": "ir-9.1", - "title": "Responsible Personnel", - "family": "IR", - "withdrawn": true - }, - { - "id": "ir-9.2", - "title": "Training", - "family": "IR" - }, - { - "id": "ir-9.3", - "title": "Post-spill Operations", - "family": "IR" - }, - { - "id": "ir-9.4", - "title": "Exposure to Unauthorized Personnel", - "family": "IR" - }, - { - "id": "ir-10", - "title": "Integrated Information Security Analysis Team", - "family": "IR", - "withdrawn": true - }, - { - "id": "ma-1", - "title": "Policy and Procedures", - "family": "MA" - }, - { - "id": "ma-2", - "title": "Controlled Maintenance", - "family": "MA" - }, - { - "id": "ma-2.1", - "title": "Record Content", - "family": "MA", - "withdrawn": true - }, - { - "id": "ma-2.2", - "title": "Automated Maintenance Activities", - "family": "MA" - }, - { - "id": "ma-3", - "title": "Maintenance Tools", - "family": "MA" - }, - { - "id": "ma-3.1", - "title": "Inspect Tools", - "family": "MA" - }, - { - "id": "ma-3.2", - "title": "Inspect Media", - "family": "MA" - }, - { - "id": "ma-3.3", - "title": "Prevent Unauthorized Removal", - "family": "MA" - }, - { - "id": "ma-3.4", - "title": "Restricted Tool Use", - "family": "MA" - }, - { - "id": "ma-3.5", - "title": "Execution with Privilege", - "family": "MA" - }, - { - "id": "ma-3.6", - "title": "Software Updates and Patches", - "family": "MA" - }, - { - "id": "ma-4", - "title": "Nonlocal Maintenance", - "family": "MA" - }, - { - "id": "ma-4.1", - "title": "Logging and Review", - "family": "MA" - }, - { - "id": "ma-4.2", - "title": "Document Nonlocal Maintenance", - "family": "MA", - "withdrawn": true - }, - { - "id": "ma-4.3", - "title": "Comparable Security and Sanitization", - "family": "MA" - }, - { - "id": "ma-4.4", - "title": "Authentication and Separation of Maintenance Sessions", - "family": "MA" - }, - { - "id": "ma-4.5", - "title": "Approvals and Notifications", - "family": "MA" - }, - { - "id": "ma-4.6", - "title": "Cryptographic Protection", - "family": "MA" - }, - { - "id": "ma-4.7", - "title": "Disconnect Verification", - "family": "MA" - }, - { - "id": "ma-5", - "title": "Maintenance Personnel", - "family": "MA" - }, - { - "id": "ma-5.1", - "title": "Individuals Without Appropriate Access", - "family": "MA" - }, - { - "id": "ma-5.2", - "title": "Security Clearances for Classified Systems", - "family": "MA" - }, - { - "id": "ma-5.3", - "title": "Citizenship Requirements for Classified Systems", - "family": "MA" - }, - { - "id": "ma-5.4", - "title": "Foreign Nationals", - "family": "MA" - }, - { - "id": "ma-5.5", - "title": "Non-system Maintenance", - "family": "MA" - }, - { - "id": "ma-6", - "title": "Timely Maintenance", - "family": "MA" - }, - { - "id": "ma-6.1", - "title": "Preventive Maintenance", - "family": "MA" - }, - { - "id": "ma-6.2", - "title": "Predictive Maintenance", - "family": "MA" - }, - { - "id": "ma-6.3", - "title": "Automated Support for Predictive Maintenance", - "family": "MA" - }, - { - "id": "ma-7", - "title": "Field Maintenance", - "family": "MA" - }, - { - "id": "mp-1", - "title": "Policy and Procedures", - "family": "MP" - }, - { - "id": "mp-2", - "title": "Media Access", - "family": "MP" - }, - { - "id": "mp-2.1", - "title": "Automated Restricted Access", - "family": "MP", - "withdrawn": true - }, - { - "id": "mp-2.2", - "title": "Cryptographic Protection", - "family": "MP", - "withdrawn": true - }, - { - "id": "mp-3", - "title": "Media Marking", - "family": "MP" - }, - { - "id": "mp-4", - "title": "Media Storage", - "family": "MP" - }, - { - "id": "mp-4.1", - "title": "Cryptographic Protection", - "family": "MP", - "withdrawn": true - }, - { - "id": "mp-4.2", - "title": "Automated Restricted Access", - "family": "MP" - }, - { - "id": "mp-5", - "title": "Media Transport", - "family": "MP" - }, - { - "id": "mp-5.1", - "title": "Protection Outside of Controlled Areas", - "family": "MP", - "withdrawn": true - }, - { - "id": "mp-5.2", - "title": "Documentation of Activities", - "family": "MP", - "withdrawn": true - }, - { - "id": "mp-5.3", - "title": "Custodians", - "family": "MP" - }, - { - "id": "mp-5.4", - "title": "Cryptographic Protection", - "family": "MP", - "withdrawn": true - }, - { - "id": "mp-6", - "title": "Media Sanitization", - "family": "MP" - }, - { - "id": "mp-6.1", - "title": "Review, Approve, Track, Document, and Verify", - "family": "MP" - }, - { - "id": "mp-6.2", - "title": "Equipment Testing", - "family": "MP" - }, - { - "id": "mp-6.3", - "title": "Nondestructive Techniques", - "family": "MP" - }, - { - "id": "mp-6.4", - "title": "Controlled Unclassified Information", - "family": "MP", - "withdrawn": true - }, - { - "id": "mp-6.5", - "title": "Classified Information", - "family": "MP", - "withdrawn": true - }, - { - "id": "mp-6.6", - "title": "Media Destruction", - "family": "MP", - "withdrawn": true - }, - { - "id": "mp-6.7", - "title": "Dual Authorization", - "family": "MP" - }, - { - "id": "mp-6.8", - "title": "Remote Purging or Wiping of Information", - "family": "MP" - }, - { - "id": "mp-7", - "title": "Media Use", - "family": "MP" - }, - { - "id": "mp-7.1", - "title": "Prohibit Use Without Owner", - "family": "MP", - "withdrawn": true - }, - { - "id": "mp-7.2", - "title": "Prohibit Use of Sanitization-resistant Media", - "family": "MP" - }, - { - "id": "mp-8", - "title": "Media Downgrading", - "family": "MP" - }, - { - "id": "mp-8.1", - "title": "Documentation of Process", - "family": "MP" - }, - { - "id": "mp-8.2", - "title": "Equipment Testing", - "family": "MP" - }, - { - "id": "mp-8.3", - "title": "Controlled Unclassified Information", - "family": "MP" - }, - { - "id": "mp-8.4", - "title": "Classified Information", - "family": "MP" - }, - { - "id": "pe-1", - "title": "Policy and Procedures", - "family": "PE" - }, - { - "id": "pe-2", - "title": "Physical Access Authorizations", - "family": "PE" - }, - { - "id": "pe-2.1", - "title": "Access by Position or Role", - "family": "PE" - }, - { - "id": "pe-2.2", - "title": "Two Forms of Identification", - "family": "PE" - }, - { - "id": "pe-2.3", - "title": "Restrict Unescorted Access", - "family": "PE" - }, - { - "id": "pe-3", - "title": "Physical Access Control", - "family": "PE" - }, - { - "id": "pe-3.1", - "title": "System Access", - "family": "PE" - }, - { - "id": "pe-3.2", - "title": "Facility and Systems", - "family": "PE" - }, - { - "id": "pe-3.3", - "title": "Continuous Guards", - "family": "PE" - }, - { - "id": "pe-3.4", - "title": "Lockable Casings", - "family": "PE" - }, - { - "id": "pe-3.5", - "title": "Tamper Protection", - "family": "PE" - }, - { - "id": "pe-3.6", - "title": "Facility Penetration Testing", - "family": "PE", - "withdrawn": true - }, - { - "id": "pe-3.7", - "title": "Physical Barriers", - "family": "PE" - }, - { - "id": "pe-3.8", - "title": "Access Control Vestibules", - "family": "PE" - }, - { - "id": "pe-4", - "title": "Access Control for Transmission", - "family": "PE" - }, - { - "id": "pe-5", - "title": "Access Control for Output Devices", - "family": "PE" - }, - { - "id": "pe-5.1", - "title": "Access to Output by Authorized Individuals", - "family": "PE", - "withdrawn": true - }, - { - "id": "pe-5.2", - "title": "Link to Individual Identity", - "family": "PE" - }, - { - "id": "pe-5.3", - "title": "Marking Output Devices", - "family": "PE", - "withdrawn": true - }, - { - "id": "pe-6", - "title": "Monitoring Physical Access", - "family": "PE" - }, - { - "id": "pe-6.1", - "title": "Intrusion Alarms and Surveillance Equipment", - "family": "PE" - }, - { - "id": "pe-6.2", - "title": "Automated Intrusion Recognition and Responses", - "family": "PE" - }, - { - "id": "pe-6.3", - "title": "Video Surveillance", - "family": "PE" - }, - { - "id": "pe-6.4", - "title": "Monitoring Physical Access to Systems", - "family": "PE" - }, - { - "id": "pe-7", - "title": "Visitor Control", - "family": "PE", - "withdrawn": true - }, - { - "id": "pe-8", - "title": "Visitor Access Records", - "family": "PE" - }, - { - "id": "pe-8.1", - "title": "Automated Records Maintenance and Review", - "family": "PE" - }, - { - "id": "pe-8.2", - "title": "Physical Access Records", - "family": "PE", - "withdrawn": true - }, - { - "id": "pe-8.3", - "title": "Limit Personally Identifiable Information Elements", - "family": "PE" - }, - { - "id": "pe-9", - "title": "Power Equipment and Cabling", - "family": "PE" - }, - { - "id": "pe-9.1", - "title": "Redundant Cabling", - "family": "PE" - }, - { - "id": "pe-9.2", - "title": "Automatic Voltage Controls", - "family": "PE" - }, - { - "id": "pe-10", - "title": "Emergency Shutoff", - "family": "PE" - }, - { - "id": "pe-10.1", - "title": "Accidental and Unauthorized Activation", - "family": "PE", - "withdrawn": true - }, - { - "id": "pe-11", - "title": "Emergency Power", - "family": "PE" - }, - { - "id": "pe-11.1", - "title": "Alternate Power Supply \u2014 Minimal Operational Capability", - "family": "PE" - }, - { - "id": "pe-11.2", - "title": "Alternate Power Supply \u2014 Self-contained", - "family": "PE" - }, - { - "id": "pe-12", - "title": "Emergency Lighting", - "family": "PE" - }, - { - "id": "pe-12.1", - "title": "Essential Mission and Business Functions", - "family": "PE" - }, - { - "id": "pe-13", - "title": "Fire Protection", - "family": "PE" - }, - { - "id": "pe-13.1", - "title": "Detection Systems \u2014 Automatic Activation and Notification", - "family": "PE" - }, - { - "id": "pe-13.2", - "title": "Suppression Systems \u2014 Automatic Activation and Notification", - "family": "PE" - }, - { - "id": "pe-13.3", - "title": "Automatic Fire Suppression", - "family": "PE", - "withdrawn": true - }, - { - "id": "pe-13.4", - "title": "Inspections", - "family": "PE" - }, - { - "id": "pe-14", - "title": "Environmental Controls", - "family": "PE" - }, - { - "id": "pe-14.1", - "title": "Automatic Controls", - "family": "PE" - }, - { - "id": "pe-14.2", - "title": "Monitoring with Alarms and Notifications", - "family": "PE" - }, - { - "id": "pe-15", - "title": "Water Damage Protection", - "family": "PE" - }, - { - "id": "pe-15.1", - "title": "Automation Support", - "family": "PE" - }, - { - "id": "pe-16", - "title": "Delivery and Removal", - "family": "PE" - }, - { - "id": "pe-17", - "title": "Alternate Work Site", - "family": "PE" - }, - { - "id": "pe-18", - "title": "Location of System Components", - "family": "PE" - }, - { - "id": "pe-18.1", - "title": "Facility Site", - "family": "PE", - "withdrawn": true - }, - { - "id": "pe-19", - "title": "Information Leakage", - "family": "PE" - }, - { - "id": "pe-19.1", - "title": "National Emissions Policies and Procedures", - "family": "PE" - }, - { - "id": "pe-20", - "title": "Asset Monitoring and Tracking", - "family": "PE" - }, - { - "id": "pe-21", - "title": "Electromagnetic Pulse Protection", - "family": "PE" - }, - { - "id": "pe-22", - "title": "Component Marking", - "family": "PE" - }, - { - "id": "pe-23", - "title": "Facility Location", - "family": "PE" - }, - { - "id": "pl-1", - "title": "Policy and Procedures", - "family": "PL" - }, - { - "id": "pl-2", - "title": "System Security and Privacy Plans", - "family": "PL" - }, - { - "id": "pl-2.1", - "title": "Concept of Operations", - "family": "PL", - "withdrawn": true - }, - { - "id": "pl-2.2", - "title": "Functional Architecture", - "family": "PL", - "withdrawn": true - }, - { - "id": "pl-2.3", - "title": "Plan and Coordinate with Other Organizational Entities", - "family": "PL", - "withdrawn": true - }, - { - "id": "pl-3", - "title": "System Security Plan Update", - "family": "PL", - "withdrawn": true - }, - { - "id": "pl-4", - "title": "Rules of Behavior", - "family": "PL" - }, - { - "id": "pl-4.1", - "title": "Social Media and External Site/Application Usage Restrictions", - "family": "PL" - }, - { - "id": "pl-5", - "title": "Privacy Impact Assessment", - "family": "PL", - "withdrawn": true - }, - { - "id": "pl-6", - "title": "Security-related Activity Planning", - "family": "PL", - "withdrawn": true - }, - { - "id": "pl-7", - "title": "Concept of Operations", - "family": "PL" - }, - { - "id": "pl-8", - "title": "Security and Privacy Architectures", - "family": "PL" - }, - { - "id": "pl-8.1", - "title": "Defense in Depth", - "family": "PL" - }, - { - "id": "pl-8.2", - "title": "Supplier Diversity", - "family": "PL" - }, - { - "id": "pl-9", - "title": "Central Management", - "family": "PL" - }, - { - "id": "pl-10", - "title": "Baseline Selection", - "family": "PL" - }, - { - "id": "pl-11", - "title": "Baseline Tailoring", - "family": "PL" - }, - { - "id": "pm-1", - "title": "Information Security Program Plan", - "family": "PM" - }, - { - "id": "pm-2", - "title": "Information Security Program Leadership Role", - "family": "PM" - }, - { - "id": "pm-3", - "title": "Information Security and Privacy Resources", - "family": "PM" - }, - { - "id": "pm-4", - "title": "Plan of Action and Milestones Process", - "family": "PM" - }, - { - "id": "pm-5", - "title": "System Inventory", - "family": "PM" - }, - { - "id": "pm-5.1", - "title": "Inventory of Personally Identifiable Information", - "family": "PM" - }, - { - "id": "pm-6", - "title": "Measures of Performance", - "family": "PM" - }, - { - "id": "pm-7", - "title": "Enterprise Architecture", - "family": "PM" - }, - { - "id": "pm-7.1", - "title": "Offloading", - "family": "PM" - }, - { - "id": "pm-8", - "title": "Critical Infrastructure Plan", - "family": "PM" - }, - { - "id": "pm-9", - "title": "Risk Management Strategy", - "family": "PM" - }, - { - "id": "pm-10", - "title": "Authorization Process", - "family": "PM" - }, - { - "id": "pm-11", - "title": "Mission and Business Process Definition", - "family": "PM" - }, - { - "id": "pm-12", - "title": "Insider Threat Program", - "family": "PM" - }, - { - "id": "pm-13", - "title": "Security and Privacy Workforce", - "family": "PM" - }, - { - "id": "pm-14", - "title": "Testing, Training, and Monitoring", - "family": "PM" - }, - { - "id": "pm-15", - "title": "Security and Privacy Groups and Associations", - "family": "PM" - }, - { - "id": "pm-16", - "title": "Threat Awareness Program", - "family": "PM" - }, - { - "id": "pm-16.1", - "title": "Automated Means for Sharing Threat Intelligence", - "family": "PM" - }, - { - "id": "pm-17", - "title": "Protecting Controlled Unclassified Information on External Systems", - "family": "PM" - }, - { - "id": "pm-18", - "title": "Privacy Program Plan", - "family": "PM" - }, - { - "id": "pm-19", - "title": "Privacy Program Leadership Role", - "family": "PM" - }, - { - "id": "pm-20", - "title": "Dissemination of Privacy Program Information", - "family": "PM" - }, - { - "id": "pm-20.1", - "title": "Privacy Policies on Websites, Applications, and Digital Services", - "family": "PM" - }, - { - "id": "pm-21", - "title": "Accounting of Disclosures", - "family": "PM" - }, - { - "id": "pm-22", - "title": "Personally Identifiable Information Quality Management", - "family": "PM" - }, - { - "id": "pm-23", - "title": "Data Governance Body", - "family": "PM" - }, - { - "id": "pm-24", - "title": "Data Integrity Board", - "family": "PM" - }, - { - "id": "pm-25", - "title": "Minimization of Personally Identifiable Information Used in Testing, Training, and Research", - "family": "PM" - }, - { - "id": "pm-26", - "title": "Complaint Management", - "family": "PM" - }, - { - "id": "pm-27", - "title": "Privacy Reporting", - "family": "PM" - }, - { - "id": "pm-28", - "title": "Risk Framing", - "family": "PM" - }, - { - "id": "pm-29", - "title": "Risk Management Program Leadership Roles", - "family": "PM" - }, - { - "id": "pm-30", - "title": "Supply Chain Risk Management Strategy", - "family": "PM" - }, - { - "id": "pm-30.1", - "title": "Suppliers of Critical or Mission-essential Items", - "family": "PM" - }, - { - "id": "pm-31", - "title": "Continuous Monitoring Strategy", - "family": "PM" - }, - { - "id": "pm-32", - "title": "Purposing", - "family": "PM" - }, - { - "id": "ps-1", - "title": "Policy and Procedures", - "family": "PS" - }, - { - "id": "ps-2", - "title": "Position Risk Designation", - "family": "PS" - }, - { - "id": "ps-3", - "title": "Personnel Screening", - "family": "PS" - }, - { - "id": "ps-3.1", - "title": "Classified Information", - "family": "PS" - }, - { - "id": "ps-3.2", - "title": "Formal Indoctrination", - "family": "PS" - }, - { - "id": "ps-3.3", - "title": "Information Requiring Special Protective Measures", - "family": "PS" - }, - { - "id": "ps-3.4", - "title": "Citizenship Requirements", - "family": "PS" - }, - { - "id": "ps-4", - "title": "Personnel Termination", - "family": "PS" - }, - { - "id": "ps-4.1", - "title": "Post-employment Requirements", - "family": "PS" - }, - { - "id": "ps-4.2", - "title": "Automated Actions", - "family": "PS" - }, - { - "id": "ps-5", - "title": "Personnel Transfer", - "family": "PS" - }, - { - "id": "ps-6", - "title": "Access Agreements", - "family": "PS" - }, - { - "id": "ps-6.1", - "title": "Information Requiring Special Protection", - "family": "PS", - "withdrawn": true - }, - { - "id": "ps-6.2", - "title": "Classified Information Requiring Special Protection", - "family": "PS" - }, - { - "id": "ps-6.3", - "title": "Post-employment Requirements", - "family": "PS" - }, - { - "id": "ps-7", - "title": "External Personnel Security", - "family": "PS" - }, - { - "id": "ps-8", - "title": "Personnel Sanctions", - "family": "PS" - }, - { - "id": "ps-9", - "title": "Position Descriptions", - "family": "PS" - }, - { - "id": "pt-1", - "title": "Policy and Procedures", - "family": "PT" - }, - { - "id": "pt-2", - "title": "Authority to Process Personally Identifiable Information", - "family": "PT" - }, - { - "id": "pt-2.1", - "title": "Data Tagging", - "family": "PT" - }, - { - "id": "pt-2.2", - "title": "Automation", - "family": "PT" - }, - { - "id": "pt-3", - "title": "Personally Identifiable Information Processing Purposes", - "family": "PT" - }, - { - "id": "pt-3.1", - "title": "Data Tagging", - "family": "PT" - }, - { - "id": "pt-3.2", - "title": "Automation", - "family": "PT" - }, - { - "id": "pt-4", - "title": "Consent", - "family": "PT" - }, - { - "id": "pt-4.1", - "title": "Tailored Consent", - "family": "PT" - }, - { - "id": "pt-4.2", - "title": "Just-in-time Consent", - "family": "PT" - }, - { - "id": "pt-4.3", - "title": "Revocation", - "family": "PT" - }, - { - "id": "pt-5", - "title": "Privacy Notice", - "family": "PT" - }, - { - "id": "pt-5.1", - "title": "Just-in-time Notice", - "family": "PT" - }, - { - "id": "pt-5.2", - "title": "Privacy Act Statements", - "family": "PT" - }, - { - "id": "pt-6", - "title": "System of Records Notice", - "family": "PT" - }, - { - "id": "pt-6.1", - "title": "Routine Uses", - "family": "PT" - }, - { - "id": "pt-6.2", - "title": "Exemption Rules", - "family": "PT" - }, - { - "id": "pt-7", - "title": "Specific Categories of Personally Identifiable Information", - "family": "PT" - }, - { - "id": "pt-7.1", - "title": "Social Security Numbers", - "family": "PT" - }, - { - "id": "pt-7.2", - "title": "First Amendment Information", - "family": "PT" - }, - { - "id": "pt-8", - "title": "Computer Matching Requirements", - "family": "PT" - }, - { - "id": "ra-1", - "title": "Policy and Procedures", - "family": "RA" - }, - { - "id": "ra-2", - "title": "Security Categorization", - "family": "RA" - }, - { - "id": "ra-2.1", - "title": "Impact-level Prioritization", - "family": "RA" - }, - { - "id": "ra-3", - "title": "Risk Assessment", - "family": "RA" - }, - { - "id": "ra-3.1", - "title": "Supply Chain Risk Assessment", - "family": "RA" - }, - { - "id": "ra-3.2", - "title": "Use of All-source Intelligence", - "family": "RA" - }, - { - "id": "ra-3.3", - "title": "Dynamic Threat Awareness", - "family": "RA" - }, - { - "id": "ra-3.4", - "title": "Predictive Cyber Analytics", - "family": "RA" - }, - { - "id": "ra-4", - "title": "Risk Assessment Update", - "family": "RA", - "withdrawn": true - }, - { - "id": "ra-5", - "title": "Vulnerability Monitoring and Scanning", - "family": "RA" - }, - { - "id": "ra-5.1", - "title": "Update Tool Capability", - "family": "RA", - "withdrawn": true - }, - { - "id": "ra-5.2", - "title": "Update Vulnerabilities to Be Scanned", - "family": "RA" - }, - { - "id": "ra-5.3", - "title": "Breadth and Depth of Coverage", - "family": "RA" - }, - { - "id": "ra-5.4", - "title": "Discoverable Information", - "family": "RA" - }, - { - "id": "ra-5.5", - "title": "Privileged Access", - "family": "RA" - }, - { - "id": "ra-5.6", - "title": "Automated Trend Analyses", - "family": "RA" - }, - { - "id": "ra-5.7", - "title": "Automated Detection and Notification of Unauthorized Components", - "family": "RA", - "withdrawn": true - }, - { - "id": "ra-5.8", - "title": "Review Historic Audit Logs", - "family": "RA" - }, - { - "id": "ra-5.9", - "title": "Penetration Testing and Analyses", - "family": "RA", - "withdrawn": true - }, - { - "id": "ra-5.10", - "title": "Correlate Scanning Information", - "family": "RA" - }, - { - "id": "ra-5.11", - "title": "Public Disclosure Program", - "family": "RA" - }, - { - "id": "ra-6", - "title": "Technical Surveillance Countermeasures Survey", - "family": "RA" - }, - { - "id": "ra-7", - "title": "Risk Response", - "family": "RA" - }, - { - "id": "ra-8", - "title": "Privacy Impact Assessments", - "family": "RA" - }, - { - "id": "ra-9", - "title": "Criticality Analysis", - "family": "RA" - }, - { - "id": "ra-10", - "title": "Threat Hunting", - "family": "RA" - }, - { - "id": "sa-1", - "title": "Policy and Procedures", - "family": "SA" - }, - { - "id": "sa-2", - "title": "Allocation of Resources", - "family": "SA" - }, - { - "id": "sa-3", - "title": "System Development Life Cycle", - "family": "SA" - }, - { - "id": "sa-3.1", - "title": "Manage Preproduction Environment", - "family": "SA" - }, - { - "id": "sa-3.2", - "title": "Use of Live or Operational Data", - "family": "SA" - }, - { - "id": "sa-3.3", - "title": "Technology Refresh", - "family": "SA" - }, - { - "id": "sa-4", - "title": "Acquisition Process", - "family": "SA" - }, - { - "id": "sa-4.1", - "title": "Functional Properties of Controls", - "family": "SA" - }, - { - "id": "sa-4.2", - "title": "Design and Implementation Information for Controls", - "family": "SA" - }, - { - "id": "sa-4.3", - "title": "Development Methods, Techniques, and Practices", - "family": "SA" - }, - { - "id": "sa-4.4", - "title": "Assignment of Components to Systems", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-4.5", - "title": "System, Component, and Service Configurations", - "family": "SA" - }, - { - "id": "sa-4.6", - "title": "Use of Information Assurance Products", - "family": "SA" - }, - { - "id": "sa-4.7", - "title": "NIAP-approved Protection Profiles ", - "family": "SA" - }, - { - "id": "sa-4.8", - "title": "Continuous Monitoring Plan for Controls", - "family": "SA" - }, - { - "id": "sa-4.9", - "title": "Functions, Ports, Protocols, and Services in Use", - "family": "SA" - }, - { - "id": "sa-4.10", - "title": "Use of Approved PIV Products", - "family": "SA" - }, - { - "id": "sa-4.11", - "title": "System of Records", - "family": "SA" - }, - { - "id": "sa-4.12", - "title": "Data Ownership", - "family": "SA" - }, - { - "id": "sa-5", - "title": "System Documentation", - "family": "SA" - }, - { - "id": "sa-5.1", - "title": "Functional Properties of Security Controls", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-5.2", - "title": "Security-relevant External System Interfaces", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-5.3", - "title": "High-level Design", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-5.4", - "title": "Low-level Design", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-5.5", - "title": "Source Code", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-6", - "title": "Software Usage Restrictions", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-7", - "title": "User-installed Software", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-8", - "title": "Security and Privacy Engineering Principles", - "family": "SA" - }, - { - "id": "sa-8.1", - "title": "Clear Abstractions", - "family": "SA" - }, - { - "id": "sa-8.2", - "title": "Least Common Mechanism", - "family": "SA" - }, - { - "id": "sa-8.3", - "title": "Modularity and Layering", - "family": "SA" - }, - { - "id": "sa-8.4", - "title": "Partially Ordered Dependencies", - "family": "SA" - }, - { - "id": "sa-8.5", - "title": "Efficiently Mediated Access", - "family": "SA" - }, - { - "id": "sa-8.6", - "title": "Minimized Sharing", - "family": "SA" - }, - { - "id": "sa-8.7", - "title": "Reduced Complexity", - "family": "SA" - }, - { - "id": "sa-8.8", - "title": "Secure Evolvability", - "family": "SA" - }, - { - "id": "sa-8.9", - "title": "Trusted Components", - "family": "SA" - }, - { - "id": "sa-8.10", - "title": "Hierarchical Trust", - "family": "SA" - }, - { - "id": "sa-8.11", - "title": "Inverse Modification Threshold", - "family": "SA" - }, - { - "id": "sa-8.12", - "title": "Hierarchical Protection", - "family": "SA" - }, - { - "id": "sa-8.13", - "title": "Minimized Security Elements", - "family": "SA" - }, - { - "id": "sa-8.14", - "title": "Least Privilege", - "family": "SA" - }, - { - "id": "sa-8.15", - "title": "Predicate Permission", - "family": "SA" - }, - { - "id": "sa-8.16", - "title": "Self-reliant Trustworthiness", - "family": "SA" - }, - { - "id": "sa-8.17", - "title": "Secure Distributed Composition", - "family": "SA" - }, - { - "id": "sa-8.18", - "title": "Trusted Communications Channels", - "family": "SA" - }, - { - "id": "sa-8.19", - "title": "Continuous Protection", - "family": "SA" - }, - { - "id": "sa-8.20", - "title": "Secure Metadata Management", - "family": "SA" - }, - { - "id": "sa-8.21", - "title": "Self-analysis", - "family": "SA" - }, - { - "id": "sa-8.22", - "title": "Accountability and Traceability", - "family": "SA" - }, - { - "id": "sa-8.23", - "title": "Secure Defaults", - "family": "SA" - }, - { - "id": "sa-8.24", - "title": "Secure Failure and Recovery", - "family": "SA" - }, - { - "id": "sa-8.25", - "title": "Economic Security", - "family": "SA" - }, - { - "id": "sa-8.26", - "title": "Performance Security", - "family": "SA" - }, - { - "id": "sa-8.27", - "title": "Human Factored Security", - "family": "SA" - }, - { - "id": "sa-8.28", - "title": "Acceptable Security", - "family": "SA" - }, - { - "id": "sa-8.29", - "title": "Repeatable and Documented Procedures", - "family": "SA" - }, - { - "id": "sa-8.30", - "title": "Procedural Rigor", - "family": "SA" - }, - { - "id": "sa-8.31", - "title": "Secure System Modification", - "family": "SA" - }, - { - "id": "sa-8.32", - "title": "Sufficient Documentation", - "family": "SA" - }, - { - "id": "sa-8.33", - "title": "Minimization", - "family": "SA" - }, - { - "id": "sa-9", - "title": "External System Services", - "family": "SA" - }, - { - "id": "sa-9.1", - "title": "Risk Assessments and Organizational Approvals", - "family": "SA" - }, - { - "id": "sa-9.2", - "title": "Identification of Functions, Ports, Protocols, and Services", - "family": "SA" - }, - { - "id": "sa-9.3", - "title": "Establish and Maintain Trust Relationship with Providers", - "family": "SA" - }, - { - "id": "sa-9.4", - "title": "Consistent Interests of Consumers and Providers", - "family": "SA" - }, - { - "id": "sa-9.5", - "title": "Processing, Storage, and Service Location", - "family": "SA" - }, - { - "id": "sa-9.6", - "title": "Organization-controlled Cryptographic Keys", - "family": "SA" - }, - { - "id": "sa-9.7", - "title": "Organization-controlled Integrity Checking", - "family": "SA" - }, - { - "id": "sa-9.8", - "title": "Processing and Storage Location \u2014 U.S. Jurisdiction", - "family": "SA" - }, - { - "id": "sa-10", - "title": "Developer Configuration Management", - "family": "SA" - }, - { - "id": "sa-10.1", - "title": "Software and Firmware Integrity Verification", - "family": "SA" - }, - { - "id": "sa-10.2", - "title": "Alternative Configuration Management Processes", - "family": "SA" - }, - { - "id": "sa-10.3", - "title": "Hardware Integrity Verification", - "family": "SA" - }, - { - "id": "sa-10.4", - "title": "Trusted Generation", - "family": "SA" - }, - { - "id": "sa-10.5", - "title": "Mapping Integrity for Version Control", - "family": "SA" - }, - { - "id": "sa-10.6", - "title": "Trusted Distribution", - "family": "SA" - }, - { - "id": "sa-10.7", - "title": "Security and Privacy Representatives", - "family": "SA" - }, - { - "id": "sa-11", - "title": "Developer Testing and Evaluation", - "family": "SA" - }, - { - "id": "sa-11.1", - "title": "Static Code Analysis", - "family": "SA" - }, - { - "id": "sa-11.2", - "title": "Threat Modeling and Vulnerability Analyses", - "family": "SA" - }, - { - "id": "sa-11.3", - "title": "Independent Verification of Assessment Plans and Evidence", - "family": "SA" - }, - { - "id": "sa-11.4", - "title": "Manual Code Reviews", - "family": "SA" - }, - { - "id": "sa-11.5", - "title": "Penetration Testing", - "family": "SA" - }, - { - "id": "sa-11.6", - "title": "Attack Surface Reviews", - "family": "SA" - }, - { - "id": "sa-11.7", - "title": "Verify Scope of Testing and Evaluation", - "family": "SA" - }, - { - "id": "sa-11.8", - "title": "Dynamic Code Analysis", - "family": "SA" - }, - { - "id": "sa-11.9", - "title": "Interactive Application Security Testing", - "family": "SA" - }, - { - "id": "sa-12", - "title": "Supply Chain Protection", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.1", - "title": "Acquisition Strategies / Tools / Methods", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.2", - "title": "Supplier Reviews", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.3", - "title": "Trusted Shipping and Warehousing", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.4", - "title": "Diversity of Suppliers", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.5", - "title": "Limitation of Harm", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.6", - "title": "Minimizing Procurement Time", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.7", - "title": "Assessments Prior to Selection / Acceptance / Update", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.8", - "title": "Use of All-source Intelligence", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.9", - "title": "Operations Security", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.10", - "title": "Validate as Genuine and Not Altered", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.11", - "title": "Penetration Testing / Analysis of Elements, Processes, and Actors", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.12", - "title": "Inter-organizational Agreements", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.13", - "title": "Critical Information System Components", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.14", - "title": "Identity and Traceability", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-12.15", - "title": "Processes to Address Weaknesses or Deficiencies", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-13", - "title": "Trustworthiness", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-14", - "title": "Criticality Analysis", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-14.1", - "title": "Critical Components with No Viable Alternative Sourcing", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-15", - "title": "Development Process, Standards, and Tools", - "family": "SA" - }, - { - "id": "sa-15.1", - "title": "Quality Metrics", - "family": "SA" - }, - { - "id": "sa-15.2", - "title": "Security and Privacy Tracking Tools", - "family": "SA" - }, - { - "id": "sa-15.3", - "title": "Criticality Analysis", - "family": "SA" - }, - { - "id": "sa-15.4", - "title": "Threat Modeling and Vulnerability Analysis", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-15.5", - "title": "Attack Surface Reduction", - "family": "SA" - }, - { - "id": "sa-15.6", - "title": "Continuous Improvement", - "family": "SA" - }, - { - "id": "sa-15.7", - "title": "Automated Vulnerability Analysis", - "family": "SA" - }, - { - "id": "sa-15.8", - "title": "Reuse of Threat and Vulnerability Information", - "family": "SA" - }, - { - "id": "sa-15.9", - "title": "Use of Live Data", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-15.10", - "title": "Incident Response Plan", - "family": "SA" - }, - { - "id": "sa-15.11", - "title": "Archive System or Component", - "family": "SA" - }, - { - "id": "sa-15.12", - "title": "Minimize Personally Identifiable Information", - "family": "SA" - }, - { - "id": "sa-15.13", - "title": "Logging Syntax", - "family": "SA" - }, - { - "id": "sa-16", - "title": "Developer-provided Training", - "family": "SA" - }, - { - "id": "sa-17", - "title": "Developer Security and Privacy Architecture and Design", - "family": "SA" - }, - { - "id": "sa-17.1", - "title": "Formal Policy Model", - "family": "SA" - }, - { - "id": "sa-17.2", - "title": "Security-relevant Components", - "family": "SA" - }, - { - "id": "sa-17.3", - "title": "Formal Correspondence", - "family": "SA" - }, - { - "id": "sa-17.4", - "title": "Informal Correspondence", - "family": "SA" - }, - { - "id": "sa-17.5", - "title": "Conceptually Simple Design", - "family": "SA" - }, - { - "id": "sa-17.6", - "title": "Structure for Testing", - "family": "SA" - }, - { - "id": "sa-17.7", - "title": "Structure for Least Privilege", - "family": "SA" - }, - { - "id": "sa-17.8", - "title": "Orchestration", - "family": "SA" - }, - { - "id": "sa-17.9", - "title": "Design Diversity", - "family": "SA" - }, - { - "id": "sa-18", - "title": "Tamper Resistance and Detection", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-18.1", - "title": "Multiple Phases of System Development Life Cycle", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-18.2", - "title": "Inspection of Systems or Components", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-19", - "title": "Component Authenticity", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-19.1", - "title": "Anti-counterfeit Training", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-19.2", - "title": "Configuration Control for Component Service and Repair", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-19.3", - "title": "Component Disposal", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-19.4", - "title": "Anti-counterfeit Scanning", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-20", - "title": "Customized Development of Critical Components", - "family": "SA" - }, - { - "id": "sa-21", - "title": "Developer Screening", - "family": "SA" - }, - { - "id": "sa-21.1", - "title": "Validation of Screening", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-22", - "title": "Unsupported System Components", - "family": "SA" - }, - { - "id": "sa-22.1", - "title": "Alternative Sources for Continued Support", - "family": "SA", - "withdrawn": true - }, - { - "id": "sa-23", - "title": "Specialization", - "family": "SA" - }, - { - "id": "sa-24", - "title": "Design For Cyber Resiliency", - "family": "SA" - }, - { - "id": "sc-1", - "title": "Policy and Procedures", - "family": "SC" - }, - { - "id": "sc-2", - "title": "Separation of System and User Functionality", - "family": "SC" - }, - { - "id": "sc-2.1", - "title": "Interfaces for Non-privileged Users", - "family": "SC" - }, - { - "id": "sc-2.2", - "title": "Disassociability", - "family": "SC" - }, - { - "id": "sc-3", - "title": "Security Function Isolation", - "family": "SC" - }, - { - "id": "sc-3.1", - "title": "Hardware Separation", - "family": "SC" - }, - { - "id": "sc-3.2", - "title": "Access and Flow Control Functions", - "family": "SC" - }, - { - "id": "sc-3.3", - "title": "Minimize Nonsecurity Functionality", - "family": "SC" - }, - { - "id": "sc-3.4", - "title": "Module Coupling and Cohesiveness", - "family": "SC" - }, - { - "id": "sc-3.5", - "title": "Layered Structures", - "family": "SC" - }, - { - "id": "sc-4", - "title": "Information in Shared System Resources", - "family": "SC" - }, - { - "id": "sc-4.1", - "title": "Security Levels", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-4.2", - "title": "Multilevel or Periods Processing", - "family": "SC" - }, - { - "id": "sc-5", - "title": "Denial-of-service Protection", - "family": "SC" - }, - { - "id": "sc-5.1", - "title": "Restrict Ability to Attack Other Systems", - "family": "SC" - }, - { - "id": "sc-5.2", - "title": "Capacity, Bandwidth, and Redundancy", - "family": "SC" - }, - { - "id": "sc-5.3", - "title": "Detection and Monitoring", - "family": "SC" - }, - { - "id": "sc-6", - "title": "Resource Availability", - "family": "SC" - }, - { - "id": "sc-7", - "title": "Boundary Protection", - "family": "SC" - }, - { - "id": "sc-7.1", - "title": "Physically Separated Subnetworks", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-7.2", - "title": "Public Access", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-7.3", - "title": "Access Points", - "family": "SC" - }, - { - "id": "sc-7.4", - "title": "External Telecommunications Services", - "family": "SC" - }, - { - "id": "sc-7.5", - "title": "Deny by Default \u2014 Allow by Exception", - "family": "SC" - }, - { - "id": "sc-7.6", - "title": "Response to Recognized Failures", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-7.7", - "title": "Split Tunneling for Remote Devices", - "family": "SC" - }, - { - "id": "sc-7.8", - "title": "Route Traffic to Authenticated Proxy Servers", - "family": "SC" - }, - { - "id": "sc-7.9", - "title": "Restrict Threatening Outgoing Communications Traffic", - "family": "SC" - }, - { - "id": "sc-7.10", - "title": "Prevent Exfiltration", - "family": "SC" - }, - { - "id": "sc-7.11", - "title": "Restrict Incoming Communications Traffic", - "family": "SC" - }, - { - "id": "sc-7.12", - "title": "Host-based Protection", - "family": "SC" - }, - { - "id": "sc-7.13", - "title": "Isolation of Security Tools, Mechanisms, and Support Components", - "family": "SC" - }, - { - "id": "sc-7.14", - "title": "Protect Against Unauthorized Physical Connections", - "family": "SC" - }, - { - "id": "sc-7.15", - "title": "Networked Privileged Accesses", - "family": "SC" - }, - { - "id": "sc-7.16", - "title": "Prevent Discovery of System Components", - "family": "SC" - }, - { - "id": "sc-7.17", - "title": "Automated Enforcement of Protocol Formats", - "family": "SC" - }, - { - "id": "sc-7.18", - "title": "Fail Secure", - "family": "SC" - }, - { - "id": "sc-7.19", - "title": "Block Communication from Non-organizationally Configured Hosts", - "family": "SC" - }, - { - "id": "sc-7.20", - "title": "Dynamic Isolation and Segregation", - "family": "SC" - }, - { - "id": "sc-7.21", - "title": "Isolation of System Components", - "family": "SC" - }, - { - "id": "sc-7.22", - "title": "Separate Subnets for Connecting to Different Security Domains", - "family": "SC" - }, - { - "id": "sc-7.23", - "title": "Disable Sender Feedback on Protocol Validation Failure", - "family": "SC" - }, - { - "id": "sc-7.24", - "title": "Personally Identifiable Information", - "family": "SC" - }, - { - "id": "sc-7.25", - "title": "Unclassified National Security System Connections", - "family": "SC" - }, - { - "id": "sc-7.26", - "title": "Classified National Security System Connections", - "family": "SC" - }, - { - "id": "sc-7.27", - "title": "Unclassified Non-national Security System Connections", - "family": "SC" - }, - { - "id": "sc-7.28", - "title": "Connections to Public Networks", - "family": "SC" - }, - { - "id": "sc-7.29", - "title": "Separate Subnets to Isolate Functions", - "family": "SC" - }, - { - "id": "sc-8", - "title": "Transmission Confidentiality and Integrity", - "family": "SC" - }, - { - "id": "sc-8.1", - "title": "Cryptographic Protection", - "family": "SC" - }, - { - "id": "sc-8.2", - "title": "Pre- and Post-transmission Handling", - "family": "SC" - }, - { - "id": "sc-8.3", - "title": "Cryptographic Protection for Message Externals", - "family": "SC" - }, - { - "id": "sc-8.4", - "title": "Conceal or Randomize Communications", - "family": "SC" - }, - { - "id": "sc-8.5", - "title": "Protected Distribution System", - "family": "SC" - }, - { - "id": "sc-9", - "title": "Transmission Confidentiality", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-10", - "title": "Network Disconnect", - "family": "SC" - }, - { - "id": "sc-11", - "title": "Trusted Path", - "family": "SC" - }, - { - "id": "sc-11.1", - "title": "Irrefutable Communications Path", - "family": "SC" - }, - { - "id": "sc-12", - "title": "Cryptographic Key Establishment and Management", - "family": "SC" - }, - { - "id": "sc-12.1", - "title": "Availability", - "family": "SC" - }, - { - "id": "sc-12.2", - "title": "Symmetric Keys", - "family": "SC" - }, - { - "id": "sc-12.3", - "title": "Asymmetric Keys", - "family": "SC" - }, - { - "id": "sc-12.4", - "title": "PKI Certificates", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-12.5", - "title": "PKI Certificates / Hardware Tokens", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-12.6", - "title": "Physical Control of Keys", - "family": "SC" - }, - { - "id": "sc-13", - "title": "Cryptographic Protection", - "family": "SC" - }, - { - "id": "sc-13.1", - "title": "FIPS-validated Cryptography", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-13.2", - "title": "NSA-approved Cryptography", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-13.3", - "title": "Individuals Without Formal Access Approvals", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-13.4", - "title": "Digital Signatures", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-14", - "title": "Public Access Protections", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-15", - "title": "Collaborative Computing Devices and Applications", - "family": "SC" - }, - { - "id": "sc-15.1", - "title": "Physical or Logical Disconnect", - "family": "SC" - }, - { - "id": "sc-15.2", - "title": "Blocking Inbound and Outbound Communications Traffic", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-15.3", - "title": "Disabling and Removal in Secure Work Areas", - "family": "SC" - }, - { - "id": "sc-15.4", - "title": "Explicitly Indicate Current Participants", - "family": "SC" - }, - { - "id": "sc-16", - "title": "Transmission of Security and Privacy Attributes", - "family": "SC" - }, - { - "id": "sc-16.1", - "title": "Integrity Verification", - "family": "SC" - }, - { - "id": "sc-16.2", - "title": "Anti-spoofing Mechanisms", - "family": "SC" - }, - { - "id": "sc-16.3", - "title": "Cryptographic Binding", - "family": "SC" - }, - { - "id": "sc-17", - "title": "Public Key Infrastructure Certificates", - "family": "SC" - }, - { - "id": "sc-18", - "title": "Mobile Code", - "family": "SC" - }, - { - "id": "sc-18.1", - "title": "Identify Unacceptable Code and Take Corrective Actions", - "family": "SC" - }, - { - "id": "sc-18.2", - "title": "Acquisition, Development, and Use", - "family": "SC" - }, - { - "id": "sc-18.3", - "title": "Prevent Downloading and Execution", - "family": "SC" - }, - { - "id": "sc-18.4", - "title": "Prevent Automatic Execution", - "family": "SC" - }, - { - "id": "sc-18.5", - "title": "Allow Execution Only in Confined Environments", - "family": "SC" - }, - { - "id": "sc-19", - "title": "Voice Over Internet Protocol", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-20", - "title": "Secure Name/Address Resolution Service (Authoritative Source)", - "family": "SC" - }, - { - "id": "sc-20.1", - "title": "Child Subspaces", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-20.2", - "title": "Data Origin and Integrity", - "family": "SC" - }, - { - "id": "sc-21", - "title": "Secure Name/Address Resolution Service (Recursive or Caching Resolver)", - "family": "SC" - }, - { - "id": "sc-21.1", - "title": "Data Origin and Integrity", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-22", - "title": "Architecture and Provisioning for Name/Address Resolution Service", - "family": "SC" - }, - { - "id": "sc-23", - "title": "Session Authenticity", - "family": "SC" - }, - { - "id": "sc-23.1", - "title": "Invalidate Session Identifiers at Logout", - "family": "SC" - }, - { - "id": "sc-23.2", - "title": "User-initiated Logouts and Message Displays", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-23.3", - "title": "Unique System-generated Session Identifiers", - "family": "SC" - }, - { - "id": "sc-23.4", - "title": "Unique Session Identifiers with Randomization", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-23.5", - "title": "Allowed Certificate Authorities", - "family": "SC" - }, - { - "id": "sc-24", - "title": "Fail in Known State", - "family": "SC" - }, - { - "id": "sc-25", - "title": "Thin Nodes", - "family": "SC" - }, - { - "id": "sc-26", - "title": "Decoys", - "family": "SC" - }, - { - "id": "sc-26.1", - "title": "Detection of Malicious Code", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-27", - "title": "Platform-independent Applications", - "family": "SC" - }, - { - "id": "sc-28", - "title": "Protection of Information at Rest", - "family": "SC" - }, - { - "id": "sc-28.1", - "title": "Cryptographic Protection", - "family": "SC" - }, - { - "id": "sc-28.2", - "title": "Offline Storage", - "family": "SC" - }, - { - "id": "sc-28.3", - "title": "Cryptographic Keys", - "family": "SC" - }, - { - "id": "sc-29", - "title": "Heterogeneity", - "family": "SC" - }, - { - "id": "sc-29.1", - "title": "Virtualization Techniques", - "family": "SC" - }, - { - "id": "sc-30", - "title": "Concealment and Misdirection", - "family": "SC" - }, - { - "id": "sc-30.1", - "title": "Virtualization Techniques", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-30.2", - "title": "Randomness", - "family": "SC" - }, - { - "id": "sc-30.3", - "title": "Change Processing and Storage Locations", - "family": "SC" - }, - { - "id": "sc-30.4", - "title": "Misleading Information", - "family": "SC" - }, - { - "id": "sc-30.5", - "title": "Concealment of System Components", - "family": "SC" - }, - { - "id": "sc-31", - "title": "Covert Channel Analysis", - "family": "SC" - }, - { - "id": "sc-31.1", - "title": "Test Covert Channels for Exploitability", - "family": "SC" - }, - { - "id": "sc-31.2", - "title": "Maximum Bandwidth", - "family": "SC" - }, - { - "id": "sc-31.3", - "title": "Measure Bandwidth in Operational Environments", - "family": "SC" - }, - { - "id": "sc-32", - "title": "System Partitioning", - "family": "SC" - }, - { - "id": "sc-32.1", - "title": "Separate Physical Domains for Privileged Functions", - "family": "SC" - }, - { - "id": "sc-33", - "title": "Transmission Preparation Integrity", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-34", - "title": "Non-modifiable Executable Programs", - "family": "SC" - }, - { - "id": "sc-34.1", - "title": "No Writable Storage", - "family": "SC" - }, - { - "id": "sc-34.2", - "title": "Integrity Protection on Read-only Media", - "family": "SC" - }, - { - "id": "sc-34.3", - "title": "Hardware-based Protection", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-35", - "title": "External Malicious Code Identification", - "family": "SC" - }, - { - "id": "sc-36", - "title": "Distributed Processing and Storage", - "family": "SC" - }, - { - "id": "sc-36.1", - "title": "Polling Techniques", - "family": "SC" - }, - { - "id": "sc-36.2", - "title": "Synchronization", - "family": "SC" - }, - { - "id": "sc-37", - "title": "Out-of-band Channels", - "family": "SC" - }, - { - "id": "sc-37.1", - "title": "Ensure Delivery and Transmission", - "family": "SC" - }, - { - "id": "sc-38", - "title": "Operations Security", - "family": "SC" - }, - { - "id": "sc-39", - "title": "Process Isolation", - "family": "SC" - }, - { - "id": "sc-39.1", - "title": "Hardware Separation", - "family": "SC" - }, - { - "id": "sc-39.2", - "title": "Separate Execution Domain Per Thread", - "family": "SC" - }, - { - "id": "sc-40", - "title": "Wireless Link Protection", - "family": "SC" - }, - { - "id": "sc-40.1", - "title": "Electromagnetic Interference", - "family": "SC" - }, - { - "id": "sc-40.2", - "title": "Reduce Detection Potential", - "family": "SC" - }, - { - "id": "sc-40.3", - "title": "Imitative or Manipulative Communications Deception", - "family": "SC" - }, - { - "id": "sc-40.4", - "title": "Signal Parameter Identification", - "family": "SC" - }, - { - "id": "sc-41", - "title": "Port and I/O Device Access", - "family": "SC" - }, - { - "id": "sc-42", - "title": "Sensor Capability and Data", - "family": "SC" - }, - { - "id": "sc-42.1", - "title": "Reporting to Authorized Individuals or Roles", - "family": "SC" - }, - { - "id": "sc-42.2", - "title": "Authorized Use", - "family": "SC" - }, - { - "id": "sc-42.3", - "title": "Prohibit Use of Devices", - "family": "SC", - "withdrawn": true - }, - { - "id": "sc-42.4", - "title": "Notice of Collection", - "family": "SC" - }, - { - "id": "sc-42.5", - "title": "Collection Minimization", - "family": "SC" - }, - { - "id": "sc-43", - "title": "Usage Restrictions", - "family": "SC" - }, - { - "id": "sc-44", - "title": "Detonation Chambers", - "family": "SC" - }, - { - "id": "sc-45", - "title": "System Time Synchronization", - "family": "SC" - }, - { - "id": "sc-45.1", - "title": "Synchronization with Authoritative Time Source", - "family": "SC" - }, - { - "id": "sc-45.2", - "title": "Secondary Authoritative Time Source", - "family": "SC" - }, - { - "id": "sc-46", - "title": "Cross Domain Policy Enforcement", - "family": "SC" - }, - { - "id": "sc-47", - "title": "Alternate Communications Paths", - "family": "SC" - }, - { - "id": "sc-48", - "title": "Sensor Relocation", - "family": "SC" - }, - { - "id": "sc-48.1", - "title": "Dynamic Relocation of Sensors or Monitoring Capabilities", - "family": "SC" - }, - { - "id": "sc-49", - "title": "Hardware-enforced Separation and Policy Enforcement", - "family": "SC" - }, - { - "id": "sc-50", - "title": "Software-enforced Separation and Policy Enforcement", - "family": "SC" - }, - { - "id": "sc-51", - "title": "Hardware-based Protection", - "family": "SC" - }, - { - "id": "si-1", - "title": "Policy and Procedures", - "family": "SI" - }, - { - "id": "si-2", - "title": "Flaw Remediation", - "family": "SI" - }, - { - "id": "si-2.1", - "title": "Central Management", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-2.2", - "title": "Automated Flaw Remediation Status", - "family": "SI" - }, - { - "id": "si-2.3", - "title": "Time to Remediate Flaws and Benchmarks for Corrective Actions", - "family": "SI" - }, - { - "id": "si-2.4", - "title": "Automated Patch Management Tools", - "family": "SI" - }, - { - "id": "si-2.5", - "title": "Automatic Software and Firmware Updates", - "family": "SI" - }, - { - "id": "si-2.6", - "title": "Removal of Previous Versions of Software and Firmware", - "family": "SI" - }, - { - "id": "si-2.7", - "title": "Root Cause Analysis", - "family": "SI" - }, - { - "id": "si-3", - "title": "Malicious Code Protection", - "family": "SI" - }, - { - "id": "si-3.1", - "title": "Central Management", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-3.2", - "title": "Automatic Updates", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-3.3", - "title": "Non-privileged Users", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-3.4", - "title": "Updates Only by Privileged Users", - "family": "SI" - }, - { - "id": "si-3.5", - "title": "Portable Storage Devices", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-3.6", - "title": "Testing and Verification", - "family": "SI" - }, - { - "id": "si-3.7", - "title": "Nonsignature-based Detection", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-3.8", - "title": "Detect Unauthorized Commands", - "family": "SI" - }, - { - "id": "si-3.9", - "title": "Authenticate Remote Commands", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-3.10", - "title": "Malicious Code Analysis", - "family": "SI" - }, - { - "id": "si-4", - "title": "System Monitoring", - "family": "SI" - }, - { - "id": "si-4.1", - "title": "System-wide Intrusion Detection System", - "family": "SI" - }, - { - "id": "si-4.2", - "title": "Automated Tools and Mechanisms for Real-time Analysis", - "family": "SI" - }, - { - "id": "si-4.3", - "title": "Automated Tool and Mechanism Integration", - "family": "SI" - }, - { - "id": "si-4.4", - "title": "Inbound and Outbound Communications Traffic", - "family": "SI" - }, - { - "id": "si-4.5", - "title": "System-generated Alerts", - "family": "SI" - }, - { - "id": "si-4.6", - "title": "Restrict Non-privileged Users", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-4.7", - "title": "Automated Response to Suspicious Events", - "family": "SI" - }, - { - "id": "si-4.8", - "title": "Protection of Monitoring Information", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-4.9", - "title": "Testing of Monitoring Tools and Mechanisms", - "family": "SI" - }, - { - "id": "si-4.10", - "title": "Visibility of Encrypted Communications", - "family": "SI" - }, - { - "id": "si-4.11", - "title": "Analyze Communications Traffic Anomalies", - "family": "SI" - }, - { - "id": "si-4.12", - "title": "Automated Organization-generated Alerts", - "family": "SI" - }, - { - "id": "si-4.13", - "title": "Analyze Traffic and Event Patterns", - "family": "SI" - }, - { - "id": "si-4.14", - "title": "Wireless Intrusion Detection", - "family": "SI" - }, - { - "id": "si-4.15", - "title": "Wireless to Wireline Communications", - "family": "SI" - }, - { - "id": "si-4.16", - "title": "Correlate Monitoring Information", - "family": "SI" - }, - { - "id": "si-4.17", - "title": "Integrated Situational Awareness", - "family": "SI" - }, - { - "id": "si-4.18", - "title": "Analyze Traffic and Covert Exfiltration", - "family": "SI" - }, - { - "id": "si-4.19", - "title": "Risk for Individuals", - "family": "SI" - }, - { - "id": "si-4.20", - "title": "Privileged Users", - "family": "SI" - }, - { - "id": "si-4.21", - "title": "Probationary Periods", - "family": "SI" - }, - { - "id": "si-4.22", - "title": "Unauthorized Network Services", - "family": "SI" - }, - { - "id": "si-4.23", - "title": "Host-based Devices", - "family": "SI" - }, - { - "id": "si-4.24", - "title": "Indicators of Compromise", - "family": "SI" - }, - { - "id": "si-4.25", - "title": "Optimize Network Traffic Analysis", - "family": "SI" - }, - { - "id": "si-5", - "title": "Security Alerts, Advisories, and Directives", - "family": "SI" - }, - { - "id": "si-5.1", - "title": "Automated Alerts and Advisories", - "family": "SI" - }, - { - "id": "si-6", - "title": "Security and Privacy Function Verification", - "family": "SI" - }, - { - "id": "si-6.1", - "title": "Notification of Failed Security Tests", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-6.2", - "title": "Automation Support for Distributed Testing", - "family": "SI" - }, - { - "id": "si-6.3", - "title": "Report Verification Results", - "family": "SI" - }, - { - "id": "si-7", - "title": "Software, Firmware, and Information Integrity", - "family": "SI" - }, - { - "id": "si-7.1", - "title": "Integrity Checks", - "family": "SI" - }, - { - "id": "si-7.2", - "title": "Automated Notifications of Integrity Violations", - "family": "SI" - }, - { - "id": "si-7.3", - "title": "Centrally Managed Integrity Tools", - "family": "SI" - }, - { - "id": "si-7.4", - "title": "Tamper-evident Packaging", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-7.5", - "title": "Automated Response to Integrity Violations", - "family": "SI" - }, - { - "id": "si-7.6", - "title": "Cryptographic Protection", - "family": "SI" - }, - { - "id": "si-7.7", - "title": "Integration of Detection and Response", - "family": "SI" - }, - { - "id": "si-7.8", - "title": "Auditing Capability for Significant Events", - "family": "SI" - }, - { - "id": "si-7.9", - "title": "Verify Boot Process", - "family": "SI" - }, - { - "id": "si-7.10", - "title": "Protection of Boot Firmware", - "family": "SI" - }, - { - "id": "si-7.11", - "title": "Confined Environments with Limited Privileges", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-7.12", - "title": "Integrity Verification", - "family": "SI" - }, - { - "id": "si-7.13", - "title": "Code Execution in Protected Environments", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-7.14", - "title": "Binary or Machine Executable Code", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-7.15", - "title": "Code Authentication", - "family": "SI" - }, - { - "id": "si-7.16", - "title": "Time Limit on Process Execution Without Supervision", - "family": "SI" - }, - { - "id": "si-7.17", - "title": "Runtime Application Self-protection", - "family": "SI" - }, - { - "id": "si-8", - "title": "Spam Protection", - "family": "SI" - }, - { - "id": "si-8.1", - "title": "Central Management", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-8.2", - "title": "Automatic Updates", - "family": "SI" - }, - { - "id": "si-8.3", - "title": "Continuous Learning Capability", - "family": "SI" - }, - { - "id": "si-9", - "title": "Information Input Restrictions", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-10", - "title": "Information Input Validation", - "family": "SI" - }, - { - "id": "si-10.1", - "title": "Manual Override Capability", - "family": "SI" - }, - { - "id": "si-10.2", - "title": "Review and Resolve Errors", - "family": "SI" - }, - { - "id": "si-10.3", - "title": "Predictable Behavior", - "family": "SI" - }, - { - "id": "si-10.4", - "title": "Timing Interactions", - "family": "SI" - }, - { - "id": "si-10.5", - "title": "Restrict Inputs to Trusted Sources and Approved Formats", - "family": "SI" - }, - { - "id": "si-10.6", - "title": "Injection Prevention", - "family": "SI" - }, - { - "id": "si-11", - "title": "Error Handling", - "family": "SI" - }, - { - "id": "si-12", - "title": "Information Management and Retention", - "family": "SI" - }, - { - "id": "si-12.1", - "title": "Limit Personally Identifiable Information Elements", - "family": "SI" - }, - { - "id": "si-12.2", - "title": "Minimize Personally Identifiable Information in Testing, Training, and Research", - "family": "SI" - }, - { - "id": "si-12.3", - "title": "Information Disposal", - "family": "SI" - }, - { - "id": "si-13", - "title": "Predictable Failure Prevention", - "family": "SI" - }, - { - "id": "si-13.1", - "title": "Transferring Component Responsibilities", - "family": "SI" - }, - { - "id": "si-13.2", - "title": "Time Limit on Process Execution Without Supervision", - "family": "SI", - "withdrawn": true - }, - { - "id": "si-13.3", - "title": "Manual Transfer Between Components", - "family": "SI" - }, - { - "id": "si-13.4", - "title": "Standby Component Installation and Notification", - "family": "SI" - }, - { - "id": "si-13.5", - "title": "Failover Capability", - "family": "SI" - }, - { - "id": "si-14", - "title": "Non-persistence", - "family": "SI" - }, - { - "id": "si-14.1", - "title": "Refresh from Trusted Sources", - "family": "SI" - }, - { - "id": "si-14.2", - "title": "Non-persistent Information", - "family": "SI" - }, - { - "id": "si-14.3", - "title": "Non-persistent Connectivity", - "family": "SI" - }, - { - "id": "si-15", - "title": "Information Output Filtering", - "family": "SI" - }, - { - "id": "si-16", - "title": "Memory Protection", - "family": "SI" - }, - { - "id": "si-17", - "title": "Fail-safe Procedures", - "family": "SI" - }, - { - "id": "si-18", - "title": "Personally Identifiable Information Quality Operations", - "family": "SI" - }, - { - "id": "si-18.1", - "title": "Automation Support", - "family": "SI" - }, - { - "id": "si-18.2", - "title": "Data Tags", - "family": "SI" - }, - { - "id": "si-18.3", - "title": "Collection", - "family": "SI" - }, - { - "id": "si-18.4", - "title": "Individual Requests", - "family": "SI" - }, - { - "id": "si-18.5", - "title": "Notice of Correction or Deletion", - "family": "SI" - }, - { - "id": "si-19", - "title": "De-identification", - "family": "SI" - }, - { - "id": "si-19.1", - "title": "Collection", - "family": "SI" - }, - { - "id": "si-19.2", - "title": "Archiving", - "family": "SI" - }, - { - "id": "si-19.3", - "title": "Release", - "family": "SI" - }, - { - "id": "si-19.4", - "title": "Removal, Masking, Encryption, Hashing, or Replacement of Direct Identifiers", - "family": "SI" - }, - { - "id": "si-19.5", - "title": "Statistical Disclosure Control", - "family": "SI" - }, - { - "id": "si-19.6", - "title": "Differential Privacy", - "family": "SI" - }, - { - "id": "si-19.7", - "title": "Validated Algorithms and Software", - "family": "SI" - }, - { - "id": "si-19.8", - "title": "Motivated Intruder", - "family": "SI" - }, - { - "id": "si-20", - "title": "Tainting", - "family": "SI" - }, - { - "id": "si-21", - "title": "Information Refresh", - "family": "SI" - }, - { - "id": "si-22", - "title": "Information Diversity", - "family": "SI" - }, - { - "id": "si-23", - "title": "Information Fragmentation", - "family": "SI" - }, - { - "id": "sr-1", - "title": "Policy and Procedures", - "family": "SR" - }, - { - "id": "sr-2", - "title": "Supply Chain Risk Management Plan", - "family": "SR" - }, - { - "id": "sr-2.1", - "title": "Establish SCRM Team", - "family": "SR" - }, - { - "id": "sr-3", - "title": "Supply Chain Controls and Processes", - "family": "SR" - }, - { - "id": "sr-3.1", - "title": "Diverse Supply Base", - "family": "SR" - }, - { - "id": "sr-3.2", - "title": "Limitation of Harm", - "family": "SR" - }, - { - "id": "sr-3.3", - "title": "Sub-tier Flow Down", - "family": "SR" - }, - { - "id": "sr-4", - "title": "Provenance", - "family": "SR" - }, - { - "id": "sr-4.1", - "title": "Identity", - "family": "SR" - }, - { - "id": "sr-4.2", - "title": "Track and Trace", - "family": "SR" - }, - { - "id": "sr-4.3", - "title": "Validate as Genuine and Not Altered", - "family": "SR" - }, - { - "id": "sr-4.4", - "title": "Supply Chain Integrity \u2014 Pedigree", - "family": "SR" - }, - { - "id": "sr-5", - "title": "Acquisition Strategies, Tools, and Methods", - "family": "SR" - }, - { - "id": "sr-5.1", - "title": "Adequate Supply", - "family": "SR" - }, - { - "id": "sr-5.2", - "title": "Assessments Prior to Selection, Acceptance, Modification, or Update", - "family": "SR" - }, - { - "id": "sr-6", - "title": "Supplier Assessments and Reviews", - "family": "SR" - }, - { - "id": "sr-6.1", - "title": "Testing and Analysis", - "family": "SR" - }, - { - "id": "sr-7", - "title": "Supply Chain Operations Security", - "family": "SR" - }, - { - "id": "sr-8", - "title": "Notification Agreements", - "family": "SR" - }, - { - "id": "sr-9", - "title": "Tamper Resistance and Detection", - "family": "SR" - }, - { - "id": "sr-9.1", - "title": "Multiple Stages of System Development Life Cycle", - "family": "SR" - }, - { - "id": "sr-10", - "title": "Inspection of Systems or Components", - "family": "SR" - }, - { - "id": "sr-11", - "title": "Component Authenticity", - "family": "SR" - }, - { - "id": "sr-11.1", - "title": "Anti-counterfeit Training", - "family": "SR" - }, - { - "id": "sr-11.2", - "title": "Configuration Control for Component Service and Repair", - "family": "SR" - }, - { - "id": "sr-11.3", - "title": "Anti-counterfeit Scanning", - "family": "SR" - }, - { - "id": "sr-12", - "title": "Component Disposal", - "family": "SR" - } - ] -} diff --git a/src/traust_engine/data/secure-code-audit/gitleaks-rules/fixtures/secrets-fixture-negative.txt b/src/traust_engine/data/secure-code-audit/gitleaks-rules/fixtures/secrets-fixture-negative.txt deleted file mode 100644 index b330eee..0000000 --- a/src/traust_engine/data/secure-code-audit/gitleaks-rules/fixtures/secrets-fixture-negative.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Negative calibration fixture: NO traust-* rule may fire on any line in -# this file (tests/test_run_gitleaks.py enforces it). Placeholder and -# templated shapes that look adjacent to the traust-dsn-url-credentials -# positives but must be dismissed at the detector layer. - -# templated / interpolated passwords -db: postgres://appuser:${DB_PASSWORD}@db.svc:5432/app -db: postgres://appuser:$(DB_PASSWORD)@db.svc:5432/app -db: postgres://appuser:$DB_PASSWORD@db.svc:5432/app -helm: postgres://{{ .Values.dbUser }}:{{ .Values.dbPassword }}@db.svc:5432/app -py: mysql://app:%(password)s@db/app -fmt: redis://user:%s@redis:6379/0 - -# angle-bracket and masked doc placeholders -doc: mongodb://:@mongo:27017/db -doc: amqp://svc:****@rabbitmq:5672/ -doc: mysql://app:xxxx@db/app - -# literal doc-example passwords -readme: postgres://user:pass@localhost:5432/db -readme: postgres://admin:password@localhost:5432/db -readme: mongodb://app:changeme@mongo:27017/db - -# no credentials at all -plain: postgres://db.svc:5432/app?sslmode=verify-full -userok: postgres://appuser@db.svc:5432/app diff --git a/src/traust_engine/data/secure-code-audit/gitleaks-rules/fixtures/secrets-fixture.txt b/src/traust_engine/data/secure-code-audit/gitleaks-rules/fixtures/secrets-fixture.txt deleted file mode 100644 index 95a4ef7..0000000 --- a/src/traust_engine/data/secure-code-audit/gitleaks-rules/fixtures/secrets-fixture.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Calibration fixture for the traust-* gitleaks rules. Every value below is -# SYNTHETIC — generated for this file, never a real credential. Each -# traust-* rule in ../gitleaks-default.toml must fire on its line here -# (tests/test_run_gitleaks.py enforces it). - -# traust-openshift-sha256-token -token: sha256~AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA - -# traust-dsn-url-credentials -DATABASE_URL=postgres://appuser:S3cretPass99@db.prod.example.com:5432/app -AMQP_URL: amqp://guest:guest@rabbitmq.svc:5672/ - -# traust-ansible-vault-payload -$ANSIBLE_VAULT;1.1;AES256 -66386439653236336462626566653063336164663966303231363934653561363964363833313662 diff --git a/src/traust_engine/data/secure-code-audit/gitleaks-rules/gitleaks-default.toml b/src/traust_engine/data/secure-code-audit/gitleaks-rules/gitleaks-default.toml deleted file mode 100644 index f5e0572..0000000 --- a/src/traust_engine/data/secure-code-audit/gitleaks-rules/gitleaks-default.toml +++ /dev/null @@ -1,67 +0,0 @@ -# Traust-owned gitleaks rules — traust-* extensions over the built-in set. -# -# Default config for the gitleaks adapter: upstream defaults plus the -# detectors below. Override via locations.yaml ``gitleaks_config``, the -# GITLEAKS_CONFIG env var, or ``--config``. -# -# Rule conventions: ids are traust-; each rule has a fixture line in -# fixtures/secrets-fixture.txt; tests/test_run_gitleaks.py asserts every -# traust-* rule fires on its fixture (calibration gate). - -title = "Traust gitleaks extension" - -[extend] -useDefault = true - -# OpenShift OAuth access tokens (`oc whoami -t` output): sha256~ prefix + -# 43 url-safe base64 chars. Live cluster credentials — highest liveness -# priority (routes to the `openshift` probe class). -[[rules]] -id = "traust-openshift-sha256-token" -description = "OpenShift OAuth access token (sha256~ bearer token)" -regex = '''sha256~[A-Za-z0-9_-]{43}''' -keywords = ["sha256~"] - -# Connection-string URLs with embedded credentials (postgres://user:pass@, -# mysql://, mongodb://, amqp://, redis://, …). Upstream gitleaks defaults -# have NO detector for this shape (verified against 8.30.1). Targets the -# config/DSN credential miss class: embedded credentials in helm values, -# docker-compose, .env files, and application config. Covers every file type -# gitleaks reads: yaml, .env, toml, properties, and code. Placeholder -# shapes (template vars, , :password@ doc examples) are -# allowlisted below; well-known DEFAULT credentials (guest:guest, -# admin:admin) are deliberately NOT allowlisted — shipped defaults are -# the miss class this rule closes. -[[rules]] -id = "traust-dsn-url-credentials" -description = "Connection-string URL with embedded credentials (DSN user:password@host)" -regex = '''(?i)\b(?:postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|amqps?|rediss?|mssql|sqlserver|jdbc:[a-z0-9]+|ldaps?|ftp|smtps?)://([A-Za-z0-9._%-]{1,64}):([^@/\s'"`]{2,128})@[A-Za-z0-9._-]+''' -secretGroup = 2 -keywords = ["://"] - -[rules.allowlist] -# Placeholder/templated passwords, not credentials: ${VAR}, $(VAR), $VAR, -# {{ template }}, , %s/%(var)s interpolation, ***/xxx -# masks, and literal doc-example passwords (:pass@ / :password@ / -# :changeme@ / :example@ / :secret@). -regexTarget = "secret" -regexes = [ - '''^\$''', - '''^\{\{.*\}\}$''', - '''^<[^>]*>$''', - '''^%(\([A-Za-z0-9_]+\))?s$''', - '''^\*{2,}$''', - '''^(?i)x{3,}$''', - '''^(?i)(pass|password|passwd|example|placeholder|redacted|dummy|your[_-]?password|change[_-]?me|changeme)$''', -] - -# Committed Ansible Vault payloads. The blob itself is encrypted, but a -# vault ciphertext in git history outlives every rotation of the vault -# password — and password files routinely leak alongside. Candidate for -# review, not proof of exposure (the audit agent weighs whether the vault -# password is co-committed or otherwise reachable). -[[rules]] -id = "traust-ansible-vault-payload" -description = "Ansible Vault ciphertext committed to the repository" -regex = '''\$ANSIBLE_VAULT;1\.[12];AES256''' -keywords = ["$ANSIBLE_VAULT"] diff --git a/src/traust_engine/depsdev/__init__.py b/src/traust_engine/depsdev/__init__.py deleted file mode 100644 index 0d0ccef..0000000 --- a/src/traust_engine/depsdev/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -"""deps.dev dependency resolution: one client, the OSV client's failure rule.""" - -from traust_engine.depsdev.client import ( - ATTRIBUTION, - DEFAULT_BASE_URL, - LICENSE, - SYSTEMS, - DepsDevClient, - Resolution, - normalize_base_url, -) - -__all__ = [ - "ATTRIBUTION", - "DEFAULT_BASE_URL", - "LICENSE", - "SYSTEMS", - "DepsDevClient", - "Resolution", - "normalize_base_url", -] diff --git a/src/traust_engine/depsdev/client.py b/src/traust_engine/depsdev/client.py deleted file mode 100644 index 158d441..0000000 --- a/src/traust_engine/depsdev/client.py +++ /dev/null @@ -1,263 +0,0 @@ -"""deps.dev dependency resolution: one client, the OSV client's failure rule. - -``GET /v3/systems/{system}/packages/{name}/versions/{version}:dependencies`` -returns the resolved dependency graph of one package version, "similar to -one produced by installing the package version on a generic 64-bit Linux -system, with no other dependencies present". Available for npm, PyPI, Maven -and Cargo. - -**A failed lookup is never an empty graph.** Each answer is a -``Resolution`` whose ``status`` is ``ok`` or ``not-found`` (HTTP 404: the -registry has no such version), or one of the OSV client's failure -statuses: ``unreachable``, ``quota-deferred`` or ``error``. - -**Terms.** Use of the API is subject to the Google APIs Terms of Service -(https://developers.google.com/terms), which bind whoever runs it. Data -deps.dev generates, which includes resolved dependency graphs, is licensed -CC-BY 4.0; ``ATTRIBUTION`` is the notice every consumer carries. deps.dev -expressly permits clients to cache data served by the API, hence the -optional on-disk cache. -""" - -from __future__ import annotations - -import hashlib -import json -import time -import urllib.error -import urllib.request -from collections import Counter -from collections.abc import Callable, Iterable -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field -from pathlib import Path -from urllib.parse import quote - -DEFAULT_BASE_URL = "https://api.deps.dev/v3" -USER_AGENT = "traust-engine-depsdev-client" -LICENSE = "CC-BY-4.0" -ATTRIBUTION = ( - "Resolved dependency data: deps.dev (Google LLC), https://deps.dev, licensed under " - "CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/); used unmodified as " - "package/version pairs. API use is subject to the Google APIs Terms of Service." -) -# OSV ecosystem -> deps.dev system -SYSTEMS = {"npm": "NPM", "PyPI": "PYPI", "Maven": "MAVEN", "crates.io": "CARGO"} - - -def normalize_base_url(url: str | None) -> str: - """Accept the base or a full ``…/v3/systems/…`` template; return ``…/v3``.""" - base = (url or DEFAULT_BASE_URL).rstrip("/") - return base.split("/systems/", 1)[0].removesuffix("/systems") - - -@dataclass -class Resolution: - status: str - nodes: list[tuple[str, str, str]] = field(default_factory=list) # (name, version, relation) - detail: str = "" - cached: bool = False - - @property - def ok(self) -> bool: - return self.status == "ok" - - -class _Failure(Exception): - def __init__(self, status: str, detail: str): - super().__init__(detail) - self.status = status - - -class DepsDevClient: - def __init__( - self, - base_url: str | None = None, - *, - timeout: float = 30.0, - retries: int = 3, - backoff: float = 2.0, - workers: int = 4, - cache_dir: Path | None = None, - cache_ttl_hours: float = 24.0, - opener: Callable | None = None, - sleep: Callable[[float], None] = time.sleep, - clock: Callable[[], float] = time.time, - ) -> None: - self.base_url = normalize_base_url(base_url) - self.timeout = timeout - self.retries = max(1, retries) - self.backoff = backoff - self.workers = max(1, workers) - self.cache_dir = Path(cache_dir) if cache_dir else None - self.cache_ttl = cache_ttl_hours * 3600 - self._open = opener or urllib.request.urlopen - self._sleep = sleep - self._clock = clock - - # -- transport --------------------------------------------------------- - - def _get(self, url: str) -> dict: - last: _Failure | None = None - for attempt in range(self.retries): - req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) - try: - with self._open(req, timeout=self.timeout) as resp: - raw = resp.read() - try: - doc = json.loads(raw) - except (json.JSONDecodeError, ValueError) as e: - raise _Failure("error", f"unparseable response from {url}: {e}") from None - if not isinstance(doc, dict): - raise _Failure("error", f"unexpected response shape from {url}") - return doc - except urllib.error.HTTPError as e: - if e.code == 404: - raise _Failure("not-found", f"HTTP 404 from {url}") from None - if e.code == 429: - last = _Failure("quota-deferred", f"HTTP 429 from {url}") - elif e.code >= 500: - last = _Failure("unreachable", f"HTTP {e.code} from {url}") - else: - raise _Failure("error", f"HTTP {e.code} from {url}") from None - except (urllib.error.URLError, TimeoutError, OSError) as e: - last = _Failure("unreachable", f"{type(e).__name__} reaching {url}") - if attempt + 1 < self.retries: - self._sleep(self.backoff * (attempt + 1)) - assert last is not None - raise last - - def url(self, ecosystem: str, name: str, version: str) -> str: - system = SYSTEMS[ecosystem] - return ( - f"{self.base_url}/systems/{system.lower()}/packages/{quote(name, safe='')}" - f"/versions/{quote(version, safe='')}:dependencies" - ) - - # -- cache ------------------------------------------------------------- - - def _cache_path(self, key: tuple[str, str, str]) -> Path | None: - if self.cache_dir is None: - return None - digest = hashlib.sha256("\x00".join(key).encode()).hexdigest() - return self.cache_dir / f"{digest}.json" - - def _cached(self, key: tuple[str, str, str]) -> Resolution | None: - path = self._cache_path(key) - if path is None: - return None - try: - doc = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return None - if doc.get("key") != list(key) or self._clock() - doc.get("fetched_at", 0) > self.cache_ttl: - return None - nodes = [tuple(n) for n in doc.get("nodes") or []] - return Resolution(doc.get("status", "error"), nodes, cached=True) - - def _store(self, key: tuple[str, str, str], res: Resolution) -> None: - path = self._cache_path(key) - if path is None or res.status not in ("ok", "not-found"): - return # failures are never cached - try: - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_suffix(".tmp") - tmp.write_text( - json.dumps( - { - "key": list(key), - "fetched_at": self._clock(), - "status": res.status, - "nodes": [list(n) for n in res.nodes], - } - ), - encoding="utf-8", - ) - tmp.replace(path) - except OSError: - pass # a cache that cannot be written only costs a refetch - - # -- queries ----------------------------------------------------------- - - def dependencies(self, ecosystem: str, name: str, version: str) -> Resolution: - """The resolved graph of one package version (``SELF`` node excluded).""" - key = (ecosystem, name, version) - if ecosystem not in SYSTEMS: - return Resolution("unsupported", detail=f"deps.dev does not resolve {ecosystem}") - hit = self._cached(key) - if hit is not None: - return hit - try: - doc = self._get(self.url(*key)) - except _Failure as f: - res = Resolution(f.status, detail=str(f)) - else: - nodes = [] - for node in doc.get("nodes") or []: - vk = (node or {}).get("versionKey") or {} - relation = node.get("relation") or "" - if relation == "SELF" or not vk.get("name") or not vk.get("version"): - continue - nodes.append((vk["name"], vk["version"], relation)) - res = Resolution("ok", sorted(set(nodes))) - self._store(key, res) - return res - - def default_version(self, ecosystem: str, name: str) -> Resolution: - """The version deps.dev marks default (what an install gets today). - - ``nodes`` holds one ``(name, version, "DEFAULT")`` entry when found. - """ - key = (ecosystem, name, "") - if ecosystem not in SYSTEMS: - return Resolution("unsupported", detail=f"deps.dev does not resolve {ecosystem}") - hit = self._cached(key) - if hit is not None: - return hit - url = ( - f"{self.base_url}/systems/{SYSTEMS[ecosystem].lower()}/packages/{quote(name, safe='')}" - ) - try: - doc = self._get(url) - except _Failure as f: - res = Resolution(f.status, detail=str(f)) - else: - default = next( - ( - (v.get("versionKey") or {}).get("version") - for v in doc.get("versions") or [] - if isinstance(v, dict) and v.get("isDefault") - ), - None, - ) - res = ( - Resolution("ok", [(name, default, "DEFAULT")]) - if default - else Resolution("not-found", detail=f"no default version for {name}") - ) - self._store(key, res) - return res - - def defaults_many(self, names: Iterable[tuple[str, str]]) -> dict[tuple[str, str], Resolution]: - unique = sorted(set(names)) - with ThreadPoolExecutor(max_workers=self.workers) as pool: - return dict( - zip(unique, pool.map(lambda k: self.default_version(*k), unique), strict=True) - ) - - def probe(self, key: tuple[str, str, str]) -> tuple[str, str] | None: - """One lookup before a run: ``None`` if deps.dev answers (404 counts).""" - res = self.dependencies(*key) - if res.status in ("ok", "not-found", "unsupported"): - return None - return res.status, res.detail - - def resolve_many( - self, keys: Iterable[tuple[str, str, str]] - ) -> tuple[dict[tuple[str, str, str], Resolution], Counter]: - unique = sorted(set(keys)) - with ThreadPoolExecutor(max_workers=self.workers) as pool: - results = dict( - zip(unique, pool.map(lambda k: self.dependencies(*k), unique), strict=True) - ) - return results, Counter(r.status for r in results.values()) diff --git a/src/traust_engine/engine.py b/src/traust_engine/engine.py index 7f41378..51ded2b 100644 --- a/src/traust_engine/engine.py +++ b/src/traust_engine/engine.py @@ -11,8 +11,11 @@ h.metrics.append("census", {"repos": 812}) ok, issues = h.metrics.verify() -Namespaces mirror the engine's submodules; `.metrics` is the first migrated. -Others (`.models`, `.compliance`, `.sweep`, …) follow the same shape. +Namespaces mirror the engine's submodules: `.metrics`, `.models`, `.adapters` (the rule-lane +stage's opengrep config), `.sweep`, `.portfolio`. + +PENDING(decision: sweep stages | portfolio graph layers | metrics journal) — kept for traust; +deleted when traust retires or ports it. """ from __future__ import annotations @@ -23,13 +26,10 @@ from traust_contracts import HarnessContext, load_context from traust_engine.adapters.ops import AdaptersOps -from traust_engine.compliance.ops import ComplianceOps from traust_engine.metrics.ops import MetricsOps from traust_engine.portfolio.ops import PortfolioOps from traust_engine.registry.ops import ModelsOps -from traust_engine.reporting.ops import ReportingOps from traust_engine.sweep.ops import SweepOps -from traust_engine.toolchain.ops import ToolchainOps class HarnessEngine: @@ -53,10 +53,6 @@ def metrics(self) -> MetricsOps: def models(self) -> ModelsOps: return ModelsOps(self._ctx) - @cached_property - def compliance(self) -> ComplianceOps: - return ComplianceOps(self._ctx) - @cached_property def adapters(self) -> AdaptersOps: return AdaptersOps(self._ctx) @@ -65,14 +61,6 @@ def adapters(self) -> AdaptersOps: def sweep(self) -> SweepOps: return SweepOps(self._ctx) - @cached_property - def reporting(self) -> ReportingOps: - return ReportingOps(self._ctx) - @cached_property def portfolio(self) -> PortfolioOps: return PortfolioOps(self._ctx) - - @cached_property - def toolchain(self) -> ToolchainOps: - return ToolchainOps(self._ctx) diff --git a/src/traust_engine/escaping.py b/src/traust_engine/escaping.py deleted file mode 100644 index ca78a70..0000000 --- a/src/traust_engine/escaping.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Shared escaping helpers for untrusted text reaching emitters. - -Public, zero-coupling text-safety helpers — safe for app emitters to import -directly as ``from traust_engine.escaping import md_cell``. - -Threat model per helper: - -- ``esc_html`` — text node / attribute value in generated HTML. -- ``json_script`` — JSON embedded in an inline ```` terminates the element early - regardless of JSON validity; ``