diff --git a/scripts/measure_sbom_coverage.py b/scripts/measure_sbom_coverage.py new file mode 100644 index 00000000..f7c0517d --- /dev/null +++ b/scripts/measure_sbom_coverage.py @@ -0,0 +1,75 @@ +"""Measure real SBOM coverage for an org — the gating question on #338. + +Not a registered pipeline; this is throwaway measurement code to answer +"how many repos have a readable dependency-graph SBOM" before investing in +the resolution heuristic or the chart. Run directly: + + uv run python measure_sbom_coverage.py + +Requires GITHUB_TOKEN in the environment (same as any other pipeline run). +Prints a coverage breakdown and writes the raw per-repo results to +sbom_coverage_raw.csv for closer inspection. +""" + +from __future__ import annotations + +import csv + +from hiero_analytics.config.paths import ORG +from hiero_analytics.data_sources.github_client import GitHubClient +from hiero_analytics.data_sources.github_ingest import fetch_org_repos_graphql +from hiero_analytics.data_sources.github_rest import fetch_org_sbom_data + + +def main(org: str = ORG) -> None: + """Fetch SBOM coverage for every repo in ``org`` and print a summary.""" + client = GitHubClient() + + repos = fetch_org_repos_graphql(client, org) + if not repos: + print(f"No repositories found for org: {org}") + return + + repo_names = [r.name for r in repos] + print(f"Fetching SBOM data for {len(repo_names)} repos in {org}...") + + coverage, packages = fetch_org_sbom_data(client, org, repo_names) + + by_status: dict[str, list] = {} + for c in coverage: + by_status.setdefault(c.status, []).append(c) + + print() + print("=== Coverage summary ===") + for status in ("ok", "disabled", "error"): + rows = by_status.get(status, []) + print(f"{status:>10}: {len(rows)} repos") + + ok_rows = by_status.get("ok", []) + nonzero = [c for c in ok_rows if c.package_count > 0] + zero = [c for c in ok_rows if c.package_count == 0] + print() + print(f"Of the {len(ok_rows)} repos with a readable SBOM:") + print(f" {len(nonzero)} have at least one dependency ({sum(c.package_count for c in nonzero)} packages total)") + print(f" {len(zero)} have a readable but empty manifest") + + print() + print(f"Total dependency edges (all ecosystems, unresolved to org repos yet): {len(packages)}") + ecosystems: dict[str, int] = {} + for p in packages: + ecosystems[p.ecosystem] = ecosystems.get(p.ecosystem, 0) + 1 + for eco, n in sorted(ecosystems.items(), key=lambda kv: -kv[1]): + print(f" {eco}: {n}") + + with open("sbom_coverage_raw.csv", "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["repo", "status", "package_count"]) + for c in sorted(coverage, key=lambda c: c.repo): + writer.writerow([c.repo, c.status, c.package_count]) + + print() + print("Wrote per-repo detail to sbom_coverage_raw.csv") + + +if __name__ == "__main__": + main() diff --git a/src/hiero_analytics/data_sources/github_rest.py b/src/hiero_analytics/data_sources/github_rest.py index eff98fb5..35685f6e 100644 --- a/src/hiero_analytics/data_sources/github_rest.py +++ b/src/hiero_analytics/data_sources/github_rest.py @@ -1,7 +1,8 @@ -"""GitHub REST helpers for CODEOWNERS presence and Actions-runner classification. +"""GitHub REST helpers for CODEOWNERS presence, Actions-runner classification, and dependency-graph SBOM fetching. -CODEOWNERS existence checks and workflow-YAML runner scanning via the REST API -(the GraphQL ingestion lives in ``github_ingest``). +CODEOWNERS existence checks, workflow-YAML runner scanning, and per-repo SBOM +package lists, all via the REST API (the GraphQL ingestion lives in +``github_ingest``). """ from __future__ import annotations @@ -9,13 +10,15 @@ import base64 import logging import re +from urllib.parse import unquote import requests import yaml +from .dataset_store import PartialOrgFetchError from .github_client import GitHubClient from .github_ingest._common import fetch_all_with_retry -from .models import RunnerRecord +from .models import DependencyManifestRecord, RunnerRecord, SbomCoverageRecord logger = logging.getLogger(__name__) @@ -60,6 +63,124 @@ def has_codeowners_file(client: GitHubClient, org: str, repo: str) -> bool: return False +def _parse_purl(purl: str) -> tuple[str, str, str | None] | None: + """Parse a package URL (purl) into (ecosystem, name, version). + + Format: ``pkg:type/namespace/name@version`` or ``pkg:type/name@version`` + (namespace optional; npm scopes and Maven groupIds arrive as the + namespace segment, percent-encoded per the purl spec — e.g. an npm scope + is ``%40scope``, not ``@scope``). Qualifiers (``?...``) and subpath + (``#...``) are dropped — irrelevant for repo resolution. Returns + ``None`` for anything that doesn't parse as ``pkg:...`` rather than + raising, since a single malformed purl shouldn't fail the whole repo's + SBOM. + """ + if not purl.startswith("pkg:"): + return None + body = purl[len("pkg:") :].split("?", 1)[0].split("#", 1)[0] + if "/" not in body: + return None + ecosystem, rest = body.split("/", 1) + name_and_version, _, version = rest.rpartition("@") + if not name_and_version: + # No '@version' segment at all -- treat the whole remainder as the name. + name_and_version, version = rest, None + return ecosystem.lower(), unquote(name_and_version), unquote(version) if version else None + + +def fetch_repo_sbom( + client: GitHubClient, org: str, repo: str +) -> tuple[SbomCoverageRecord, list[DependencyManifestRecord]]: + """Fetch and parse one repository's dependency-graph SBOM. + + Only a 404 counts as ``"disabled"`` (dependency graph off for this repo) — + the same 404-vs-error contract ``has_codeowners_file``/``fetch_repo_workflows`` + use. A 403 is ambiguous (rate limiting, an insufficiently-scoped token, a + private repo) rather than an unambiguous disabled-state signal, so it is + reported as ``"error"`` like any other HTTP failure, so the org-wide + fan-out in ``fetch_org_sbom_data`` can retry it instead of a transient or + permission failure being silently misclassified as "no dependency graph". + Non-HTTP request failures (timeouts, connection errors) still propagate + so ``fetch_org_sbom_data`` can retry them too. + """ + url = f"https://api.github.com/repos/{org}/{repo}/dependency-graph/sbom" + logger.info("Fetching SBOM for %s/%s", org, repo) + + try: + payload = client.get(url) + except requests.HTTPError as exc: + status_code = exc.response.status_code if exc.response is not None else None + + if status_code == 404: + logger.info( + "SBOM unavailable for %s/%s (404)", + org, + repo, + ) + return SbomCoverageRecord( + repo=repo, + status="disabled", + package_count=0, + ), [] + + logger.warning( + "SBOM request failed for %s/%s with HTTP %s; propagating for retry", + org, + repo, + status_code, + ) + raise + + sbom = payload.get("sbom") if isinstance(payload, dict) else None + if not isinstance(sbom, dict): + logger.error("Malformed SBOM response for %s/%s", org, repo) + return SbomCoverageRecord( + repo=repo, + status="error", + package_count=0, + ), [] + + raw_packages = sbom.get("packages") + if raw_packages is not None and not isinstance(raw_packages, list): + logger.error("Malformed SBOM package list for %s/%s", org, repo) + return SbomCoverageRecord( + repo=repo, + status="error", + package_count=0, + ), [] + packages = raw_packages or [] + described_ids = set(sbom.get("documentDescribes") or []) + + records: list[DependencyManifestRecord] = [] + for pkg in packages: + if not isinstance(pkg, dict) or pkg.get("SPDXID") in described_ids: + continue + purls = [ + ref.get("referenceLocator", "") + for ref in (pkg.get("externalRefs") or []) + if isinstance(ref, dict) and ref.get("referenceType") == "purl" + ] + parsed = next((p for purl in purls if (p := _parse_purl(purl)) is not None), None) + if parsed is None: + continue + ecosystem, package_name, version = parsed + records.append( + DependencyManifestRecord(repo=repo, package_name=package_name, ecosystem=ecosystem, version=version) + ) + + logger.info( + "SBOM fetched for %s/%s: %d dependency packages", + org, + repo, + len(records), + ) + return SbomCoverageRecord( + repo=repo, + status="ok", + package_count=len(records), + ), records + + def _is_self_hosted(label: str) -> bool | None: """ Determines if a runner is self-hosted. @@ -166,3 +287,42 @@ def fetch_repo_workflows(client: GitHubClient, org: str, repo: str) -> list[Runn task_desc=f"workflow files ({repo})", describe=lambda wf: str(wf.get("name", "?")), ) + + +# Concurrency for the org-wide SBOM fan-out (one call per repo). +_SBOM_FETCH_WORKERS = 8 + + +def fetch_org_sbom_data( + client: GitHubClient, + org: str, + repo_names: list[str], + max_workers: int = _SBOM_FETCH_WORKERS, +) -> tuple[list[SbomCoverageRecord], list[DependencyManifestRecord]]: + """Fetch and parse dependency-graph SBOMs for every repo in ``repo_names``. + + Fans ``fetch_repo_sbom`` out across the organization with retry handling. + A 404 is treated as an unavailable dependency graph. Other HTTP and + network failures propagate through the fan-out so transient failures can + be retried. Repositories that still fail after the retry are represented + as ``"error"`` coverage rows rather than being silently dropped. + """ + + def per_repo(repo: str) -> list[SbomCoverageRecord | DependencyManifestRecord]: + coverage, packages = fetch_repo_sbom(client, org, repo) + return [coverage, *packages] + + try: + combined = fetch_all_with_retry( + repo_names, + max_workers, + per_repo, + task_desc="SBOM data", + describe=str, + ) + except PartialOrgFetchError as exc: + combined = list(exc.records) + combined.extend(SbomCoverageRecord(repo=repo, status="error", package_count=0) for repo in exc.failed_repos) + coverage = [r for r in combined if isinstance(r, SbomCoverageRecord)] + packages = [r for r in combined if isinstance(r, DependencyManifestRecord)] + return coverage, packages diff --git a/src/hiero_analytics/data_sources/models.py b/src/hiero_analytics/data_sources/models.py index 7ffb804b..7a677d73 100644 --- a/src/hiero_analytics/data_sources/models.py +++ b/src/hiero_analytics/data_sources/models.py @@ -12,6 +12,7 @@ from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime +from typing import Literal from hiero_analytics.domain.bots import is_bot_login from hiero_analytics.domain.hip_references import extract_hip_mentions @@ -442,6 +443,39 @@ class ScorecardRecord: date: datetime +@dataclass(frozen=True) +class DependencyManifestRecord: + """One SBOM package entry for a repo, before org-repo resolution. + + ``package_name`` and ``ecosystem`` are parsed from the package URL + (purl) in the SBOM's ``externalRefs``, not the raw SPDX ``name`` field — + purl gives a consistent ecosystem+name split across npm/PyPI/Maven/ + Cargo/Go, where the raw name field's shape varies per ecosystem. + """ + + repo: str + package_name: str + ecosystem: str + version: str | None = None + + +@dataclass(frozen=True) +class SbomCoverageRecord: + """Per-repo SBOM fetch outcome — kept distinct from the package list itself. + + ``status`` is one of ``"ok"`` (SBOM available, packages parsed — + ``package_count`` may still be 0 for a repo with a genuinely empty + manifest), ``"disabled"`` (dependency graph unavilable for this repo, or the + endpoint 404 response), or ``"error"`` (fetch failed for a reason other + than disablement; surfaced rather than silently dropped). This is what + makes "no edges" distinguishable from "no data" downstream. + """ + + repo: str + status: Literal["ok", "disabled", "error"] + package_count: int + + @dataclass(frozen=True) class CodeOwnersRecord: """Represents the presence of a CODEOWNERS file in a repository.""" diff --git a/tests/data_sources/test_github_rest.py b/tests/data_sources/test_github_rest.py index 63a6aedb..b6cf78df 100644 --- a/tests/data_sources/test_github_rest.py +++ b/tests/data_sources/test_github_rest.py @@ -15,10 +15,14 @@ from hiero_analytics.data_sources.github_rest import ( _is_self_hosted, + _parse_purl, _process_workflow_file, + fetch_org_sbom_data, + fetch_repo_sbom, fetch_repo_workflows, has_codeowners_file, ) +from hiero_analytics.data_sources.models import DependencyManifestRecord, SbomCoverageRecord def _http_error(status_code: int) -> requests.HTTPError: @@ -141,3 +145,196 @@ def test_fetch_repo_workflows_scans_only_yaml_files(): assert [r.workflow_file for r in records] == ["ci.yml"] assert records[0].is_self_hosted is True + + +# -- _parse_purl -------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("purl", "expected"), + [ + ("pkg:npm/left-pad@1.0.1", ("npm", "left-pad", "1.0.1")), + ("pkg:npm/%40scope/pkg@2.0.0", ("npm", "@scope/pkg", "2.0.0")), + ("pkg:maven/com.example/thing@2.0", ("maven", "com.example/thing", "2.0")), + ("pkg:pypi/requests@2.31.0", ("pypi", "requests", "2.31.0")), + ("pkg:cargo/serde@1.0.0?extra=1", ("cargo", "serde", "1.0.0")), # qualifiers dropped + ("pkg:golang/github.com/org/mod@v1.2.3", ("golang", "github.com/org/mod", "v1.2.3")), + ("pkg:npm/no-version", ("npm", "no-version", None)), # no '@version' segment at all + ], +) +def test_parse_purl_handles_each_ecosystem_shape(purl, expected): + """Each ecosystem's purl shape parses to (ecosystem, name, version), scope/groupId decoded.""" + assert _parse_purl(purl) == expected + + +@pytest.mark.parametrize("malformed", ["not-a-purl", "pkg:", "pkg:npm-no-slash"]) +def test_parse_purl_returns_none_for_malformed_input(malformed): + """Malformed input returns None rather than raising -- one bad purl shouldn't fail a repo's SBOM.""" + assert _parse_purl(malformed) is None + + +# -- fetch_repo_sbom: the 404-vs-error contract, and SBOM parsing ------------ + + +def test_fetch_repo_sbom_parses_packages_and_excludes_the_described_root(): + """Packages parse to DependencyManifestRecord; the repo's own root package is excluded.""" + client = Mock() + client.get.return_value = { + "sbom": { + "documentDescribes": ["SPDXRef-root"], + "packages": [ + {"SPDXID": "SPDXRef-root", "name": "org/repo"}, # the repo itself -- must be excluded + { + "SPDXID": "SPDXRef-1", + "name": "lodash", + "externalRefs": [{"referenceType": "purl", "referenceLocator": "pkg:npm/lodash@4.17.21"}], + }, + ], + } + } + + coverage, records = fetch_repo_sbom(client, "org", "repo") + + assert coverage == SbomCoverageRecord(repo="repo", status="ok", package_count=1) + assert records == [DependencyManifestRecord(repo="repo", package_name="lodash", ecosystem="npm", version="4.17.21")] + + +def test_fetch_repo_sbom_skips_packages_without_a_parseable_purl(): + """A package with no purl external ref is skipped, not fabricated from the raw name.""" + client = Mock() + client.get.return_value = { + "sbom": { + "documentDescribes": [], + "packages": [{"SPDXID": "SPDXRef-1", "name": "mystery-pkg", "externalRefs": []}], + } + } + + coverage, records = fetch_repo_sbom(client, "org", "repo") + + assert records == [] + assert coverage.status == "ok" + assert coverage.package_count == 0 + + +def test_fetch_repo_sbom_treats_404_as_disabled(): + """404 is the only unambiguous 'dependency graph off for this repo' signal.""" + client = Mock() + client.get.side_effect = _http_error(404) + + coverage, records = fetch_repo_sbom(client, "org", "repo") + + assert coverage == SbomCoverageRecord(repo="repo", status="disabled", package_count=0) + assert records == [] + + +@pytest.mark.parametrize("status_code", [403, 500]) +def test_fetch_repo_sbom_propagates_http_errors(status_code): + """HTTP failures must propagate so the org-level retry can handle them.""" + client = Mock() + client.get.side_effect = _http_error(status_code) + + with pytest.raises(requests.HTTPError): + fetch_repo_sbom(client, "org", "repo") + + +@pytest.mark.parametrize("malformed_sbom", [{"sbom": "not-an-object"}, {"sbom": None}, "not-an-object", None]) +def test_fetch_repo_sbom_reports_error_when_sbom_is_not_an_object(malformed_sbom): + """A malformed payload/sbom must not be reported as an available SBOM with zero packages.""" + client = Mock() + client.get.return_value = malformed_sbom + + coverage, records = fetch_repo_sbom(client, "org", "repo") + + assert coverage == SbomCoverageRecord(repo="repo", status="error", package_count=0) + assert records == [] + + +@pytest.mark.parametrize("malformed_packages", [{"not": "a list"}, "not-a-list", 5]) +def test_fetch_repo_sbom_reports_error_when_packages_is_not_a_list(malformed_packages): + """'packages' present but the wrong type must not silently resolve to an empty manifest.""" + client = Mock() + client.get.return_value = {"sbom": {"documentDescribes": [], "packages": malformed_packages}} + + coverage, records = fetch_repo_sbom(client, "org", "repo") + + assert coverage == SbomCoverageRecord(repo="repo", status="error", package_count=0) + assert records == [] + + +# -- fetch_org_sbom_data: the org-wide fan-out -------------------------------- + + +def test_fetch_org_sbom_data_returns_one_coverage_row_per_repo(): + """Every input repo gets exactly one coverage row, regardless of outcome.""" + + def fake_get(url): + if "repo-a" in url: + return { + "sbom": { + "documentDescribes": [], + "packages": [ + { + "SPDXID": "x", + "externalRefs": [{"referenceType": "purl", "referenceLocator": "pkg:npm/left-pad@1.0.0"}], + } + ], + } + } + raise _http_error(404) + + client = Mock() + client.get.side_effect = fake_get + + coverage, packages = fetch_org_sbom_data(client, "org", ["repo-a", "repo-b"], max_workers=2) + + assert {c.repo for c in coverage} == {"repo-a", "repo-b"} + assert {c.repo: c.status for c in coverage} == {"repo-a": "ok", "repo-b": "disabled"} + assert [p.repo for p in packages] == ["repo-a"] + + +def test_fetch_org_sbom_data_retries_http_failures(): + """HTTP failures from a repo fetch reach the org-level retry mechanism.""" + calls = {"repo-a": 0} + + def fake_get(url): + if "repo-a" in url: + calls["repo-a"] += 1 + + if calls["repo-a"] == 1: + raise _http_error(500) + + return { + "sbom": { + "documentDescribes": [], + "packages": [ + { + "SPDXID": "x", + "externalRefs": [ + { + "referenceType": "purl", + "referenceLocator": "pkg:npm/left-pad@1.0.0", + } + ], + } + ], + } + } + + raise _http_error(404) + + client = Mock() + client.get.side_effect = fake_get + + coverage, packages = fetch_org_sbom_data( + client, + "org", + ["repo-a", "repo-b"], + max_workers=1, + ) + + assert calls["repo-a"] == 2 + assert {c.repo: c.status for c in coverage} == { + "repo-a": "ok", + "repo-b": "disabled", + } + assert len(packages) == 1