feat(deps): SBOM ingestion + a standalone coverage-measurement script…

scripts/measure_sbom_coverage.py

@@ -0,0 +1,75 @@
+"""Measure real SBOM coverage for an org — the gating question on #338.
+
+Not a registered pipeline; this is throwaway measurement code to answer
+"how many repos have a readable dependency-graph SBOM" before investing in
+the resolution heuristic or the chart. Run directly:
+
+    uv run python measure_sbom_coverage.py
+
+Requires GITHUB_TOKEN in the environment (same as any other pipeline run).
+Prints a coverage breakdown and writes the raw per-repo results to
+sbom_coverage_raw.csv for closer inspection.
+"""
+
+from __future__ import annotations
+
+import csv
+
+from hiero_analytics.config.paths import ORG
+from hiero_analytics.data_sources.github_client import GitHubClient
+from hiero_analytics.data_sources.github_ingest import fetch_org_repos_graphql
+from hiero_analytics.data_sources.github_rest import fetch_org_sbom_data
+
+
+def main(org: str = ORG) -> None:
+    """Fetch SBOM coverage for every repo in ``org`` and print a summary."""
+    client = GitHubClient()
+
+    repos = fetch_org_repos_graphql(client, org)
+    if not repos:
+        print(f"No repositories found for org: {org}")
+        return
+
+    repo_names = [r.name for r in repos]
+    print(f"Fetching SBOM data for {len(repo_names)} repos in {org}...")
+
+    coverage, packages = fetch_org_sbom_data(client, org, repo_names)
+
+    by_status: dict[str, list] = {}
+    for c in coverage:
+        by_status.setdefault(c.status, []).append(c)
+
+    print()
+    print("=== Coverage summary ===")
+    for status in ("ok", "disabled", "error"):
+        rows = by_status.get(status, [])
+        print(f"{status:>10}: {len(rows)} repos")
+
+    ok_rows = by_status.get("ok", [])
+    nonzero = [c for c in ok_rows if c.package_count > 0]
+    zero = [c for c in ok_rows if c.package_count == 0]
+    print()
+    print(f"Of the {len(ok_rows)} repos with a readable SBOM:")
+    print(f"  {len(nonzero)} have at least one dependency ({sum(c.package_count for c in nonzero)} packages total)")
+    print(f"  {len(zero)} have a readable but empty manifest")
+
+    print()
+    print(f"Total dependency edges (all ecosystems, unresolved to org repos yet): {len(packages)}")
+    ecosystems: dict[str, int] = {}
+    for p in packages:
+        ecosystems[p.ecosystem] = ecosystems.get(p.ecosystem, 0) + 1
+    for eco, n in sorted(ecosystems.items(), key=lambda kv: -kv[1]):
+        print(f"  {eco}: {n}")
+
+    with open("sbom_coverage_raw.csv", "w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        writer.writerow(["repo", "status", "package_count"])
+        for c in sorted(coverage, key=lambda c: c.repo):
+            writer.writerow([c.repo, c.status, c.package_count])
+
+    print()
+    print("Wrote per-repo detail to sbom_coverage_raw.csv")
+
+
+if __name__ == "__main__":
+    main()

src/hiero_analytics/data_sources/github_rest.py

@@ -1,21 +1,24 @@
-"""GitHub REST helpers for CODEOWNERS presence and Actions-runner classification.
+"""GitHub REST helpers for CODEOWNERS presence, Actions-runner classification, and dependency-graph SBOM fetching.
 
-CODEOWNERS existence checks and workflow-YAML runner scanning via the REST API
-(the GraphQL ingestion lives in ``github_ingest``).
+CODEOWNERS existence checks, workflow-YAML runner scanning, and per-repo SBOM
+package lists, all via the REST API (the GraphQL ingestion lives in
+``github_ingest``).
 """
 
 from __future__ import annotations
 
 import base64
 import logging
 import re
+from urllib.parse import unquote
 
 import requests
 import yaml
 
+from .dataset_store import PartialOrgFetchError
 from .github_client import GitHubClient
 from .github_ingest._common import fetch_all_with_retry
-from .models import RunnerRecord
+from .models import DependencyManifestRecord, RunnerRecord, SbomCoverageRecord
 
 logger = logging.getLogger(__name__)
 
@@ -60,6 +63,124 @@ def has_codeowners_file(client: GitHubClient, org: str, repo: str) -> bool:
     return False
 
 
+def _parse_purl(purl: str) -> tuple[str, str, str | None] | None:
+    """Parse a package URL (purl) into (ecosystem, name, version).
+
+    Format: ``pkg:type/namespace/name@version`` or ``pkg:type/name@version``
+    (namespace optional; npm scopes and Maven groupIds arrive as the
+    namespace segment, percent-encoded per the purl spec — e.g. an npm scope
+    is ``%40scope``, not ``@scope``). Qualifiers (``?...``) and subpath
+    (``#...``) are dropped — irrelevant for repo resolution. Returns
+    ``None`` for anything that doesn't parse as ``pkg:...`` rather than
+    raising, since a single malformed purl shouldn't fail the whole repo's
+    SBOM.
+    """
+    if not purl.startswith("pkg:"):
+        return None
+    body = purl[len("pkg:") :].split("?", 1)[0].split("#", 1)[0]
+    if "/" not in body:
+        return None
+    ecosystem, rest = body.split("/", 1)
+    name_and_version, _, version = rest.rpartition("@")
+    if not name_and_version:
+        # No '@version' segment at all -- treat the whole remainder as the name.
+        name_and_version, version = rest, None
+    return ecosystem.lower(), unquote(name_and_version), unquote(version) if version else None
+
+
+def fetch_repo_sbom(
+    client: GitHubClient, org: str, repo: str
+) -> tuple[SbomCoverageRecord, list[DependencyManifestRecord]]:
+    """Fetch and parse one repository's dependency-graph SBOM.
+
+    Only a 404 counts as ``"disabled"`` (dependency graph off for this repo) —
+    the same 404-vs-error contract ``has_codeowners_file``/``fetch_repo_workflows``
+    use. A 403 is ambiguous (rate limiting, an insufficiently-scoped token, a
+    private repo) rather than an unambiguous disabled-state signal, so it is
+    reported as ``"error"`` like any other HTTP failure, so the org-wide
+    fan-out in ``fetch_org_sbom_data`` can retry it instead of a transient or
+    permission failure being silently misclassified as "no dependency graph".
+    Non-HTTP request failures (timeouts, connection errors) still propagate
+    so ``fetch_org_sbom_data`` can retry them too.
+    """
+    url = f"https://api.github.com/repos/{org}/{repo}/dependency-graph/sbom"
+    logger.info("Fetching SBOM for %s/%s", org, repo)
+
+    try:
+        payload = client.get(url)
+    except requests.HTTPError as exc:
+        status_code = exc.response.status_code if exc.response is not None else None
+
+        if status_code == 404:
+            logger.info(
+                "SBOM unavailable for %s/%s (404)",
+                org,
+                repo,
+            )
+            return SbomCoverageRecord(
+                repo=repo,
+                status="disabled",
+                package_count=0,
+            ), []
+
+        logger.warning(
+            "SBOM request failed for %s/%s with HTTP %s; propagating for retry",
+            org,
+            repo,
+            status_code,
+        )
+        raise
+
+    sbom = payload.get("sbom") if isinstance(payload, dict) else None
+    if not isinstance(sbom, dict):
+        logger.error("Malformed SBOM response for %s/%s", org, repo)
+        return SbomCoverageRecord(
+            repo=repo,
+            status="error",
+            package_count=0,
+        ), []
+
+    raw_packages = sbom.get("packages")
+    if raw_packages is not None and not isinstance(raw_packages, list):
+        logger.error("Malformed SBOM package list for %s/%s", org, repo)
+        return SbomCoverageRecord(
+            repo=repo,
+            status="error",
+            package_count=0,
+        ), []
+    packages = raw_packages or []
+    described_ids = set(sbom.get("documentDescribes") or [])
+
+    records: list[DependencyManifestRecord] = []
+    for pkg in packages:
+        if not isinstance(pkg, dict) or pkg.get("SPDXID") in described_ids:
+            continue
+        purls = [
+            ref.get("referenceLocator", "")
+            for ref in (pkg.get("externalRefs") or [])
+            if isinstance(ref, dict) and ref.get("referenceType") == "purl"
+        ]
+        parsed = next((p for purl in purls if (p := _parse_purl(purl)) is not None), None)
+        if parsed is None:
+            continue
+        ecosystem, package_name, version = parsed
+        records.append(
+            DependencyManifestRecord(repo=repo, package_name=package_name, ecosystem=ecosystem, version=version)
+        )
+
+    logger.info(
+        "SBOM fetched for %s/%s: %d dependency packages",
+        org,
+        repo,
+        len(records),
+    )
+    return SbomCoverageRecord(
+        repo=repo,
+        status="ok",
+        package_count=len(records),
+    ), records
+
+
 def _is_self_hosted(label: str) -> bool | None:
     """
     Determines if a runner is self-hosted.
@@ -166,3 +287,42 @@ def fetch_repo_workflows(client: GitHubClient, org: str, repo: str) -> list[Runn
         task_desc=f"workflow files ({repo})",
         describe=lambda wf: str(wf.get("name", "?")),
     )
+
+
+# Concurrency for the org-wide SBOM fan-out (one call per repo).
+_SBOM_FETCH_WORKERS = 8
+
+
+def fetch_org_sbom_data(
+    client: GitHubClient,
+    org: str,
+    repo_names: list[str],
+    max_workers: int = _SBOM_FETCH_WORKERS,
+) -> tuple[list[SbomCoverageRecord], list[DependencyManifestRecord]]:
+    """Fetch and parse dependency-graph SBOMs for every repo in ``repo_names``.
+
+    Fans ``fetch_repo_sbom`` out across the organization with retry handling.
+    A 404 is treated as an unavailable dependency graph. Other HTTP and
+    network failures propagate through the fan-out so transient failures can
+    be retried. Repositories that still fail after the retry are represented
+    as ``"error"`` coverage rows rather than being silently dropped.
+    """
+
+    def per_repo(repo: str) -> list[SbomCoverageRecord | DependencyManifestRecord]:
+        coverage, packages = fetch_repo_sbom(client, org, repo)
+        return [coverage, *packages]
+
+    try:
+        combined = fetch_all_with_retry(
+            repo_names,
+            max_workers,
+            per_repo,
+            task_desc="SBOM data",
+            describe=str,
+        )
+    except PartialOrgFetchError as exc:
+        combined = list(exc.records)
+        combined.extend(SbomCoverageRecord(repo=repo, status="error", package_count=0) for repo in exc.failed_repos)
+    coverage = [r for r in combined if isinstance(r, SbomCoverageRecord)]
+    packages = [r for r in combined if isinstance(r, DependencyManifestRecord)]
+    return coverage, packages

src/hiero_analytics/data_sources/models.py

@@ -12,6 +12,7 @@
 from collections.abc import Mapping
 from dataclasses import dataclass
 from datetime import datetime
+from typing import Literal
 
 from hiero_analytics.domain.bots import is_bot_login
 from hiero_analytics.domain.hip_references import extract_hip_mentions
@@ -442,6 +443,39 @@ class ScorecardRecord:
     date: datetime
 
 
+@dataclass(frozen=True)
+class DependencyManifestRecord:
+    """One SBOM package entry for a repo, before org-repo resolution.
+
+    ``package_name`` and ``ecosystem`` are parsed from the package URL
+    (purl) in the SBOM's ``externalRefs``, not the raw SPDX ``name`` field —
+    purl gives a consistent ecosystem+name split across npm/PyPI/Maven/
+    Cargo/Go, where the raw name field's shape varies per ecosystem.
+    """
+
+    repo: str
+    package_name: str
+    ecosystem: str
+    version: str | None = None
+
+
+@dataclass(frozen=True)
+class SbomCoverageRecord:
+    """Per-repo SBOM fetch outcome — kept distinct from the package list itself.
+
+    ``status`` is one of ``"ok"`` (SBOM available, packages parsed —
+    ``package_count`` may still be 0 for a repo with a genuinely empty
+    manifest), ``"disabled"`` (dependency graph unavilable for this repo, or the
+    endpoint 404 response), or ``"error"`` (fetch failed for a reason other
+    than disablement; surfaced rather than silently dropped). This is what
+    makes "no edges" distinguishable from "no data" downstream.
+    """
+
+    repo: str
+    status: Literal["ok", "disabled", "error"]
+    package_count: int
+
+
 @dataclass(frozen=True)
 class CodeOwnersRecord:
     """Represents the presence of a CODEOWNERS file in a repository."""