diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ee79711..785b370e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -190,6 +190,28 @@ schemas and the `paramify` CLI — not the internal code. `change_before_approval_field` `true`, and the new `summary.changes_recorded_before_approval_field` counts these changes. +- **`github_image_signing` 0.1.0 shows whether GitHub Actions signs and + verifies container images with cosign.** It reads the workflow files on + each default branch and records every `cosign sign` step with the kind of + key it uses (a key management service, keyless or other), whether the + command names the image by digest, whether it skips the transparency log, + and whether the step can be skipped or fail without failing the run + (an `if`, `continue-on-error`, or `|| true` in the script). It records + every `cosign verify` step and what it checks against, and lists the + verify steps that cannot fail the run. It lists the workflows that mention + cosign without a cosign command in the file, where signing happens + somewhere the file does not show, and the workflows that call a reusable + workflow, whose signing it does not read. For each workflow that signs, it + reads the runs in the window by outcome and the latest successful run, + leaving out pull request runs, which it counts apart. The summary lists the repositories that sign + without verifying. It needs Contents for the files and Actions for the + runs; with `include_runs: false` it reads the files only and every run + figure is null. It reads workflow files, not the registry, and copies no + script text into the evidence. KSI-SVC-VRI. Not yet run against a live + organization. The workflow file reader and the newest-first run walk moved + from `github_actions_security` and `github_workflow_runs` into the shared + GitHub client so that the three fetchers use one copy; their evidence is + unchanged. - **`wiz_vulnerability_findings` 0.4.0: SLA columns.** After the 43 legacy columns (unchanged, so existing intake presets keep mapping), the CSV adds the Wiz fields Paramify's remediation SLAs depend on: `Is Internet Reachable` and diff --git a/docs/ksi_mapping.md b/docs/ksi_mapping.md index e1bcc72e..f7ea26c9 100644 --- a/docs/ksi_mapping.md +++ b/docs/ksi_mapping.md @@ -160,7 +160,7 @@ python tools/gen_ksi_mapping.py *Controls:* `cm-2.2`, `cm-8.3`, `sc-13`, `sc-23`, `si-7`, `si-7.1`, `sr-10` -*9 fetchers:* [`azure_container_registry_configuration`](../fetchers/azure/container_registry_configuration), [`azure_devops_pipelines`](../fetchers/azure/devops_pipelines), [`azure_vm_hardening_status`](../fetchers/azure/vm_hardening_status), [`gcp_compute_instance_configuration`](../fetchers/gcp/compute_instance_configuration), [`gcp_gke_cluster_configuration`](../fetchers/gcp/gke_cluster_configuration), [`github_branch_protection`](../fetchers/github/branch_protection), [`github_release_integrity`](../fetchers/github/release_integrity), [`oci_compute_instances`](../fetchers/oci/compute_instances), [`wiz_file_integrity_monitoring`](../fetchers/wiz/file_integrity_monitoring) +*10 fetchers:* [`azure_container_registry_configuration`](../fetchers/azure/container_registry_configuration), [`azure_devops_pipelines`](../fetchers/azure/devops_pipelines), [`azure_vm_hardening_status`](../fetchers/azure/vm_hardening_status), [`gcp_compute_instance_configuration`](../fetchers/gcp/compute_instance_configuration), [`gcp_gke_cluster_configuration`](../fetchers/gcp/gke_cluster_configuration), [`github_branch_protection`](../fetchers/github/branch_protection), [`github_image_signing`](../fetchers/github/image_signing), [`github_release_integrity`](../fetchers/github/release_integrity), [`oci_compute_instances`](../fetchers/oci/compute_instances), [`wiz_file_integrity_monitoring`](../fetchers/wiz/file_integrity_monitoring) ### MLA — Monitoring, Logging, and Auditing (5/5) @@ -330,7 +330,7 @@ python tools/gen_ksi_mapping.py *Controls:* `cm-2.2`, `cm-7.5`, `cm-8`, `cm-8.1`, `cm-12`, `cm-12.1`, `cp-2.8` -*19 fetchers:* [`aws_detect_new_aws_resource`](../fetchers/aws/detect_new_aws_resource), [`aws_resource_inventory`](../fetchers/aws/resource_inventory), [`azure_entra_service_principals`](../fetchers/azure/entra_service_principals), [`azure_resource_inventory`](../fetchers/azure/resource_inventory), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_storage_encryption_status`](../fetchers/azure/storage_encryption_status), [`crowdstrike_hosts`](../fetchers/crowdstrike/hosts), [`datadog_agent_hosts`](../fetchers/datadog/agent_hosts), [`datadog_apm_services`](../fetchers/datadog/apm_services), [`datadog_containers`](../fetchers/datadog/containers), [`github_repository_inventory`](../fetchers/github/repository_inventory), [`gitlab_project_summary`](../fetchers/gitlab/project_summary), [`k8s_eks_pod_inventory`](../fetchers/k8s/eks_pod_inventory), [`rippling_devices`](../fetchers/rippling/devices), [`sentinelone_agents`](../fetchers/sentinelone/agents), [`sentinelone_xdr_assets`](../fetchers/sentinelone/xdr_assets), [`sophos_endpoint_inventory`](../fetchers/sophos/endpoint_inventory), [`wiz_inventory`](../fetchers/wiz/inventory), [`wiz_scan_coverage`](../fetchers/wiz/scan_coverage) +*20 fetchers:* [`aws_config_inventory`](../fetchers/aws/config_inventory), [`aws_detect_new_aws_resource`](../fetchers/aws/detect_new_aws_resource), [`aws_resource_inventory`](../fetchers/aws/resource_inventory), [`azure_entra_service_principals`](../fetchers/azure/entra_service_principals), [`azure_resource_inventory`](../fetchers/azure/resource_inventory), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_storage_encryption_status`](../fetchers/azure/storage_encryption_status), [`crowdstrike_hosts`](../fetchers/crowdstrike/hosts), [`datadog_agent_hosts`](../fetchers/datadog/agent_hosts), [`datadog_apm_services`](../fetchers/datadog/apm_services), [`datadog_containers`](../fetchers/datadog/containers), [`github_repository_inventory`](../fetchers/github/repository_inventory), [`gitlab_project_summary`](../fetchers/gitlab/project_summary), [`k8s_eks_pod_inventory`](../fetchers/k8s/eks_pod_inventory), [`rippling_devices`](../fetchers/rippling/devices), [`sentinelone_agents`](../fetchers/sentinelone/agents), [`sentinelone_xdr_assets`](../fetchers/sentinelone/xdr_assets), [`sophos_endpoint_inventory`](../fetchers/sophos/endpoint_inventory), [`wiz_inventory`](../fetchers/wiz/inventory), [`wiz_scan_coverage`](../fetchers/wiz/scan_coverage) #### `KSI-PIY-RES` — Reviewing Executive Support @@ -418,9 +418,9 @@ python tools/gen_ksi_mapping.py ## By fetcher -330 of 346 fetchers carry a mapping. +332 of 348 fetchers carry a mapping. -### aws (82) +### aws (83) | Fetcher | Indicators | |---|---| @@ -441,6 +441,7 @@ python tools/gen_ksi_mapping.py | [`aws_codepipeline_config`](../fetchers/aws/codepipeline_config) | `KSI-CMT-RMV`, `KSI-CMT-VTD` | | [`aws_component_ssl_enforcement_status`](../fetchers/aws/component_ssl_enforcement_status) | `KSI-SVC-SIN` | | [`aws_config_conformance_packs`](../fetchers/aws/config_conformance_packs) | `KSI-CNA-IBP`, `KSI-MLA-EVC` | +| [`aws_config_inventory`](../fetchers/aws/config_inventory) | `KSI-PIY-GIV` | | [`aws_config_monitoring`](../fetchers/aws/config_monitoring) | `KSI-CMT-LMC`, `KSI-MLA-EVC`, `KSI-SVC-ACM` | | [`aws_database_high_availability`](../fetchers/aws/database_high_availability) | `KSI-CNA-OFA` | | [`aws_detect_new_aws_resource`](../fetchers/aws/detect_new_aws_resource) | `KSI-CMT-LMC`, `KSI-PIY-GIV` | @@ -700,7 +701,7 @@ python tools/gen_ksi_mapping.py | [`gcp_secret_manager_configuration`](../fetchers/gcp/secret_manager_configuration) | `KSI-SVC-ASM` | | [`gcp_vpc_network_configuration`](../fetchers/gcp/vpc_network_configuration) | `KSI-CNA-ULN` | -### github (16) +### github (17) | Fetcher | Indicators | |---|---| @@ -711,6 +712,7 @@ python tools/gen_ksi_mapping.py | [`github_code_scanning_alerts`](../fetchers/github/code_scanning_alerts) | `KSI-MLA-EVC`, `KSI-SVC-EIS` | | [`github_dependabot_alerts`](../fetchers/github/dependabot_alerts) | `KSI-SCR-MON`, `KSI-SVC-EIS` | | [`github_deployment_controls`](../fetchers/github/deployment_controls) | `KSI-CMT-LMC`, `KSI-CMT-RMV`, `KSI-CMT-VTD` | +| [`github_image_signing`](../fetchers/github/image_signing) | `KSI-SVC-VRI` | | [`github_non_user_credentials`](../fetchers/github/non_user_credentials) | `KSI-IAM-SNU`, `KSI-SVC-ASM` | | [`github_org_security_settings`](../fetchers/github/org_security_settings) | `KSI-CNA-EIS`, `KSI-IAM-APM`, `KSI-IAM-ELP` | | [`github_pull_request_checks`](../fetchers/github/pull_request_checks) | `KSI-CMT-LMC`, `KSI-CMT-VTD` | diff --git a/examples/github_run.yaml b/examples/github_run.yaml index af65e697..505603d8 100644 --- a/examples/github_run.yaml +++ b/examples/github_run.yaml @@ -53,6 +53,8 @@ run: - use: github_workflow_runs targets: [*github_target] + - use: github_image_signing + targets: [*github_target] - use: github_vulnerability_disclosure targets: [*github_target] diff --git a/fetchers/_categories/github.yaml b/fetchers/_categories/github.yaml index 288c8c72..2362db59 100644 --- a/fetchers/_categories/github.yaml +++ b/fetchers/_categories/github.yaml @@ -3,7 +3,7 @@ display_name: GitHub description: >- GitHub change-management, supply-chain and code-security evidence: branch protection, pull request review and CI checks, Actions security, release - integrity, deployment controls and approvals, workflow run history, Dependabot, code scanning and secret + integrity, deployment controls and approvals, workflow run history, image signing, Dependabot, code scanning and secret scanning alerts, vulnerability disclosure, organization security settings, access, non-user credentials, the audit log, FedRAMP significant change notifications, and an inventory of every repository. One organization per target, each with its own token and API diff --git a/fetchers/github/README.md b/fetchers/github/README.md index 04b3a74e..28a63e8b 100644 --- a/fetchers/github/README.md +++ b/fetchers/github/README.md @@ -26,6 +26,7 @@ organization, each with its own token and API host. | `github_release_integrity` | Build provenance on each release asset (by digest), immutable releases, release tags locked by tag rulesets | SVC-VRI, SCR-MIT | | `github_deployment_controls` | Environments' reviewers, self-review, wait timers, branch policies and admin bypass; production deployments in a window with commit, creator, whether Actions made them and who approved them | CMT-RMV, CMT-VTD, CMT-LMC | | `github_workflow_runs` | Every workflow run in a window with its trigger, branch, commit, actor and result, and the artifacts each run kept (metadata only) | CMT-VTD, CMT-RMV | +| `github_image_signing` | The cosign sign and verify steps in each workflow file: the kind of signing key, whether the image is named by digest, whether the step can be skipped; and how each signing workflow ran in a window | SVC-VRI | | `github_vulnerability_disclosure` | Private vulnerability reporting, security policy, repository security advisories with state and age | PIY-RVD | | `github_significant_change_notifications` | FedRAMP SCN objects from pull requests flagged as significant changes, schema-validated and checked against SCN-CSO-INF | CMT-LMC | | `github_audit_log` | Audit log event counts and every access, protection and control change in a window (**Enterprise Cloud only**) | MLA-LET, MLA-OSM, IAM-SUS | @@ -61,9 +62,9 @@ a missing grant is reported as e.g. `(needs: administration=read)`. |---|---|---| | Metadata (repository) | `metadata` | all | | Administration (repository) | `administration` | branch_protection (also rule-suite bypasses), pull_request_checks (required checks), dependabot_alerts (security updates), code_scanning_alerts (default setup), secret_scanning (settings), non_user_credentials (deploy keys), actions_security (Actions policy, runners), release_integrity (immutable releases) | -| Contents (repository) | `contents` | pull_request_checks (commits), dependabot_alerts (dependency graph), actions_security (workflow files), branch_protection (CODEOWNERS errors) | +| Contents (repository) | `contents` | pull_request_checks (commits), dependabot_alerts (dependency graph), actions_security (workflow files), branch_protection (CODEOWNERS errors), image_signing (workflow files) | | Pull requests (repository) | `pull_requests` | pull_request_checks, significant_change_notifications | -| Actions (repository) | `actions` | deployment_controls (environments, run approvals), workflow_runs, actions_security (OIDC subject, only in repositories with a workflow that uses OIDC) | +| Actions (repository) | `actions` | deployment_controls (environments, run approvals), workflow_runs, actions_security (OIDC subject, only in repositories with a workflow that uses OIDC), image_signing (runs of the signing workflows; or set `include_runs: false`) | | Deployments (repository) | `deployments` | deployment_controls | | Attestations (repository) | `attestations` | release_integrity | | Repository security advisories (repository) | `repository_advisories` | vulnerability_disclosure. **Without it GitHub lists published advisories only, with no error**: drafts and private reports disappear and nothing says so | @@ -88,7 +89,9 @@ fetchers at version 0.1.0. That measurement does not include `workflow_runs`, the run approvals in `deployment_controls` 0.2.0, or the runner groups and OIDC subjects in `actions_security` 0.2.0. Their rows in the table above are not yet measured with a minimal token. If one is wrong, -the failure names the permission that GitHub wants. +the failure names the permission that GitHub wants. `image_signing` 0.1.0 is +built against GitHub's API description and covered by unit tests; it has not +yet been run against a live organization. Two settings are visible only to an **organization owner**: the two-factor requirement and the list of members without two-factor. Collected by a @@ -243,6 +246,38 @@ repositories; each returns well-formed, misleading data without an error. ## Not visible to these fetchers +- **Image signing is read from workflow files, not from the registry.** + `image_signing` finds `cosign sign` and `cosign verify` commands in the + `run` scripts of the workflow files on each default branch. It does not + read the registry, so it cannot say which images carry a signature today. + cosign called from a script file, a make target, a composite action or a + reusable workflow in another file is not seen: the file row lists the + reusable workflows it calls, and the summary lists the workflows that + mention cosign (an installer, a make target) without a cosign command in + the file, and every workflow that calls a reusable workflow. A script that writes cosign commands into a file with a + here-document reads as running them. Other signing tools are not + recognized. Each cosign command is one entry: a step that runs two + commands counts twice, and one command in a loop that signs many images + counts once. +- **A signing workflow's run is not its signing step.** `image_signing` + reads each run's result, not each step's. A signing step under an `if`, or + with `continue-on-error`, can be skipped or fail in a run that succeeds, + and so can a command the script follows with `|| true`, `|| :` or a + message; the summary lists those steps so the run figures are read with + that in mind, and lists verify steps of that kind as not enforced. Two + other ways a script hides a failure are not detected: `set +e`, and a + pipe from cosign in a step that does not set `shell: bash` (GitHub's + default shell runs without `pipefail`). Runs on every branch are counted, while the steps are those of the + default branch's file. Pull request runs (`pull_request`, + `pull_request_target`) are counted apart and left out of every other run + figure: a signing workflow often runs on pull requests with its sign step + skipped, and such a run still succeeds. +- **A signing key is classified only from what the file says.** The key is + a key management service only when the `--key` value, written out or set + in the workflow's own `env`, starts with that service's scheme. A value + from a secret, a variable or a shell expression reads as `other`, cosign's + `env://` form reads as `environment_variable`, and no `--key` and no + `--sk` reads as `keyless`. - **Review or merge gates enforced outside GitHub** (a bot merge queue such as Prow, or comment-driven approval) do not appear as reviews or rules; such a repository reads as unreviewed and unprotected. `pull_request_checks` @@ -273,8 +308,10 @@ repositories; each returns well-formed, misleading data without an error. ## Free text in the evidence The evidence holds text that people write: commit subjects, pull request -titles, advisory summaries, change notification sections and deploy key -titles. A web application firewall can refuse an upload that contains text +titles, advisory summaries, change notification sections, deploy key +titles and workflow step names. `image_signing` copies no script text. +From a workflow's `env` it keeps one thing, a signing key's reference, and +only when that is a plain key management service address. A web application firewall can refuse an upload that contains text such as `../`, `/bin/sh` or ` Optional[Dict[str, List[Tuple[str, Optional[str]]]]]: + """{repository: [(file name, text)]} for the workflow files on each default branch, or None on failure. + + GitHub runs only .yml and .yaml files directly in .github/workflows. text is None where GraphQL returned no + readable content (a binary or truncated file): its content is unknown, not empty. + """ + found = client.graphql_repos("GraphQL workflow files", org, [r["name"] for r in repos], WORKFLOW_FILES) + if found is None: + return None + out: Dict[str, List[Tuple[str, Optional[str]]]] = {} + for name, data in found.items(): + entries = (data.get("object") or {}).get("entries") or [] + rows: List[Tuple[str, Optional[str]]] = [] + for e in sorted(entries, key=lambda e: e.get("name") or ""): + if e.get("type") != "blob" or not str(e.get("name")).endswith((".yml", ".yaml")): + continue + blob = e.get("object") or {} + unreadable = blob.get("text") is None or blob.get("isTruncated") or blob.get("isBinary") + rows.append((e["name"], None if unreadable else blob["text"])) + out[name] = rows + return out + + +def newest_since(client: GitHubClient, path: str, key: str, since: float, cap: int, what: str, + env: str) -> Optional[List[dict]]: + """Items of a newest-first list created since `since`. None on failure or when more than `cap` are in the window. + + Walked page by page rather than filtered with `created=`: a filtered list stops at 1,000 results and then reports + an empty page and total_count 0 (measured), which would read as a quiet window. + """ + seen: List[dict] = [] + found = client.paginate_required(path, key=key, + until=lambda x: seen.append(x) or (to_epoch(x.get("created_at")) or since) < since + or len(seen) > cap) + if found is None: + return None + if len(seen) > cap and (to_epoch(seen[-1].get("created_at")) or since) >= since: + client.fail(f"GET {path}", "Truncated", f"more than {cap} {what} in the window; raise {env}", "partial_failure") + return None + return found + + # --- the entry point --------------------------------------------------------- def run(name: str, collect: Callable[[GitHubClient, dict, List[dict], Dict[str, Any]], Optional[dict]], diff --git a/fetchers/github/actions_security/fetcher.py b/fetchers/github/actions_security/fetcher.py index 4e35a9c0..2197c393 100644 --- a/fetchers/github/actions_security/fetcher.py +++ b/fetchers/github/actions_security/fetcher.py @@ -8,12 +8,10 @@ import yaml sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "_shared")) -from github_client import run # noqa: E402 +from github_client import run, workflow_file_texts # noqa: E402 NAME = "github_actions_security" CONFIG = {"include_self_hosted_runners": ("GITHUB_INCLUDE_SELF_HOSTED_RUNNERS", True)} -WORKFLOW_FIELDS = ('object(expression: "HEAD:.github/workflows") { ... on Tree { entries { name type ' - 'object { ... on Blob { text isBinary isTruncated } } } } }') FULL_SHA = re.compile(r"[0-9a-f]{40}\Z") # Triggers that run with the base repository's secrets and a write token on an event an outsider can cause. PRIVILEGED_TRIGGERS = ("pull_request_target", "workflow_run") @@ -125,23 +123,12 @@ def analyse(name, text): def workflow_files(client, org, repos): """{repository: [file analyses]}, or None. GitHub runs only .yml and .yaml files directly in .github/workflows.""" - found = client.graphql_repos("GraphQL workflow files", org, [r["name"] for r in repos], WORKFLOW_FIELDS) + found = workflow_file_texts(client, org, repos) if found is None: return None - out = {} - for name, data in found.items(): - entries = (data.get("object") or {}).get("entries") or [] - rows = [] - for e in sorted(entries, key=lambda e: e.get("name") or ""): - if e.get("type") != "blob" or not str(e.get("name")).endswith((".yml", ".yaml")): - continue - blob = e.get("object") or {} - if blob.get("text") is None or blob.get("isTruncated") or blob.get("isBinary"): - rows.append({"file": e["name"], "parsed": False, "error": "content not readable over GraphQL"}) - else: - rows.append(analyse(e["name"], blob["text"])) - out[name] = rows - return out + return {name: [{"file": f, "parsed": False, "error": "content not readable over GraphQL"} if text is None + else analyse(f, text) for f, text in files] + for name, files in found.items()} def selected(client, base, policy): diff --git a/fetchers/github/image_signing/fetcher.py b/fetchers/github/image_signing/fetcher.py new file mode 100644 index 00000000..e88098e3 --- /dev/null +++ b/fetchers/github/image_signing/fetcher.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +"""Whether GitHub Actions signs and verifies container images with cosign: the steps in each workflow file, and how +the signing workflows ran.""" + +import re +import sys +from pathlib import Path +from urllib.parse import quote + +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "_shared")) +from github_client import iso, newest_since, now_epoch, run, workflow_file_texts # noqa: E402 + +NAME = "github_image_signing" +CONFIG = { + "include_runs": ("GITHUB_IMAGE_SIGNING_INCLUDE_RUNS", True), + "days_back": ("GITHUB_IMAGE_SIGNING_DAYS_BACK", 30), + "max_runs": ("GITHUB_IMAGE_SIGNING_MAX_RUNS", 1000), +} +MAX_DAYS_BACK = 366 +INSTALLER = "sigstore/cosign-installer" +# Longest first, so "verify-attestation" and "sign-blob" are not read as "verify" and "sign". +SUBCOMMANDS = ("verify-attestation", "verify-blob", "sign-blob", "attest", "verify", "sign") +# cosign as a command (also by path, or as a container image's entry), any global flags, then the subcommand. +COSIGN = re.compile(r"(?- + Container image signing in GitHub Actions: which workflows sign images with + cosign, with what kind of key, whether any workflow verifies signatures, and + how signing workflows ran. + +category: github + +supports_targets: true + +target_schema: + org: + type: string + required: true + env: GITHUB_ORG + description: Organization login, as in github.com/. + api_url: + type: string + required: false + default: https://api.github.com + env: GITHUB_API_URL + description: >- + REST API base URL. https://api.github.com for github.com, + https://api..ghe.com for GHE.com, https:///api/v3 for + GitHub Enterprise Server. + repositories: + type: string + required: false + env: GITHUB_REPOSITORIES + description: >- + Comma-separated repository names to collect. Unset collects every + repository the token can see; a name the token cannot see fails the run. + include_archived: + type: boolean + required: false + default: false + env: GITHUB_INCLUDE_ARCHIVED + description: Include archived repositories, which cannot change. + +runtime: + type: python + entry: fetcher.py + timeout: 1800 + +output: + type: json + path: github_image_signing.json + aggregation: per_target + +config_schema: + include_runs: + type: boolean + required: false + default: true + env: GITHUB_IMAGE_SIGNING_INCLUDE_RUNS + description: >- + Read how each signing workflow ran in the window, which needs the Actions + permission. Set false to read the workflow files only; every run field + is then null, never zero. + days_back: + type: integer + required: false + default: 30 + env: GITHUB_IMAGE_SIGNING_DAYS_BACK + description: Days of signing workflow run history to collect (1-366). + max_runs: + type: integer + required: false + default: 1000 + env: GITHUB_IMAGE_SIGNING_MAX_RUNS + description: >- + Most runs read per signing workflow; more in the window fails the run + rather than cutting the history short. + +secrets: [] + +ksis: + - KSI-SVC-VRI + +evidence_set: + reference_id: EVD-GITHUB-IMAGE-SIGNING + name: GitHub Image Signing + instructions: |- + Script: github/image_signing/fetcher.py + + Access: a GitHub App or fine-grained token with Metadata and Contents read on the repositories. Optional: Actions read, for how the signing workflows ran; without it, set include_runs false and every run field reads null. + + Commands Executed: + - GET orgs/ — the organization and its repository counts + - GET orgs//repos (or GET repos// per listed repository) — the repositories in scope + - GraphQL HEAD:.github/workflows per repository — the workflow files on the default branch + - GET repos///actions/workflows//runs (per signing workflow, when include_runs is true) — its runs in the window + + Output: github_image_signing.json, one per organization — per repository, each workflow file's cosign sign steps (kind of key, whether the image is named by digest, whether the step can be skipped), verify steps, and for each signing workflow its runs in the window by outcome and the latest successful run. + + Summary fields: + + What was read? + - repositories_total — Number of repositories in scope + - workflow_files — Number of workflow files read on the default branches + - workflow_files_unparsed — List of workflow files that could not be read or parsed as a workflow; their steps are unknown + - not_collected — List of repositories that could not be read; their steps are unknown, not absent + + Which workflows sign images, and how? + - repositories_signing — List of repositories with at least one cosign sign step + - signing_workflows — List of workflow files with a cosign sign step + - signing_steps — Number of cosign sign steps + - signing_steps_by_key — Number of sign steps per kind of key: azure_key_vault, aws_kms, gcp_kms, hashicorp_vault, kubernetes_secret, environment_variable, keyless or other + - signing_steps_naming_a_digest — Number of sign steps that name the image by digest rather than by tag + - signing_steps_skipping_transparency_log — Number of sign steps that do not record the signature in the transparency log + - signing_steps_conditional — List of sign steps (repository/file:job:step) that run only when a condition holds + - signing_steps_continue_on_error — List of sign steps whose failure does not fail the job + - signing_steps_discarding_failure — List of sign steps whose script carries on whatever cosign returned + - workflows_calling_reusable_workflows — List of workflow files that call a reusable workflow (jobs..uses); what the called workflow signs is not read, so a repository that signs only there reads as not signing + - workflows_mentioning_cosign_without_a_cosign_command — List of workflow files that mention cosign but run it some way this fetcher does not read, such as a script + + Are signatures verified? + - verification_workflows — List of workflow files with a cosign verify step + - verification_steps — Number of cosign verify steps + - verification_steps_not_enforced — List of verify steps whose failure cannot fail the run + - repositories_signing_without_verification — List of repositories that sign images but never verify a signature + + How did the signing workflows run? + - runs_read — True when every signing workflow's runs were read; false when any could not be; null when include_runs is false + - signing_workflow_runs — Number of signing workflow runs in the window, pull request runs left out; null unless runs_read is true + - signing_workflow_runs_successful — Number of those runs that succeeded; null unless runs_read is true + - signing_workflow_pull_request_runs — Number of pull_request and pull_request_target runs of signing workflows, left out of the other run figures because a sign step is usually skipped on them while the run still succeeds; null unless runs_read is true + - signing_workflows_without_successful_run — List of signing workflows with no successful run in the window, pull request runs left out; null unless runs_read is true + - latest_successful_signing_run — Time of the newest successful signing workflow run, pull request runs left out; null when none or unread diff --git a/fetchers/github/workflow_runs/fetcher.py b/fetchers/github/workflow_runs/fetcher.py index c3aa923c..e1b384a4 100644 --- a/fetchers/github/workflow_runs/fetcher.py +++ b/fetchers/github/workflow_runs/fetcher.py @@ -5,7 +5,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "_shared")) -from github_client import iso, now_epoch, run, to_epoch # noqa: E402 +from github_client import iso, newest_since, now_epoch, run # noqa: E402 NAME = "github_workflow_runs" CONFIG = { @@ -17,24 +17,6 @@ FAILED = ("failure", "timed_out", "startup_failure") -def newest_since(client, path, key, since, cap, what, env): - """Items of a newest-first list created since `since`. None on failure or when more than `cap` are in the window. - - Walked page by page rather than filtered with `created=`: a filtered list stops at 1,000 results and then reports - an empty page and total_count 0 (measured), which would read as a quiet window. - """ - seen = [] - found = client.paginate_required(path, key=key, - until=lambda x: seen.append(x) or (to_epoch(x.get("created_at")) or since) < since - or len(seen) > cap) - if found is None: - return None - if len(seen) > cap and (to_epoch(seen[-1].get("created_at")) or since) >= since: - client.fail(f"GET {path}", "Truncated", f"more than {cap} {what} in the window; raise {env}", "partial_failure") - return None - return found - - def run_row(r, default_branch): head_repo = r.get("head_repository") or {} # A pull request from a fork can carry a branch of the same name as the default branch (measured). diff --git a/tests/test_github_fetchers.py b/tests/test_github_fetchers.py index 0247da34..85dbcdb3 100644 --- a/tests/test_github_fetchers.py +++ b/tests/test_github_fetchers.py @@ -15,7 +15,7 @@ import sys import time from pathlib import Path -from urllib.parse import parse_qs, urlencode, urlparse +from urllib.parse import parse_qs, quote, urlencode, urlparse import pytest import requests @@ -320,7 +320,11 @@ def test_every_fetcher_follows_the_shape(name): assert spec["evidence_set"]["reference_id"].startswith("EVD-GITHUB-") declared = {f["env"] for f in spec.get("config_schema", {}).values()} assert {env for env, _ in getattr(m, "CONFIG", {}).values()} <= declared - assert len(spec["evidence_set"]["instructions"].split()) <= 80 + instructions = spec["evidence_set"]["instructions"] + if "\nCommands Executed:\n" not in instructions: + # Old-style prose instructions stay short. Those in the Script/Access/Commands/Output/Summary format are + # held to that format by test_fetcher_text_follows_the_format, which lists every summary field. + assert len(instructions.split()) <= 80 @pytest.mark.parametrize("api_url,graphql", [ @@ -2923,3 +2927,527 @@ def test_workflow_run_configuration_errors(fake, monkeypatch, tmp_path, env): fake(runs_routes()) code, _, status = run_fetcher("workflow_runs", monkeypatch, tmp_path, **env) assert code == 1 and status["code"] == "bad_config" + + +# --- image_signing ------------------------------------------------------------------- + +SIGNING = MODS["image_signing"] +# A key reference as cosign writes one for a key management service: a name, never key material. +KMS_KEY = "azurekms://example-vault.vault.azure.net/image-signing" +IMAGE = "registry.example/app" + + +def workflow(*steps, job=None, env=None): + """A workflow file with one job, `promote`.""" + return yaml.safe_dump({"on": "push", **({"env": env} if env else {}), + "jobs": {"promote": {"runs-on": "ubuntu-latest", **(job or {}), "steps": list(steps)}}}) + + +def script(text, **step): + return {"run": text, **step} + + +def signing_steps(text, **kw): + return SIGNING.analyse("f.yml", workflow(script(text), **kw))["signing_steps"] + + +@pytest.mark.parametrize("command,env,kind,reference", [ + (f"cosign sign --key {KMS_KEY} {IMAGE}@sha256:0123456789ab", None, "azure_key_vault", KMS_KEY), + (f"cosign sign --key={KMS_KEY} {IMAGE}", None, "azure_key_vault", KMS_KEY), + (f"cosign sign --key awskms:///alias/image-signing {IMAGE}", None, "aws_kms", "awskms:///alias/image-signing"), + (f"cosign sign --key gcpkms://projects/p/locations/global/keyRings/r/cryptoKeys/k {IMAGE}", None, "gcp_kms", + "gcpkms://projects/p/locations/global/keyRings/r/cryptoKeys/k"), + (f"cosign sign --key hashivault://image-signing {IMAGE}", None, "hashicorp_vault", "hashivault://image-signing"), + (f"cosign sign --key k8s://signing/image-signing {IMAGE}", None, "kubernetes_secret", "k8s://signing/image-signing"), + (f"cosign sign --key cosign.key {IMAGE}", None, "other", None), + (f"cosign sign --key env://COSIGN_PRIVATE_KEY {IMAGE}", None, "environment_variable", None), + (f'cosign sign --key "$SIGNING_KEY" {IMAGE}', {"SIGNING_KEY": "env://COSIGN_PRIVATE_KEY"}, "environment_variable", None), + (f"cosign sign --sk {IMAGE}", None, "other", None), + (f"cosign sign --yes {IMAGE}", None, "keyless", None), + # A key named through the workflow's own env is followed, in each way a script can spell it. + (f'cosign sign --key "$SIGNING_KEY" {IMAGE}', {"SIGNING_KEY": KMS_KEY}, "azure_key_vault", KMS_KEY), + (f"cosign sign --key ${{SIGNING_KEY}} {IMAGE}", {"SIGNING_KEY": KMS_KEY}, "azure_key_vault", KMS_KEY), + (f"cosign sign --key ${{{{ env.SIGNING_KEY }}}} {IMAGE}", {"SIGNING_KEY": KMS_KEY}, "azure_key_vault", KMS_KEY), + # Not followed: a value that only exists when the workflow runs is never guessed. + (f'cosign sign --key "$SIGNING_KEY" {IMAGE}', None, "other", None), + (f"cosign sign --key ${{{{ secrets.SIGNING_KEY }}}} {IMAGE}", None, "other", None), + (f"cosign sign --key ${{{{ vars.SIGNING_KEY }}}} {IMAGE}", None, "other", None), + # The service is known from the scheme, but a reference built from an expression is not a name to record. + (f"cosign sign --key azurekms://${{{{ vars.VAULT }}}}/image-signing {IMAGE}", None, "azure_key_vault", None), +]) +def test_the_signing_key_is_classified_and_only_a_plain_reference_is_kept(command, env, kind, reference): + [step] = signing_steps(command, env=env) + assert (step["key"], step["key_reference"]) == (kind, reference) + + +def test_step_env_overrides_job_env_overrides_workflow_env(): + other = "awskms:///alias/other" + text = workflow(script(f'cosign sign --key "$K" {IMAGE}', env={"K": KMS_KEY}), script(f'cosign sign --key "$K" {IMAGE}'), + job={"env": {"K": other}}, env={"K": "cosign.key"}) + first, second = SIGNING.analyse("f.yml", text)["signing_steps"] + assert (first["key"], second["key"], second["key_reference"]) == ("azure_key_vault", "aws_kms", other) + + +@pytest.mark.parametrize("text,signs,verifies,others", [ + (f"cosign sign {IMAGE}", 1, 0, []), + (f"/usr/local/bin/cosign sign {IMAGE}", 1, 0, []), + (f"docker run gcr.io/projectsigstore/cosign sign {IMAGE}", 1, 0, []), + (f"cosign --verbose sign {IMAGE}", 1, 0, []), + (f"cosign sign \\\n --key cosign.key \\\n {IMAGE}", 1, 0, []), + (f"cosign sign {IMAGE}\ncosign sign {IMAGE}-worker", 2, 0, []), + (f"cosign sign {IMAGE} && cosign verify --key cosign.pub {IMAGE}", 1, 1, []), + (f"cosign verify --key cosign.pub {IMAGE} | jq .", 0, 1, []), + # Not image signing: other subcommands are recorded as what they are. + ("cosign sign-blob --key cosign.key release.tar.gz", 0, 0, ["sign-blob"]), + ("cosign verify-blob --key cosign.pub release.tar.gz", 0, 0, ["verify-blob"]), + (f"cosign verify-attestation --key cosign.pub {IMAGE}", 0, 0, ["verify-attestation"]), + (f"cosign attest --predicate sbom.json {IMAGE}", 0, 0, ["attest"]), + # Not commands at all. + (f"# cosign sign {IMAGE}", 0, 0, []), + (f" # cosign sign {IMAGE}", 0, 0, []), + (f"make build # then cosign sign {IMAGE}", 0, 0, []), + (f'echo "run cosign sign {IMAGE} by hand"', 0, 0, []), + (f"printf 'cosign sign {IMAGE}'", 0, 0, []), + (f"mycosign sign {IMAGE}", 0, 0, []), + (f"cosign-helper sign {IMAGE}", 0, 0, []), + ("cosign version", 0, 0, []), + ("cosign signature-of something", 0, 0, []), +]) +def test_cosign_commands_are_found_in_a_script_and_nothing_else_is(text, signs, verifies, others): + row = SIGNING.analyse("f.yml", workflow(script(text))) + assert (len(row["signing_steps"]), len(row["verification_steps"]), + [s["command"] for s in row["other_cosign_steps"]]) == (signs, verifies, others) + + +def test_a_flag_of_the_next_command_is_not_read_as_this_ones(): + [step] = signing_steps(f"cosign sign {IMAGE} && cosign verify --key {KMS_KEY} {IMAGE}@sha256:0123456789ab") + assert (step["key"], step["names_digest"]) == ("keyless", False) + + +@pytest.mark.parametrize("target,by_digest", [ + (f"{IMAGE}@sha256:0123456789ab", True), (f"{IMAGE}@${{{{ steps.push.outputs.digest }}}}", True), + (f'"{IMAGE}@$DIGEST"', True), (f"{IMAGE}@${{DIGEST}}", True), + # A tag, a variable or an address with a user in it is not evidence of a digest. + (f"{IMAGE}:1.4.2", False), ('"$IMAGE"', False), ("user@registry.example/app:1.4.2", False), +]) +def test_a_digest_is_recorded_only_when_the_command_names_one(target, by_digest): + assert signing_steps(f"cosign sign --yes {target}")[0]["names_digest"] is by_digest + + +@pytest.mark.parametrize("flags,skipped", [("--tlog-upload=false", True), ("--tlog-upload false", True), + ("--tlog-upload=FALSE", True), ("--tlog-upload=true", False), ("", False)]) +def test_signing_without_the_transparency_log_is_recorded(flags, skipped): + assert signing_steps(f"cosign sign {flags} {IMAGE}")[0]["skips_transparency_log"] is skipped + + +@pytest.mark.parametrize("flags,checks,skips", [ + ("--key cosign.pub", "key", False), + ("--certificate-identity-regexp '^https://github.com/acme/' --certificate-oidc-issuer https://token.example", + "certificate_identity", False), + ("", "unspecified", False), ("--key cosign.pub --insecure-ignore-tlog", "key", True), +]) +def test_what_a_verify_step_checks_against(flags, checks, skips): + [step] = SIGNING.analyse("f.yml", workflow(script(f"cosign verify {flags} {IMAGE}")))["verification_steps"] + assert (step["checks"], step["skips_transparency_log"]) == (checks, skips) + + +@pytest.mark.parametrize("step,job,conditional,continues", [ + ({}, {}, False, False), + ({"if": "github.ref == 'refs/heads/main'"}, {}, True, False), + ({}, {"if": "github.event_name == 'push'"}, True, False), + ({"continue-on-error": True}, {}, False, True), + ({}, {"continue-on-error": True}, False, True), + ({"continue-on-error": "${{ matrix.experimental }}"}, {}, False, True), # an expression can turn it on + ({"continue-on-error": False}, {"continue-on-error": "false"}, False, False), +]) +def test_a_step_that_can_be_skipped_or_fail_quietly_is_marked(step, job, conditional, continues): + [row] = SIGNING.analyse("f.yml", workflow(script(f"cosign sign {IMAGE}", **step), job=job))["signing_steps"] + assert (row["conditional"], row["continue_on_error"]) == (conditional, continues) + + +def test_a_step_is_located_by_job_number_and_name(): + text = workflow({"uses": "actions/checkout@v4"}, script(f"cosign sign {IMAGE}", name="Sign the image"), + script(f"cosign sign {IMAGE}")) + named, unnamed = SIGNING.analyse("promote.yml", text)["signing_steps"] + assert (named["job"], named["step_number"], named["step"]) == ("promote", 2, "Sign the image") + assert (unnamed["step_number"], unnamed["step"]) == (3, None) + + +def test_the_installer_and_reusable_workflows_are_noted_because_their_signing_is_not_read(): + text = yaml.safe_dump({"on": "push", "jobs": { + "setup": {"runs-on": "ubuntu-latest", "steps": [{"uses": "Sigstore/Cosign-Installer@v3"}, + script("./scripts/sign-images.sh")]}, + "sign": {"uses": "acme/platform/.github/workflows/sign.yml@main"}}}) + row = SIGNING.analyse("f.yml", text) + assert row["installs_cosign"] is True and row["signing_steps"] == [] + assert row["reusable_workflows_called"] == ["acme/platform/.github/workflows/sign.yml@main"] + + +def test_workflows_that_call_a_reusable_workflow_are_listed_in_the_summary(fake, monkeypatch, tmp_path): + caller = yaml.safe_dump({"on": "push", "jobs": {"sign": {"uses": "acme/platform/.github/workflows/sign.yml@main"}}}) + fake(signing_routes({"app": {"release.yml": caller, "ci.yml": PLAIN}, "site": {"ci.yml": PLAIN}})) + code, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path) + s = ev["summary"] + # The shared workflow may sign, but nothing here shows it: the repository is not counted as signing. + assert code == 0 and s["repositories_signing"] == [] + assert s["workflows_calling_reusable_workflows"] == ["app:.github/workflows/release.yml"] + assert s["workflows_mentioning_cosign_without_a_cosign_command"] == [] + + +def test_pull_request_runs_of_a_signing_workflow_are_not_counted_as_signing(fake, monkeypatch, tmp_path): + """GitHub's docker-publish starter runs on pull requests too, with the sign step skipped and the run a success.""" + publish = yaml.safe_dump({"on": {"push": {"branches": ["main"]}, "pull_request": {"branches": ["main"]}}, + "jobs": {"build": {"runs-on": "ubuntu-latest", "steps": [ + {"uses": "sigstore/cosign-installer@v3"}, + script(f"cosign sign --yes {IMAGE}@${{{{ steps.build.outputs.digest }}}}", + **{"if": "github.event_name != 'pull_request'"})]}}}) + pr = wf_run(3, 1, "docker-publish.yml", fork=True) + target = {**wf_run(2, 2, "docker-publish.yml"), "event": "pull_request_target"} + failed_push = wf_run(1, 3, "docker-publish.yml", conclusion="failure") + fake(signing_routes({"app": {"docker-publish.yml": publish}}, + runs={("app", "docker-publish.yml"): [pr, target, failed_push]})) + code, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path) + s = ev["summary"] + assert code == 0 + assert (s["signing_workflow_runs"], s["signing_workflow_runs_successful"], + s["signing_workflow_pull_request_runs"]) == (1, 0, 2) + assert s["signing_workflows_without_successful_run"] == ["app:.github/workflows/docker-publish.yml"] + assert s["latest_successful_signing_run"] is None + [w] = ev["repositories"][0]["workflows"] + assert w["runs"]["by_outcome"] == {"failure": 1} and w["runs"]["latest_outcome"] == "failure" + + +@pytest.mark.parametrize("text,error", [("jobs: [unclosed", None), ("- a list\n- not a workflow", "not a mapping"), + ("", "not a mapping")]) +def test_a_file_github_cannot_run_is_reported_not_guessed_at(text, error): + row = SIGNING.analyse("f.yml", text) + assert row["parsed"] is False and row["signing_steps"] == [] and (error is None or row["error"] == error) + + +@pytest.mark.parametrize("doc", [{"jobs": ["a", "b"]}, {"jobs": {"a": "text"}}, {"jobs": {"a": {"steps": "text"}}}, + {"jobs": {"a": {"steps": [None, "text", {"run": 7}, {"run": None, "name": 3}]}}}, + {"env": ["x"], "jobs": {"a": {"env": "x", "steps": [{"env": 3, "run": "cosign sign i"}]}}}]) +def test_odd_shapes_in_a_workflow_file_do_not_break_the_read(doc): + row = SIGNING.analyse("f.yml", yaml.safe_dump(doc)) + assert row["parsed"] is True and len(row["signing_steps"]) <= 1 + + +NOT_COPIED = "TEXT-THAT-MUST-STAY-IN-THE-WORKFLOW" +PROMOTE = workflow( + {"uses": "sigstore/cosign-installer@v3"}, + script(f"cosign sign --tlog-upload=false --key {KMS_KEY} {IMAGE}@${{{{ steps.push.outputs.digest }}}}\n" + f"notes={NOT_COPIED}", name="Sign the image", env={"NOTES": NOT_COPIED}), + script(f"cosign verify --key {KMS_KEY} {IMAGE}@sha256:0123456789ab", name="Verify the signature")) +RELEASE = workflow(script(f"cosign sign --yes {IMAGE}:latest", **{"if": "github.ref == 'refs/heads/main'", + "continue-on-error": True})) +PLAIN = workflow(script("make test")) +SIGNING_FILES = {"app": {"image-promote.yml": PROMOTE, "ci.yml": PLAIN}, + "site": {"release.yml": RELEASE}, + "docs": {"broken.yml": "jobs: [unclosed", "lint.yml": PLAIN}, + "tools": {}} + + +def runs_path(name, file): + return f"repos/{ORG}/{name}/actions/workflows/{quote(file, safe='')}/runs" + + +def signing_routes(files=None, runs=None, refused=()): + files = SIGNING_FILES if files is None else files + routes = org_routes([repo(n) for n in files]) + for n, tree in files.items(): + routes[f"repos/{ORG}/{n}/.workflows"] = tree or None + for f in tree: + path = runs_path(n, f) + routes[path] = ((403, {"message": "Resource not accessible by integration"}) if (n, f) in refused + else paged(path, "workflow_runs", (runs or {}).get((n, f), []))) + return routes + + +def run_calls(fake_github): + return sorted({path for path, _ in fake_github.calls if "/actions/workflows/" in path}) + + +def test_image_signing_verdicts(fake, monkeypatch, tmp_path, assert_summary_documented): + newest = wf_run(5, 1, "image-promote.yml") + runs = {("app", "image-promote.yml"): [wf_run(6, 0.5, "image-promote.yml", conclusion=None, status="in_progress"), + newest, wf_run(4, 2, "image-promote.yml", conclusion="failure"), + wf_run(3, 3, "image-promote.yml"), wf_run(1, 40, "image-promote.yml")], + ("site", "release.yml"): [wf_run(9, 2, "release.yml", conclusion="failure")]} + github = fake(signing_routes(runs=runs)) + code, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path) + assert code == 0 and ev["metadata"]["partial_failure"] is False + assert_summary_documented("github/image_signing", ev["summary"]) + assert ev["summary"] == { + "repositories_total": 4, + "workflow_files": 5, + "workflow_files_unparsed": ["docs:.github/workflows/broken.yml"], + "repositories_signing": ["app", "site"], + "signing_workflows": ["app:.github/workflows/image-promote.yml", "site:.github/workflows/release.yml"], + "signing_steps": 2, + "signing_steps_by_key": {"azure_key_vault": 1, "aws_kms": 0, "gcp_kms": 0, "hashicorp_vault": 0, + "kubernetes_secret": 0, "environment_variable": 0, "keyless": 1, "other": 0}, + "signing_steps_naming_a_digest": 1, + "signing_steps_skipping_transparency_log": 1, + "signing_steps_conditional": ["site:.github/workflows/release.yml:promote:1"], + "signing_steps_continue_on_error": ["site:.github/workflows/release.yml:promote:1"], + "signing_steps_discarding_failure": [], + "verification_workflows": ["app:.github/workflows/image-promote.yml"], + "verification_steps": 1, + "verification_steps_not_enforced": [], + "repositories_signing_without_verification": ["site"], + "workflows_calling_reusable_workflows": [], + "workflows_mentioning_cosign_without_a_cosign_command": [], + "runs_read": True, + "signing_workflow_runs": 5, # run 1 is outside the window + "signing_workflow_runs_successful": 2, + "signing_workflow_pull_request_runs": 0, + "signing_workflows_without_successful_run": ["site:.github/workflows/release.yml"], + "latest_successful_signing_run": newest["created_at"], + "not_collected": [], + } + # Runs are read only for the workflows that sign. + assert run_calls(github) == [runs_path("app", "image-promote.yml"), runs_path("site", "release.yml")] + app = next(r for r in ev["repositories"] if r["repository"] == "app") + promote = next(w for w in app["workflows"] if w["file"] == "image-promote.yml") + assert promote["installs_cosign"] is True and promote["signing_steps"] == [{ + "job": "promote", "step": "Sign the image", "step_number": 2, "command": "sign", "names_digest": True, + "conditional": False, "continue_on_error": False, "failure_discarded": False, "key": "azure_key_vault", + "key_reference": KMS_KEY, + "skips_transparency_log": True}] + assert promote["runs"] == {"read": True, "pull_request_runs": 0, "total": 4, "successful": 2, + "by_outcome": {"failure": 1, "in_progress": 1, "success": 2}, + "latest_run_at": runs[("app", "image-promote.yml")][0]["created_at"], + "latest_outcome": "in_progress", "latest_success_at": newest["created_at"]} + assert next(w for w in app["workflows"] if w["file"] == "ci.yml")["runs"] is None + + +def test_no_script_text_or_env_value_reaches_the_evidence(fake, monkeypatch, tmp_path): + """Only what is derived from a cosign command is written: a script can hold anything.""" + fake(signing_routes()) + _, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path) + written = json.dumps(ev) + assert NOT_COPIED not in written and "display_title" not in written and "never collected" not in written + assert "make test" not in written and "steps.push.outputs" not in written + + +def test_with_runs_turned_off_no_run_is_read_and_no_run_figure_is_given(fake, monkeypatch, tmp_path): + github = fake(signing_routes()) + code, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path, GITHUB_IMAGE_SIGNING_INCLUDE_RUNS="false") + s = ev["summary"] + assert code == 0 and run_calls(github) == [] and ev["metadata"]["since"] is None + assert (s["runs_read"], s["signing_workflow_runs"], s["signing_workflow_runs_successful"], + s["signing_workflows_without_successful_run"], s["latest_successful_signing_run"]) == (None,) * 5 + # What the files say is still reported in full. + assert (s["signing_steps"], s["repositories_signing_without_verification"]) == (2, ["site"]) + assert all(w["runs"] is None for r in ev["repositories"] for w in r.get("workflows", [])) + + +def test_unreadable_runs_are_unknown_not_zero_and_the_files_are_still_reported(fake, monkeypatch, tmp_path): + runs = {("app", "image-promote.yml"): [wf_run(5, 1, "image-promote.yml")]} + fake(signing_routes(runs=runs, refused=[("site", "release.yml")])) + code, ev, status = run_fetcher("image_signing", monkeypatch, tmp_path) + s = ev["summary"] + assert code == 1 and ev["metadata"]["partial_failure"] is True and status["code"] == "not_authorized" + assert (s["runs_read"], s["signing_workflow_runs"], s["signing_workflow_runs_successful"], + s["signing_workflows_without_successful_run"], s["latest_successful_signing_run"]) == (False, None, None, + None, None) + assert (s["signing_steps"], s["verification_steps"], s["not_collected"]) == (2, 1, []) + site = next(r for r in ev["repositories"] if r["repository"] == "site") + assert site["workflows"][0]["runs"] == {"read": False} + + +def test_more_runs_than_the_cap_fails_rather_than_cutting_the_history(fake, monkeypatch, tmp_path): + runs = {("app", "image-promote.yml"): [wf_run(i, 1, "image-promote.yml") for i in range(9, 0, -1)]} + fake(signing_routes(runs=runs)) + code, ev, status = run_fetcher("image_signing", monkeypatch, tmp_path, GITHUB_IMAGE_SIGNING_MAX_RUNS="5") + assert code == 1 and status["code"] == "partial_failure" and "GITHUB_IMAGE_SIGNING_MAX_RUNS" in status["error"] + assert ev["summary"]["runs_read"] is False and ev["summary"]["signing_workflow_runs"] is None + + +def test_no_signing_anywhere_is_a_complete_answer(fake, monkeypatch, tmp_path): + github = fake(signing_routes({"docs": {"lint.yml": PLAIN}, "tools": {}})) + code, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path) + s = ev["summary"] + assert code == 0 and run_calls(github) == [] + assert (s["repositories_signing"], s["signing_steps"], s["runs_read"], s["signing_workflow_runs"], + s["signing_workflows_without_successful_run"], s["latest_successful_signing_run"]) == ([], 0, True, 0, [], None) + + +def test_a_signing_workflow_that_never_ran_in_the_window_is_named(fake, monkeypatch, tmp_path): + fake(signing_routes({"site": {"release.yml": RELEASE}})) + code, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path) + s = ev["summary"] + assert code == 0 and (s["signing_workflow_runs"], s["signing_workflows_without_successful_run"]) == ( + 0, ["site:.github/workflows/release.yml"]) + + +@pytest.mark.parametrize("steps", [ + [{"uses": "sigstore/cosign-installer@v3"}, script("./scripts/sign-images.sh")], + [script('make cosign-sign IMAGE="${IMAGE}" DIGEST="${DIGEST}"')], # signing behind a make target + [script("cosign version")], +]) +def test_cosign_used_in_a_way_the_file_does_not_show_is_listed_not_counted(fake, monkeypatch, tmp_path, steps): + fake(signing_routes({"app": {"promote.yml": workflow(*steps)}, "docs": {"lint.yml": PLAIN}})) + _, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path) + s = ev["summary"] + assert s["workflows_mentioning_cosign_without_a_cosign_command"] == ["app:.github/workflows/promote.yml"] + assert (s["repositories_signing"], s["signing_steps"]) == ([], 0) + + +def test_a_comment_about_cosign_is_not_a_mention(): + row = SIGNING.analyse("f.yml", workflow(script("# cosign sign would go here\nmake build"))) + assert row["mentions_cosign"] is False + + +def test_a_workflow_file_name_is_escaped_in_the_runs_path(fake, monkeypatch, tmp_path): + github = fake(signing_routes({"app": {"image promote#1.yml": RELEASE}})) + code, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path) + assert code == 0 and run_calls(github) == [f"repos/{ORG}/app/actions/workflows/image%20promote%231.yml/runs"] + + +def test_a_workflow_file_with_no_readable_content_makes_its_repository_unknown(fake, monkeypatch, tmp_path): + routes = signing_routes({"app": {"image-promote.yml": PROMOTE}, "site": {"release.yml": RELEASE}}) + + def graphql(query, variables): + status, body = gql_repos(routes, query, variables) + for node in body["data"].values(): + for entry in (node.get("object") or {}).get("entries", []): + if entry["name"] == "release.yml": + entry["object"] = {"text": None, "isBinary": False, "isTruncated": True} + return status, body + routes["graphql"] = graphql + github = fake(routes) + code, ev, status = run_fetcher("image_signing", monkeypatch, tmp_path) + s = ev["summary"] + assert code == 1 and status["code"] == "partial_failure" and ev["metadata"]["partial_failure"] is True + assert (s["not_collected"], s["repositories_total"], s["repositories_signing"]) == (["site"], 2, ["app"]) + assert s["repositories_signing_without_verification"] == [] and run_calls(github) == [runs_path("app", "image-promote.yml")] + assert next(r for r in ev["repositories"] if r["repository"] == "site") == {"repository": "site", "collected": False} + + +def test_a_failed_read_of_the_workflow_files_leaves_no_figures(fake, monkeypatch, tmp_path): + routes = signing_routes() + routes["graphql"] = (200, {"errors": [{"type": "FORBIDDEN", "message": "Resource not accessible"}]}) + github = fake(routes) + code, ev, status = run_fetcher("image_signing", monkeypatch, tmp_path) + assert code == 1 and status["code"] == "not_authorized" and ev["summary"] == {} and run_calls(github) == [] + + +@pytest.mark.parametrize("env", [{"GITHUB_IMAGE_SIGNING_DAYS_BACK": "0"}, {"GITHUB_IMAGE_SIGNING_DAYS_BACK": "367"}, + {"GITHUB_IMAGE_SIGNING_MAX_RUNS": "0"}, {"GITHUB_IMAGE_SIGNING_INCLUDE_RUNS": "maybe"}, + {"GITHUB_IMAGE_SIGNING_DAYS_BACK": "0", "GITHUB_IMAGE_SIGNING_INCLUDE_RUNS": "false"}]) +def test_image_signing_configuration_errors(fake, monkeypatch, tmp_path, env): + github = fake(signing_routes()) + monkeypatch.setenv("GITHUB_ORG", ORG) + monkeypatch.setenv("GITHUB_TOKEN", "test-token") + monkeypatch.setenv("GITHUB_API_URL", API) + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setenv("FETCHER_STATUS_FILE", str(tmp_path / "status.json")) + for key, value in env.items(): + monkeypatch.setenv(key, value) + assert gc.run(SIGNING.NAME, SIGNING.collect, SIGNING.CONFIG) == 1 + assert json.loads((tmp_path / "status.json").read_text())["code"] == "bad_config" + assert not any(path == "graphql" or "/actions/" in path for path, _ in github.calls) + + +def test_a_repository_that_only_verifies_is_not_counted_as_signing(fake, monkeypatch, tmp_path): + verify_only = workflow(script(f"cosign verify --key {KMS_KEY} {IMAGE}@sha256:0123456789ab")) + github = fake(signing_routes({"deploy": {"deploy.yml": verify_only}, "site": {"release.yml": RELEASE}})) + code, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path) + s = ev["summary"] + assert code == 0 and s["verification_workflows"] == ["deploy:.github/workflows/deploy.yml"] + assert (s["repositories_signing"], s["repositories_signing_without_verification"]) == (["site"], ["site"]) + assert run_calls(github) == [runs_path("site", "release.yml")] + + +# Shapes taken from public workflow files, where the first version of this fetcher went wrong. +@pytest.mark.parametrize("text,signs,verifies", [ + # GitHub's own starter workflow: the command comes after a pipe from echo. + ('echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}', 1, 0), + (f'echo "Signing {IMAGE}" && cosign sign --yes {IMAGE}', 1, 0), + ('for tag in $TAGS; do\n cosign sign \\\n -a "sha=${GITHUB_SHA}" \\\n -y \\\n "$tag"@${DIGEST}\ndone', 1, 0), + (f"test $# -gt 0 && cosign sign {IMAGE}", 1, 0), # $# is not a comment + (f"cosign.exe sign {IMAGE}", 1, 0), + (f"if ! cosign verify --key cosign.pub {IMAGE}; then exit 1; fi", 0, 1), + (f'bash -c "cosign sign {IMAGE} && cosign verify --key cosign.pub {IMAGE}"', 1, 1), + # A separator or a # inside quotes is text. + (f'echo "first; cosign sign {IMAGE}"', 0, 0), + (f"echo 'next | cosign sign {IMAGE}'", 0, 0), + (f'printf "%s\\n" "see # cosign sign {IMAGE}"', 0, 0), + (f"cosign sign {IMAGE} # then cosign verify {IMAGE}", 1, 0), + (f'make cosign-sign IMAGE="{IMAGE}"', 0, 0), +]) +def test_commands_are_split_the_way_a_shell_splits_them(text, signs, verifies): + row = SIGNING.analyse("f.yml", workflow(script(text))) + assert (len(row["signing_steps"]), len(row["verification_steps"])) == (signs, verifies) + + +def test_the_starter_workflow_line_is_read_in_full(): + [step] = signing_steps('echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}') + assert (step["key"], step["names_digest"], step["failure_discarded"]) == ("keyless", True, False) + + +def test_an_expression_or_a_quoted_pattern_does_not_cut_a_command_short(): + [sign] = signing_steps(f"cosign sign --key ${{{{ inputs.key || 'cosign.key' }}}} --tlog-upload=false {IMAGE}") + assert (sign["key"], sign["skips_transparency_log"]) == ("other", True) + text = f"cosign verify --certificate-identity-regexp '^https://github.com/(acme|example)/' --insecure-ignore-tlog {IMAGE}" + [verify] = SIGNING.analyse("f.yml", workflow(script(text)))["verification_steps"] + assert (verify["checks"], verify["skips_transparency_log"]) == ("certificate_identity", True) + + +@pytest.mark.parametrize("tail,discarded", [ + ("|| true", True), ("|| :", True), ('|| echo "signing failed"', True), ("||true", True), + ("|| exit 1", False), ("|| { echo failed; exit 1; }", False), ("&& true", False), ("; true", False), ("", False), + ("| tee sign.log", False), +]) +def test_a_failure_the_script_throws_away_is_marked(tail, discarded): + [step] = signing_steps(f"cosign sign --yes {IMAGE} {tail}") + assert step["failure_discarded"] is discarded + + +def test_a_verify_step_that_cannot_fail_the_run_is_listed_as_not_enforced(fake, monkeypatch, tmp_path): + text = workflow(script(f"cosign sign --yes {IMAGE} || true", name="sign"), + script(f"cosign verify --key cosign.pub {IMAGE} || true", name="soft verify"), + script(f"cosign verify --key cosign.pub {IMAGE}", name="warn only", **{"continue-on-error": True}), + script(f"cosign verify --key cosign.pub {IMAGE}", name="real verify")) + fake(signing_routes({"app": {"promote.yml": text}})) + code, ev, _ = run_fetcher("image_signing", monkeypatch, tmp_path) + s = ev["summary"] + at = "app:.github/workflows/promote.yml:promote" + assert code == 0 and (s["verification_steps"], s["verification_steps_not_enforced"]) == (3, [f"{at}:2", f"{at}:3"]) + assert s["signing_steps_discarding_failure"] == [f"{at}:1"] and s["signing_steps_continue_on_error"] == [] + + +def test_no_workflow_shape_can_break_the_read(): + """Random documents built from the keys a workflow uses, with values of every YAML type.""" + import random + rng = random.Random(20261113) + keys = ["jobs", "steps", "run", "uses", "env", "if", "continue-on-error", "name", "on", "a", "promote"] + leaves = [None, True, False, 0, 7, 1.5, "", "text", "cosign sign img", "cosign verify img || true", "${{ x }}", + "sigstore/cosign-installer@v3", "echo \"unclosed", "it's # cosign", "\\", "$(", "`", "'", '"', "&&", "||"] + + def value(depth): + roll = rng.random() + if depth > 4 or roll < 0.35: + return rng.choice(leaves) + if roll < 0.7: + return {rng.choice(keys): value(depth + 1) for _ in range(rng.randint(0, 4))} + return [value(depth + 1) for _ in range(rng.randint(0, 4))] + for _ in range(3000): + doc = {"jobs": value(0), "env": value(3)} if rng.random() < 0.8 else value(0) + row = SIGNING.analyse("f.yml", yaml.safe_dump(doc)) + assert isinstance(row["signing_steps"], list) and isinstance(row["parsed"], bool) + + +@pytest.mark.parametrize("text", ["", "'", '"', "\\", "${{", "${{ a", "a ${{ b }} c", "# only a comment", "a\\\nb", "||", "&&;|&", + "cosign sign 'unclosed", 'cosign sign "a\\"b" img', "x" * 20000 + " cosign sign img"]) +def test_the_splitter_ends_on_any_script(text): + assert isinstance(SIGNING.commands(text), list) and SIGNING.segments(text)[-1][1] == "" + + +def test_two_commands_in_one_quoted_string_keep_their_own_flags(): + text = f'bash -c "cosign sign {IMAGE} && cosign verify --key {KMS_KEY} {IMAGE}@sha256:0123456789ab"' + row = SIGNING.analyse("f.yml", workflow(script(text))) + [sign], [verify] = row["signing_steps"], row["verification_steps"] + assert (sign["key"], sign["key_reference"], sign["names_digest"]) == ("keyless", None, False) + assert (verify["checks"], verify["names_digest"]) == ("key", True)