From 26edf681aa7039f18fb38fcb0ad9b7896a2df071 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Sat, 17 Oct 2026 19:54:55 +0100 Subject: [PATCH 1/2] Instrument ci.yml with shared-actions telemetry Start ci-telemetry-start after the checkout and report ci-telemetry-finish under if: always() in coverage-check, linux-full and windows-compat, pinned to shared-actions 81374b6f. Finish reads the sccache-stats.json the existing recorder writes and takes setup-rust's sccache-status. A contract holds placement, always(), the status wiring and the pin. --- .github/workflows/ci.yml | 27 +++++ docs/developers-guide.md | 14 +++ tests/workflow_contracts/ci_telemetry_test.py | 103 ++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 tests/workflow_contracts/ci_telemetry_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13ee5788..1d92692d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,9 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Start CI telemetry + uses: leynos/shared-actions/.github/actions/ci-telemetry-start@81374b6f183e59f0570ad36217ded90d3240fda6 + - name: Bound concurrency to the runner shape run: | { @@ -254,6 +257,12 @@ jobs: # or reads); isolated errors only warn. run: python3 scripts/check_sccache_health.py --expect-location ghac sccache-stats.json + - name: Report CI telemetry + if: always() + uses: leynos/shared-actions/.github/actions/ci-telemetry-finish@81374b6f183e59f0570ad36217ded90d3240fda6 + with: + sccache-status: ${{ steps.setup-rust.outputs.sccache-status }} + linux-full: runs-on: ubicloud-standard-2-ubuntu-2404 timeout-minutes: 60 @@ -266,6 +275,9 @@ jobs: with: persist-credentials: false + - name: Start CI telemetry + uses: leynos/shared-actions/.github/actions/ci-telemetry-start@81374b6f183e59f0570ad36217ded90d3240fda6 + - name: Bound concurrency to the runner shape run: | { @@ -455,6 +467,12 @@ jobs: # or reads); isolated errors only warn. run: python3 scripts/check_sccache_health.py --expect-location ghac sccache-stats.json + - name: Report CI telemetry + if: always() + uses: leynos/shared-actions/.github/actions/ci-telemetry-finish@81374b6f183e59f0570ad36217ded90d3240fda6 + with: + sccache-status: ${{ steps.setup-rust.outputs.sccache-status }} + - name: Record disk headroom before the saves if: always() run: bash scripts/record-cache-observations.sh headroom @@ -521,6 +539,9 @@ jobs: with: persist-credentials: false + - name: Start CI telemetry + uses: leynos/shared-actions/.github/actions/ci-telemetry-start@81374b6f183e59f0570ad36217ded90d3240fda6 + - name: Restore Cargo registry id: cargo-registry uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -593,6 +614,12 @@ jobs: if-no-files-found: error retention-days: 14 + - name: Report CI telemetry + if: always() + uses: leynos/shared-actions/.github/actions/ci-telemetry-finish@81374b6f183e59f0570ad36217ded90d3240fda6 + with: + sccache-status: ${{ steps.setup-rust.outputs.sccache-status }} + - name: Record disk headroom before the saves if: always() run: bash scripts/record-cache-observations.sh headroom diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 35ab7366..b8602d18 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -833,6 +833,20 @@ every step that records, uploads or health-checks the statistics by the command it runs, in any job, and refuses one whose condition omits, inverts, negates or `||`-composes the guard. +Every job in `ci.yml` also starts `ci-telemetry-start` right after the checkout +and ends its work with `ci-telemetry-finish` under `if: always()`, both from +shared-actions and pinned to one commit. Finish prints one `DF12_CI_TELEMETRY` +line that the central CI observability collector reads from the job log: the +busy cores (average, 95th percentile and maximum), the peak memory and the +smallest free disk, plus the sccache hit ratio. It is given `setup-rust`'s +`sccache-status` and reads the `sccache-stats.json` that +`scripts/record-sccache-effectiveness.sh` writes, so it asks sccache for +nothing a second time, and a fallback run carries null counts rather than zeros. +`ci_telemetry_test.py` holds the placement, the `if: always()`, the status +wiring and the pin. `windows-compat` carries the pair too; its record says +`unsupported-os` and has no CPU figures. A job added to `ci.yml` without the +pair reads as a missing record to the collector. + On the three lanes that use the `gha` backend, `coverage-check`, `linux-full` and `coverage-upload`, the statistics are uploaded as a `sccache-stats-` artefact under `if: always()`, and then diff --git a/tests/workflow_contracts/ci_telemetry_test.py b/tests/workflow_contracts/ci_telemetry_test.py new file mode 100644 index 00000000..2e5cf6fc --- /dev/null +++ b/tests/workflow_contracts/ci_telemetry_test.py @@ -0,0 +1,103 @@ +"""Contracts for the CI telemetry that every `ci.yml` job carries. + +The central CI observability collector reads one `DF12_CI_TELEMETRY` record +from each job in a listed workflow. `ci.yml` is listed, so each of its jobs +starts `ci-telemetry-start` right after the checkout and ends its work with +`ci-telemetry-finish` under `if: always()`: a job killed by its shape is the one +whose record matters most. A job without the pair reads as a missing record to +the collector, which is the fault the coverage numbers count. + +`ci.yml` has no roll-up job, so there are no exemptions here. The finish step +passes `setup-rust`'s `sccache-status`, and reads the `sccache-stats.json` that +`scripts/record-sccache-effectiveness.sh` writes, which is the action's default +file, so the record carries the hit ratio without a second statistics call. + +Run via ``make test-workflow-contracts``. +""" + +from __future__ import annotations + +import typing as typ + +import pytest + +from ubicloud_workflow_support import all_jobs + +PIN = "81374b6f183e59f0570ad36217ded90d3240fda6" +ACTIONS = "leynos/shared-actions/.github/actions" +START = f"{ACTIONS}/ci-telemetry-start@{PIN}" +FINISH = f"{ACTIONS}/ci-telemetry-finish@{PIN}" +STATUS = "${{ steps.setup-rust.outputs.sccache-status }}" +#: What may follow the report: the disk record and the cache saves, which are +#: housekeeping after the measured work. +AFTER_REPORT_PREFIXES = ("actions/cache/save@",) +AFTER_REPORT_NAMES = frozenset({"Record disk headroom before the saves"}) + +Job = tuple[str, str, dict[str, typ.Any]] + + +def _ci_jobs() -> list[Job]: + return [j for j in all_jobs() if j[0] == "ci.yml"] + + +def _ids(jobs: list[Job]) -> list[str]: + return [f"{w}:{n}" for w, n, _ in jobs] + + +def _steps(job: dict[str, typ.Any]) -> list[dict[str, typ.Any]]: + return [s for s in job.get("steps", []) if isinstance(s, dict)] + + +def _uses(step: dict[str, typ.Any]) -> str: + return str(step.get("uses", "")) + + +def test_the_selector_matches_the_three_jobs() -> None: + """Guard against a selector that silently matches nothing.""" + assert {n for _, n, _ in _ci_jobs()} == { + "coverage-check", + "linux-full", + "windows-compat", + } + + +@pytest.mark.parametrize("job", _ci_jobs(), ids=_ids(_ci_jobs())) +def test_the_start_follows_the_checkout_directly(job: Job) -> None: + """Start telemetry right after the checkout, before any other work.""" + steps = _steps(job[2]) + assert _uses(steps[0]).startswith("actions/checkout@") + assert _uses(steps[1]) == START, f"{job[1]} must start telemetry second" + assert sum(_uses(s) == START for s in steps) == 1 + + +@pytest.mark.parametrize("job", _ci_jobs(), ids=_ids(_ci_jobs())) +def test_the_report_runs_always_after_the_work(job: Job) -> None: + """Report once, under `if: always()`, with only housekeeping after it.""" + steps = _steps(job[2]) + at = [i for i, s in enumerate(steps) if _uses(s) == FINISH] + assert len(at) == 1, f"{job[1]} must report telemetry exactly once" + report = steps[at[0]] + assert report.get("if") == "always()" + for step in steps[at[0] + 1 :]: + assert ( + _uses(step).startswith(AFTER_REPORT_PREFIXES) + or step.get("name") in AFTER_REPORT_NAMES + ), f"{job[1]}: {step.get('name')!r} follows the report but is work" + + +@pytest.mark.parametrize("job", _ci_jobs(), ids=_ids(_ci_jobs())) +def test_the_report_carries_the_sccache_status(job: Job) -> None: + """The record learns setup-rust's status, so a fallback never reads healthy.""" + steps = _steps(job[2]) + assert any(s.get("id") == "setup-rust" for s in steps) + report = next(s for s in steps if _uses(s) == FINISH) + assert (report.get("with") or {}).get("sccache-status") == STATUS + + +@pytest.mark.parametrize("job", _ci_jobs(), ids=_ids(_ci_jobs())) +def test_the_report_follows_the_statistics_it_reads(job: Job) -> None: + """Finish reads `sccache-stats.json`, so the recorder must run before it.""" + steps = _steps(job[2]) + names = [str(s.get("name", "")) for s in steps] + report_at = next(i for i, s in enumerate(steps) if _uses(s) == FINISH) + assert names.index("Record sccache effectiveness") < report_at From 3bdcad47e1b25fdfffb6255d98fe43b96cbebefc Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Sat, 17 Oct 2026 20:58:29 +0100 Subject: [PATCH 2/2] Document the telemetry contract helpers --- tests/workflow_contracts/ci_telemetry_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/workflow_contracts/ci_telemetry_test.py b/tests/workflow_contracts/ci_telemetry_test.py index 2e5cf6fc..cbff40e3 100644 --- a/tests/workflow_contracts/ci_telemetry_test.py +++ b/tests/workflow_contracts/ci_telemetry_test.py @@ -37,18 +37,22 @@ def _ci_jobs() -> list[Job]: + """Return the jobs of `ci.yml`.""" return [j for j in all_jobs() if j[0] == "ci.yml"] def _ids(jobs: list[Job]) -> list[str]: + """Return readable parameter identifiers for a job list.""" return [f"{w}:{n}" for w, n, _ in jobs] def _steps(job: dict[str, typ.Any]) -> list[dict[str, typ.Any]]: + """Return a job's step mappings.""" return [s for s in job.get("steps", []) if isinstance(s, dict)] def _uses(step: dict[str, typ.Any]) -> str: + """Return a step's `uses` reference, or an empty string.""" return str(step.get("uses", ""))