From 9ae81fd0a911dd7edb032ad50d2590537bb6843c Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Mon, 19 Oct 2026 14:17:36 +0100 Subject: [PATCH 1/3] Free the unused Android SDK and .NET before the compiler cache is written A version bump misses every Rust sccache entry, so the archive grows to about 3 GiB and no longer fits in the 2 GB left on ubicloud-standard-2. Free the two preinstalled toolchains in every job that saves a compiler cache, and add a contract that holds each such job to it. --- .github/workflows/ci.yml | 45 +++++++++++++++++++++++++++++++ .github/workflows/loom.yml | 15 +++++++++++ tests/test_ci_free_disk.py | 54 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 tests/test_ci_free_disk.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a611ffc0..2045a6a47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,21 @@ jobs: # token must not stay in .git/config where that code could reach it. persist-credentials: false + # The preinstalled Android SDK and .NET are 16.9 GB on a 73 GB volume and + # nothing here uses them. A version bump misses every Rust compiler-cache + # entry, so the archive grows to about 3 GiB and, with 2 GB free, cannot + # be written next to the directory it packs (run 38987616012 onward). The + # step runs before the first cache restore; `rm -rf` of an absent path is + # a no-op, so an image without them still passes. + - name: Free preinstalled toolchains the job does not use + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + df -m / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet + df -m / + - name: Start resource sampler uses: ./.github/actions/resource-sampler with: @@ -504,6 +519,16 @@ jobs: with: persist-credentials: false + # Frees the same 16.9 GB as in `lint-test`; see the comment there. + - name: Free preinstalled toolchains the job does not use + if: runner.os == 'Linux' && env.LEG_RUNS == 'true' + shell: bash + run: | + set -euo pipefail + df -m / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet + df -m / + # Started before the first cache restore, so the archive downloads and # every build below fall inside the measured window. `ubicloud-standard-2` # starts with about 31 GB free, and disk, not memory, is what kills a job @@ -748,6 +773,16 @@ jobs: # must not stay in .git/config where that code could reach it. persist-credentials: false + # Frees the same 16.9 GB as in `lint-test`; see the comment there. + - name: Free preinstalled toolchains the job does not use + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + df -m / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet + df -m / + # Started before the first cache restore, so the archive downloads and # every build below fall inside the measured window. `ubicloud-standard-2` # starts with about 31 GB free, and disk, not memory, is what kills a job @@ -1648,6 +1683,16 @@ jobs: fetch-depth: 0 persist-credentials: false + # Frees the same 16.9 GB as in `lint-test`; see the comment there. + - name: Free preinstalled toolchains the job does not use + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + df -m / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet + df -m / + # No resource sampler here, deliberately. This job times commands that # each run for about half a second and compares the ratio of two of # them, so a background loop waking every 15 s to shell out to `free` diff --git a/.github/workflows/loom.yml b/.github/workflows/loom.yml index 949ff2a76..01f4ba97d 100644 --- a/.github/workflows/loom.yml +++ b/.github/workflows/loom.yml @@ -30,6 +30,21 @@ jobs: with: persist-credentials: false + # The preinstalled Android SDK and .NET are 16.9 GB on a 73 GB volume and + # nothing here uses them. A version bump misses every Rust compiler-cache + # entry, so the archive grows to about 3 GiB and, with 2 GB free, cannot + # be written next to the directory it packs (run 38987616012 onward). The + # step runs before the first cache restore; `rm -rf` of an absent path is + # a no-op, so an image without them still passes. + - name: Free preinstalled toolchains the job does not use + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + df -m / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet + df -m / + - name: Compute cache keys uses: ./.github/actions/cache-keys with: diff --git a/tests/test_ci_free_disk.py b/tests/test_ci_free_disk.py new file mode 100644 index 000000000..be8e01c4e --- /dev/null +++ b/tests/test_ci_free_disk.py @@ -0,0 +1,54 @@ +"""Contract test that cache-writing jobs free disk before any cache step. + +A Rust compiler-cache miss (every version bump causes one) grows the archive to +about 3 GiB, which does not fit beside the directory it packs on the 2 GB a +Linux job is left with. Pull-request runs never save, so only a push to `main` +shows the failure. +""" + +from __future__ import annotations + +import pytest + +from tests.helpers.ci_runners import ( + CACHE_PLAIN, + CACHE_RESTORE, + CACHE_SAVE, + CACHE_WRITERS, + steps, +) + +FREE_DISK_STEP = "Free preinstalled toolchains the job does not use" +FREED_DIRECTORIES = ("/usr/local/lib/android", "/usr/share/dotnet") +CACHE_USES = (CACHE_RESTORE, CACHE_SAVE, CACHE_PLAIN) + + +@pytest.mark.parametrize( + ("workflow_name", "job_name"), sorted(CACHE_WRITERS["SCCACHE_CACHE_KEY"]) +) +def test_compiler_cache_writers_free_disk_before_any_cache_step( + workflow_name: str, job_name: str +) -> None: + """Leave room for the archive: free unused toolchains before every cache.""" + job_steps = steps(workflow_name, job_name) + freeing = [ + index + for index, step in enumerate(job_steps) + if step.get("name") == FREE_DISK_STEP + ] + where = f"{workflow_name}:{job_name}" + assert len(freeing) == 1, f"{where} must declare {FREE_DISK_STEP!r} once" + step = job_steps[freeing[0]] + command = step.get("run") + assert isinstance(command, str), f"{where} freeing step must run a command" + missing = [path for path in FREED_DIRECTORIES if path not in command] + assert not missing, f"{where} freeing step must remove {missing}" + assert "runner.os == 'Linux'" in str(step.get("if", "")), ( + f"{where} freeing step must be guarded to Linux runners" + ) + first_cache = min( + index + for index, other in enumerate(job_steps) + if other.get("uses") in CACHE_USES + ) + assert freeing[0] < first_cache, f"{where} must free disk before its first cache" From 7f7be35dc5cc1dc07ed5bf9074d746fa1686cad5 Mon Sep 17 00:00:00 2001 From: leynos Date: Tue, 20 Oct 2026 15:31:40 +0200 Subject: [PATCH 2/3] Strengthen disk cleanup contract coverage Parse cleanup commands as shell invocations so workflow contracts reject mere path mentions that do not recursively remove both directories. Document the cleanup steps and cache-writer conditions for maintainers. --- docs/developers-guide.md | 10 +++++ tests/test_ci_free_disk.py | 80 +++++++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 6881b4d69..41cb67ac1 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -307,6 +307,16 @@ miss can be explained from the run alone. | `tool-` | `~/.cargo/bin`, `~/.local/bin`, `~/.cache/uv`, `~/.local/share/uv`, `.uv-cache`, `.uv-tools` | the above plus Ubuntu release, Python version, nextest pin, hash of `uv.lock`, `pyproject.toml`, `Makefile`, and the sccache action | `typecheck-test` (3.12, 3.14, 3.15a), `extension-tests` (3.13) | | `sccache-` | `~/.cache/sccache` | generation, OS, arch, runner environment, Ubuntu release, run identifier | `typecheck-test`, `lint-test`, and the other compiling jobs | +Compiler-cache writers remove the unused preinstalled Android SDK and .NET +installation before their first cache operation. The cleanup step in +`.github/workflows/ci.yml` and `.github/workflows/loom.yml` is guarded by +`runner.os == 'Linux'` and removes `/usr/local/lib/android` and +`/usr/share/dotnet`. It runs in `lint-test`, `typecheck-test`, +`extension-tests`, `benchmark-ratchet`, and `loom`, without changing runner +selection. The `typecheck-test` condition also requires `LEG_RUNS == 'true'`. +`tests/test_ci_free_disk.py` enforces one cleanup step per writer, both removal +paths, the Linux guard, and placement before the first cache operation. + Four rules hold that table together, each with a contract test: - **One owner per path.** No two cache steps in a job list the same path. diff --git a/tests/test_ci_free_disk.py b/tests/test_ci_free_disk.py index be8e01c4e..88c1c005e 100644 --- a/tests/test_ci_free_disk.py +++ b/tests/test_ci_free_disk.py @@ -8,6 +8,8 @@ from __future__ import annotations +import shlex + import pytest from tests.helpers.ci_runners import ( @@ -23,6 +25,79 @@ CACHE_USES = (CACHE_RESTORE, CACHE_SAVE, CACHE_PLAIN) +def _is_recursive_removal(tokens: list[str], paths: tuple[str, ...]) -> bool: + """Return whether a command segment recursively removes every path.""" + command = tokens[1:] if tokens[:1] == ["sudo"] else tokens + if not command or command[0] != "rm": + return False + + arguments = command[1:] + end_options = arguments.index("--") if "--" in arguments else len(arguments) + options = arguments[:end_options] + operands = ( + arguments[end_options + 1 :] if end_options < len(arguments) else arguments + ) + is_recursive = any( + option == "--recursive" + or ( + option.startswith("-") and not option.startswith("--") and "r" in option[1:] + ) + for option in options + ) + return is_recursive and all(path in operands for path in paths) + + +def _has_recursive_removal(command: str, paths: tuple[str, ...]) -> bool: + """Find a shell command segment that recursively removes every path.""" + for line in command.splitlines(): + lexer = shlex.shlex(line, posix=True, punctuation_chars=";&|") + lexer.whitespace_split = True + segments: list[list[str]] = [[]] + for token in lexer: + if token and not token.strip(";&|"): + segments.append([]) + else: + segments[-1].append(token) + if any(_is_recursive_removal(segment, paths) for segment in segments): + return True + return False + + +@pytest.mark.parametrize( + ("command", "should_remove"), + [ + ("sudo rm -rf /usr/local/lib/android /usr/share/dotnet", True), + ("rm --recursive --force /usr/local/lib/android /usr/share/dotnet", True), + ("echo /usr/local/lib/android /usr/share/dotnet", False), + ("test -d /usr/local/lib/android && test -d /usr/share/dotnet", False), + ( + "printf '%s\\n' 'sudo rm -rf /usr/local/lib/android /usr/share/dotnet'", + False, + ), + ("# sudo rm -rf /usr/local/lib/android /usr/share/dotnet", False), + ("sudo rm -f /usr/local/lib/android /usr/share/dotnet", False), + ("sudo rm -rf /usr/local/lib/android && echo /usr/share/dotnet", False), + ], + ids=[ + "sudo-recursive-removal", + "long-recursive-option", + "printed-paths", + "inspected-paths", + "quoted-diagnostic", + "commented-command", + "non-recursive-removal", + "paths-in-separate-commands", + ], +) +def test_cleanup_command_must_recursively_remove_both_directories( + command: str, should_remove: bool +) -> None: + """Only an executable recursive removal of both paths satisfies cleanup.""" + assert _has_recursive_removal(command, FREED_DIRECTORIES) is should_remove, ( + f"unexpected recursive removal result for {command!r}" + ) + + @pytest.mark.parametrize( ("workflow_name", "job_name"), sorted(CACHE_WRITERS["SCCACHE_CACHE_KEY"]) ) @@ -41,8 +116,9 @@ def test_compiler_cache_writers_free_disk_before_any_cache_step( step = job_steps[freeing[0]] command = step.get("run") assert isinstance(command, str), f"{where} freeing step must run a command" - missing = [path for path in FREED_DIRECTORIES if path not in command] - assert not missing, f"{where} freeing step must remove {missing}" + assert _has_recursive_removal(command, FREED_DIRECTORIES), ( + f"{where} freeing step must recursively remove {FREED_DIRECTORIES}" + ) assert "runner.os == 'Linux'" in str(step.get("if", "")), ( f"{where} freeing step must be guarded to Linux runners" ) From 7c4e7041542aac8e7dd1190b32dbc02c883cc2d0 Mon Sep 17 00:00:00 2001 From: leynos Date: Tue, 20 Oct 2026 18:00:31 +0200 Subject: [PATCH 3/3] Simplify disk cleanup command parsing Split recursive-option recognition and argument parsing into focused helpers. This keeps the workflow contract test readable and below the CodeScene complexity threshold. --- tests/test_ci_free_disk.py | 44 ++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/tests/test_ci_free_disk.py b/tests/test_ci_free_disk.py index 88c1c005e..d6b308775 100644 --- a/tests/test_ci_free_disk.py +++ b/tests/test_ci_free_disk.py @@ -28,23 +28,39 @@ def _is_recursive_removal(tokens: list[str], paths: tuple[str, ...]) -> bool: """Return whether a command segment recursively removes every path.""" command = tokens[1:] if tokens[:1] == ["sudo"] else tokens - if not command or command[0] != "rm": - return False - arguments = command[1:] - end_options = arguments.index("--") if "--" in arguments else len(arguments) - options = arguments[:end_options] - operands = ( - arguments[end_options + 1 :] if end_options < len(arguments) else arguments + options, operands = _split_rm_arguments(arguments) + return ( + command[:1] == ["rm"] + and _has_recursive_option(options) + and all(path in operands for path in paths) ) - is_recursive = any( - option == "--recursive" - or ( - option.startswith("-") and not option.startswith("--") and "r" in option[1:] - ) - for option in options + + +def _has_recursive_option(options: list[str]) -> bool: + """Return whether rm options request recursive removal.""" + return any(_is_recursive_option(option) for option in options) + + +def _is_recursive_option(option: str) -> bool: + """Recognize GNU rm's recursive long and short options.""" + return option == "--recursive" or ( + option.startswith("-") + and not option.startswith("--") + and any(flag in option[1:] for flag in "rR") ) - return is_recursive and all(path in operands for path in paths) + + +def _split_rm_arguments(arguments: list[str]) -> tuple[list[str], list[str]]: + """Separate rm options and operands, respecting the option terminator.""" + try: + end_options = arguments.index("--") + except ValueError: + return ( + [argument for argument in arguments if argument.startswith("-")], + [argument for argument in arguments if not argument.startswith("-")], + ) + return arguments[:end_options], arguments[end_options + 1 :] def _has_recursive_removal(command: str, paths: tuple[str, ...]) -> bool: