diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a611ffc0..2045a6a47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,21 @@ jobs: # token must not stay in .git/config where that code could reach it. persist-credentials: false + # The preinstalled Android SDK and .NET are 16.9 GB on a 73 GB volume and + # nothing here uses them. A version bump misses every Rust compiler-cache + # entry, so the archive grows to about 3 GiB and, with 2 GB free, cannot + # be written next to the directory it packs (run 38987616012 onward). The + # step runs before the first cache restore; `rm -rf` of an absent path is + # a no-op, so an image without them still passes. + - name: Free preinstalled toolchains the job does not use + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + df -m / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet + df -m / + - name: Start resource sampler uses: ./.github/actions/resource-sampler with: @@ -504,6 +519,16 @@ jobs: with: persist-credentials: false + # Frees the same 16.9 GB as in `lint-test`; see the comment there. + - name: Free preinstalled toolchains the job does not use + if: runner.os == 'Linux' && env.LEG_RUNS == 'true' + shell: bash + run: | + set -euo pipefail + df -m / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet + df -m / + # Started before the first cache restore, so the archive downloads and # every build below fall inside the measured window. `ubicloud-standard-2` # starts with about 31 GB free, and disk, not memory, is what kills a job @@ -748,6 +773,16 @@ jobs: # must not stay in .git/config where that code could reach it. persist-credentials: false + # Frees the same 16.9 GB as in `lint-test`; see the comment there. + - name: Free preinstalled toolchains the job does not use + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + df -m / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet + df -m / + # Started before the first cache restore, so the archive downloads and # every build below fall inside the measured window. `ubicloud-standard-2` # starts with about 31 GB free, and disk, not memory, is what kills a job @@ -1648,6 +1683,16 @@ jobs: fetch-depth: 0 persist-credentials: false + # Frees the same 16.9 GB as in `lint-test`; see the comment there. + - name: Free preinstalled toolchains the job does not use + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + df -m / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet + df -m / + # No resource sampler here, deliberately. This job times commands that # each run for about half a second and compares the ratio of two of # them, so a background loop waking every 15 s to shell out to `free` diff --git a/.github/workflows/loom.yml b/.github/workflows/loom.yml index 949ff2a76..01f4ba97d 100644 --- a/.github/workflows/loom.yml +++ b/.github/workflows/loom.yml @@ -30,6 +30,21 @@ jobs: with: persist-credentials: false + # The preinstalled Android SDK and .NET are 16.9 GB on a 73 GB volume and + # nothing here uses them. A version bump misses every Rust compiler-cache + # entry, so the archive grows to about 3 GiB and, with 2 GB free, cannot + # be written next to the directory it packs (run 38987616012 onward). The + # step runs before the first cache restore; `rm -rf` of an absent path is + # a no-op, so an image without them still passes. + - name: Free preinstalled toolchains the job does not use + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + df -m / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet + df -m / + - name: Compute cache keys uses: ./.github/actions/cache-keys with: diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 6881b4d69..41cb67ac1 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -307,6 +307,16 @@ miss can be explained from the run alone. | `tool-` | `~/.cargo/bin`, `~/.local/bin`, `~/.cache/uv`, `~/.local/share/uv`, `.uv-cache`, `.uv-tools` | the above plus Ubuntu release, Python version, nextest pin, hash of `uv.lock`, `pyproject.toml`, `Makefile`, and the sccache action | `typecheck-test` (3.12, 3.14, 3.15a), `extension-tests` (3.13) | | `sccache-` | `~/.cache/sccache` | generation, OS, arch, runner environment, Ubuntu release, run identifier | `typecheck-test`, `lint-test`, and the other compiling jobs | +Compiler-cache writers remove the unused preinstalled Android SDK and .NET +installation before their first cache operation. The cleanup step in +`.github/workflows/ci.yml` and `.github/workflows/loom.yml` is guarded by +`runner.os == 'Linux'` and removes `/usr/local/lib/android` and +`/usr/share/dotnet`. It runs in `lint-test`, `typecheck-test`, +`extension-tests`, `benchmark-ratchet`, and `loom`, without changing runner +selection. The `typecheck-test` condition also requires `LEG_RUNS == 'true'`. +`tests/test_ci_free_disk.py` enforces one cleanup step per writer, both removal +paths, the Linux guard, and placement before the first cache operation. + Four rules hold that table together, each with a contract test: - **One owner per path.** No two cache steps in a job list the same path. diff --git a/tests/test_ci_free_disk.py b/tests/test_ci_free_disk.py new file mode 100644 index 000000000..d6b308775 --- /dev/null +++ b/tests/test_ci_free_disk.py @@ -0,0 +1,146 @@ +"""Contract test that cache-writing jobs free disk before any cache step. + +A Rust compiler-cache miss (every version bump causes one) grows the archive to +about 3 GiB, which does not fit beside the directory it packs on the 2 GB a +Linux job is left with. Pull-request runs never save, so only a push to `main` +shows the failure. +""" + +from __future__ import annotations + +import shlex + +import pytest + +from tests.helpers.ci_runners import ( + CACHE_PLAIN, + CACHE_RESTORE, + CACHE_SAVE, + CACHE_WRITERS, + steps, +) + +FREE_DISK_STEP = "Free preinstalled toolchains the job does not use" +FREED_DIRECTORIES = ("/usr/local/lib/android", "/usr/share/dotnet") +CACHE_USES = (CACHE_RESTORE, CACHE_SAVE, CACHE_PLAIN) + + +def _is_recursive_removal(tokens: list[str], paths: tuple[str, ...]) -> bool: + """Return whether a command segment recursively removes every path.""" + command = tokens[1:] if tokens[:1] == ["sudo"] else tokens + arguments = command[1:] + options, operands = _split_rm_arguments(arguments) + return ( + command[:1] == ["rm"] + and _has_recursive_option(options) + and all(path in operands for path in paths) + ) + + +def _has_recursive_option(options: list[str]) -> bool: + """Return whether rm options request recursive removal.""" + return any(_is_recursive_option(option) for option in options) + + +def _is_recursive_option(option: str) -> bool: + """Recognize GNU rm's recursive long and short options.""" + return option == "--recursive" or ( + option.startswith("-") + and not option.startswith("--") + and any(flag in option[1:] for flag in "rR") + ) + + +def _split_rm_arguments(arguments: list[str]) -> tuple[list[str], list[str]]: + """Separate rm options and operands, respecting the option terminator.""" + try: + end_options = arguments.index("--") + except ValueError: + return ( + [argument for argument in arguments if argument.startswith("-")], + [argument for argument in arguments if not argument.startswith("-")], + ) + return arguments[:end_options], arguments[end_options + 1 :] + + +def _has_recursive_removal(command: str, paths: tuple[str, ...]) -> bool: + """Find a shell command segment that recursively removes every path.""" + for line in command.splitlines(): + lexer = shlex.shlex(line, posix=True, punctuation_chars=";&|") + lexer.whitespace_split = True + segments: list[list[str]] = [[]] + for token in lexer: + if token and not token.strip(";&|"): + segments.append([]) + else: + segments[-1].append(token) + if any(_is_recursive_removal(segment, paths) for segment in segments): + return True + return False + + +@pytest.mark.parametrize( + ("command", "should_remove"), + [ + ("sudo rm -rf /usr/local/lib/android /usr/share/dotnet", True), + ("rm --recursive --force /usr/local/lib/android /usr/share/dotnet", True), + ("echo /usr/local/lib/android /usr/share/dotnet", False), + ("test -d /usr/local/lib/android && test -d /usr/share/dotnet", False), + ( + "printf '%s\\n' 'sudo rm -rf /usr/local/lib/android /usr/share/dotnet'", + False, + ), + ("# sudo rm -rf /usr/local/lib/android /usr/share/dotnet", False), + ("sudo rm -f /usr/local/lib/android /usr/share/dotnet", False), + ("sudo rm -rf /usr/local/lib/android && echo /usr/share/dotnet", False), + ], + ids=[ + "sudo-recursive-removal", + "long-recursive-option", + "printed-paths", + "inspected-paths", + "quoted-diagnostic", + "commented-command", + "non-recursive-removal", + "paths-in-separate-commands", + ], +) +def test_cleanup_command_must_recursively_remove_both_directories( + command: str, should_remove: bool +) -> None: + """Only an executable recursive removal of both paths satisfies cleanup.""" + assert _has_recursive_removal(command, FREED_DIRECTORIES) is should_remove, ( + f"unexpected recursive removal result for {command!r}" + ) + + +@pytest.mark.parametrize( + ("workflow_name", "job_name"), sorted(CACHE_WRITERS["SCCACHE_CACHE_KEY"]) +) +def test_compiler_cache_writers_free_disk_before_any_cache_step( + workflow_name: str, job_name: str +) -> None: + """Leave room for the archive: free unused toolchains before every cache.""" + job_steps = steps(workflow_name, job_name) + freeing = [ + index + for index, step in enumerate(job_steps) + if step.get("name") == FREE_DISK_STEP + ] + where = f"{workflow_name}:{job_name}" + assert len(freeing) == 1, f"{where} must declare {FREE_DISK_STEP!r} once" + step = job_steps[freeing[0]] + command = step.get("run") + assert isinstance(command, str), f"{where} freeing step must run a command" + assert _has_recursive_removal(command, FREED_DIRECTORIES), ( + f"{where} freeing step must recursively remove {FREED_DIRECTORIES}" + ) + assert "runner.os == 'Linux'" in str(step.get("if", "")), ( + f"{where} freeing step must be guarded to Linux runners" + ) + first_cache = min( + index + for index, other in enumerate(job_steps) + if other.get("uses") in CACHE_USES + ) + assert freeing[0] < first_cache, f"{where} must free disk before its first cache"