diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 79f65c3..1b22214 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -19,10 +19,10 @@ env: jobs: bootstrap: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 30 env: - DEVENV_FETCH_BRANCH: master + DEVENV_FETCH_BRANCH: devenv-post-fetch-linux-apt-packages SNTY_DEVENV_BRANCH: "${{ github.event.pull_request && github.head_ref || github.ref_name }}" steps: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f8c7970..9f8ba8c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -22,7 +22,7 @@ env: jobs: bootstrap: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 20 env: SENTRY_BRANCH: master diff --git a/README.md b/README.md index 886c39d..dce72a6 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Everything devenv needs is in `~/.local/share/sentry-devenv`. - `direnv` - we currently rely on direnv and a minimal [`[reporoot]/.envrc`](#direnv) to add `[reporoot]/.devenv/bin` to PATH - see [examples](#examples) for .envrc suggestions - - global tools: `docker` (cli), `colima` + - global tools (macos only; you are otherwise expected to install docker yourself): `docker` (cli), `colima` ### runtime diff --git a/devenv/bootstrap.py b/devenv/bootstrap.py index 0dcbfeb..b3f146a 100644 --- a/devenv/bootstrap.py +++ b/devenv/bootstrap.py @@ -54,9 +54,15 @@ def main(context: Context, argv: Sequence[str] | None = None) -> ExitCode: except RuntimeError: return "Failed to find git. Run xcode-select --install, then re-run bootstrap when done." + if constants.LINUX and not ( + shutil.which("docker") and shutil.which("dockerd") + ): + raise SystemExit("docker engine not installed; required on linux") + # even though this is called before colima starts, # better to try and potentially (although unlikely) fail earlier rather than later - rosetta.ensure() + if constants.DARWIN: + rosetta.ensure() github.add_to_known_hosts() @@ -103,11 +109,15 @@ def main(context: Context, argv: Sequence[str] | None = None) -> ExitCode: """ ) os.makedirs(f"{constants.root}/bin", exist_ok=True) - brew.install() - docker.install_global() + + if constants.DARWIN: + # we only install brew and colima-related stuff on macos + brew.install() + docker.install_global() + colima.install_global() + limactl.install_global() + direnv.install() - colima.install_global() - limactl.install_global() os.makedirs(context["code_root"], exist_ok=True) diff --git a/devenv/checks/colimaSsh.py b/devenv/checks/colimaSsh.py index 8a8ea05..e3c9bef 100644 --- a/devenv/checks/colimaSsh.py +++ b/devenv/checks/colimaSsh.py @@ -8,7 +8,7 @@ from devenv.lib_check.types import checker from devenv.lib_check.types import fixer -tags: set[str] = {"builtin"} +tags: set[str] = {"builtin", "colima"} name = "colima ssh credentials should only be owner rw" diff --git a/devenv/checks/diskfree.py b/devenv/checks/diskfree.py index 51e2b70..8293ad4 100644 --- a/devenv/checks/diskfree.py +++ b/devenv/checks/diskfree.py @@ -14,11 +14,11 @@ def check() -> tuple[bool, str]: disk_total, disk_used, disk_free = shutil.disk_usage("/") disk_gib_free = disk_free / (1024**3) - if disk_gib_free < 10000: + if disk_gib_free < 10: return ( False, f"You have less than 10 GiB disk free ({disk_gib_free} GiB free). " - "You might start to encounter various problems when using colima.", + "You might start to encounter various problems when using docker.", ) return True, "" diff --git a/devenv/checks/dockerConfig.py b/devenv/checks/dockerConfig.py index e9cdd13..5c66318 100644 --- a/devenv/checks/dockerConfig.py +++ b/devenv/checks/dockerConfig.py @@ -7,8 +7,8 @@ from devenv.lib_check.types import checker from devenv.lib_check.types import fixer -tags: set[str] = {"builtin"} -name = "correct docker configuration" +tags: set[str] = {"builtin", "colima"} +name = "correct docker configuration for colima" @checker diff --git a/devenv/checks/dockerDesktop.py b/devenv/checks/dockerDesktop.py index 868e8b4..c144895 100644 --- a/devenv/checks/dockerDesktop.py +++ b/devenv/checks/dockerDesktop.py @@ -6,8 +6,8 @@ from devenv.lib_check.types import checker from devenv.lib_check.types import fixer -tags: set[str] = {"builtin"} -name = "docker desktop shouldn't be running" +tags: set[str] = {"builtin", "colima"} +name = "docker desktop shouldn't be running if you're trying to use colima" def docker_desktop_is_running() -> bool: @@ -20,7 +20,7 @@ def check() -> tuple[bool, str]: if docker_desktop_is_running(): return ( False, - "Docker Desktop is running. We don't support it, and it conflicts with colima.", + "Docker Desktop is running. It cannot be running at the same time as colima.", ) return True, "" diff --git a/devenv/checks/limaDns.py b/devenv/checks/limaDns.py index b2b6dfc..190fe91 100644 --- a/devenv/checks/limaDns.py +++ b/devenv/checks/limaDns.py @@ -7,7 +7,7 @@ from devenv.lib_check.types import checker from devenv.lib_check.types import fixer -tags: set[str] = {"builtin"} +tags: set[str] = {"builtin", "colima"} name = "colima's DNS isn't working" diff --git a/devenv/constants.py b/devenv/constants.py index 2f6d247..fd5e6d4 100644 --- a/devenv/constants.py +++ b/devenv/constants.py @@ -4,6 +4,7 @@ import os import platform import pwd +import shutil import typing troubleshooting_help = """\ @@ -27,6 +28,8 @@ SYSTEM = platform.system().lower() MACHINE = platform.machine() DARWIN = SYSTEM == "darwin" +# we only support apt-based linuxes +LINUX = shutil.which("dpkg") is not None INTEL_MAC = DARWIN and (MACHINE == "x86_64") SHELL_UNSET = "(SHELL unset)" DEBUG = os.getenv("SNTY_DEVENV_DEBUG", os.getenv("DEBUG", "")) diff --git a/devenv/doctor.py b/devenv/doctor.py index ab7f101..4f2c9e0 100644 --- a/devenv/doctor.py +++ b/devenv/doctor.py @@ -14,6 +14,7 @@ from typing import List import devenv.checks +from devenv import constants from devenv.lib.context import Context from devenv.lib.modules import DevModuleInfo from devenv.lib.repository import Repository @@ -78,7 +79,9 @@ def __init__(self, module: ModuleType): super().__init__() -def load_checks(repo: Repository, match_tags: set[str]) -> List[Check]: +def load_checks( + repo: Repository, match_tags: set[str], exclude_tags: set[str] +) -> List[Check]: """ Load all checks from the checks directory. Optionally filter by tags. @@ -104,11 +107,15 @@ def load_checks(repo: Repository, match_tags: set[str]) -> List[Check]: continue if match_tags and not check.tags.issuperset(match_tags): continue + if check.tags & exclude_tags: + continue checks.append(check) return checks -def load_builtin_checks(match_tags: set[str]) -> List[Check]: +def load_builtin_checks( + match_tags: set[str], exclude_tags: set[str] +) -> List[Check]: """ Loads builtin checks. Optionally filter by tags. @@ -126,6 +133,8 @@ def load_builtin_checks(match_tags: set[str]) -> List[Check]: continue if match_tags and not check.tags.issuperset(match_tags): continue + if check.tags & exclude_tags: + continue checks.append(check) return checks @@ -193,23 +202,33 @@ def main(context: Context, argv: Sequence[str] | None = None) -> int: action="append", help="Used to match a subset of checks.", ) + parser.add_argument( + "--exclude-tag", + type=str, + action="append", + help="Used to unmatch checks.", + ) parser.add_argument( "--check-only", action="store_true", help="Do not run fixers." ) args = parser.parse_args(argv) match_tags: set[str] = set(args.tag if args.tag else ()) + exclude_tags: set[str] = set(args.exclude_tag if args.exclude_tag else ()) + + if not constants.DARWIN: + exclude_tags.add("colima") # First, we load builtin checks. These are not repo specific. - checks = load_builtin_checks(match_tags) + checks = load_builtin_checks(match_tags, exclude_tags) # Then we load any repo specific checks if any. repo = context.get("repo") if repo is not None: - checks.extend(load_checks(repo, match_tags)) + checks.extend(load_checks(repo, match_tags, exclude_tags)) if not checks: - print(f"No checks found for tags: {args.tag}") + print("No checks found.") return 1 # We run every check on a separate thread, aggregate the results, diff --git a/devenv/fetch.py b/devenv/fetch.py index 0cebe34..bad76a0 100644 --- a/devenv/fetch.py +++ b/devenv/fetch.py @@ -6,10 +6,7 @@ import sys from collections.abc import Sequence -from devenv.constants import CI -from devenv.constants import DARWIN -from devenv.constants import EXTERNAL_CONTRIBUTOR -from devenv.constants import homebrew_bin +from devenv import constants from devenv.lib import proc from devenv.lib.context import Context from devenv.lib.modules import DevModuleInfo @@ -35,33 +32,16 @@ def main(context: Context, argv: Sequence[str] | None = None) -> ExitCode: "getsentry/sentry", "getsentry/getsentry", ]: - fetch(code_root, "getsentry/sentry", auth=CI is None, sync=False) - - if DARWIN: - print("Installing sentry's brew dependencies...") - if CI: - # Installing everything from brew takes too much time, - # and chromedriver cask flakes occasionally. Really all we need to - # set up the devenv is colima and docker-cli. - # This is also required for arm64 macOS GHA runners. - # We manage colima, so just need to install docker + qemu here. - proc.run(("brew", "install", "docker", "qemu")) - else: - proc.run( - (f"{homebrew_bin}/brew", "bundle"), - cwd=f"{code_root}/sentry", - ) - else: - print( - "Not on MacOS; assuming you have a docker cli and runtime installed." - ) + fetch( + code_root, "getsentry/sentry", auth=constants.CI is None, sync=False + ) proc.run( (sys.executable, "-P", "-m", "devenv", "sync"), cwd=f"{code_root}/sentry", ) - if not CI and not EXTERNAL_CONTRIBUTOR: + if not constants.CI and not constants.EXTERNAL_CONTRIBUTOR: fetch(code_root, "getsentry/getsentry") print( @@ -88,25 +68,24 @@ def fetch( if os.path.exists(reporoot): print(f"{reporoot} already exists") - return - - print(f"fetching {repo} into {reporoot}") - - additional_args = ( - # git@ clones forces the use of cloning through SSH which is what we want, - # though CI must clone open source repos via https (no git authentication) - (f"git@github.com:{repo}",) - if auth - else ( - "--depth", - "1", - "--single-branch", - f"--branch={os.environ['DEVENV_FETCH_BRANCH']}", - f"https://github.com/{repo}", + else: + print(f"fetching {repo} into {reporoot}") + + additional_args = ( + # git@ clones forces the use of cloning through SSH which is what we want, + # though CI must clone open source repos via https (no git authentication) + (f"git@github.com:{repo}",) + if auth + else ( + "--depth", + "1", + "--single-branch", + f"--branch={os.environ['DEVENV_FETCH_BRANCH']}", + f"https://github.com/{repo}", + ) ) - ) - proc.run(("git", "-C", coderoot, "clone", *additional_args), exit=True) + proc.run(("git", "-C", coderoot, "clone", *additional_args), exit=True) context_post_fetch = { "reporoot": reporoot, @@ -117,6 +96,7 @@ def fetch( # optional post-fetch, meant for recommended but not required defaults fp = f"{reporoot}/devenv/post_fetch.py" if os.path.exists(fp): + print(f"running {fp}") spec = importlib.util.spec_from_file_location("post_fetch", fp) module = importlib.util.module_from_spec(spec) # type: ignore @@ -127,6 +107,7 @@ def fetch( print(f"warning! failed running {fp} (code {rc})") if sync: + print("running devenv sync") proc.run((sys.executable, "-P", "-m", "devenv", "sync"), cwd=reporoot) diff --git a/devenv/lib/colima.py b/devenv/lib/colima.py index 2f7715d..1e448a1 100644 --- a/devenv/lib/colima.py +++ b/devenv/lib/colima.py @@ -63,8 +63,6 @@ def install_global() -> None: "darwin_x86_64_sha256": "791330c62c60389f70e5e1c33a56c35502a9e36e544a418daea0273e539acbf4", "darwin_arm64": f"https://github.com/abiosoft/colima/releases/download/{version}/colima-Darwin-arm64", "darwin_arm64_sha256": "c266fcb272b39221ef6152d2093bb02a1ebadc26042233ad359e1ae52d5d5922", - "linux_x86_64": f"https://github.com/abiosoft/colima/releases/download/{version}/colima-Linux-x86_64", - "linux_x86_64_sha256": "f2d6664a79ff3aa35f0718aac2ba9f6b531772e1464f3b096c1ac2aab404943e", } binroot = f"{root}/bin" diff --git a/devenv/lib/docker.py b/devenv/lib/docker.py index 8f72e4b..f2d4b48 100644 --- a/devenv/lib/docker.py +++ b/devenv/lib/docker.py @@ -113,8 +113,6 @@ def install_global() -> None: "darwin_x86_64_sha256": "1b621d4c9a57ff361811cf29754aafb0c28bc113c70011927af8d73c2c162186", "darwin_arm64": f"https://download.docker.com/mac/static/stable/aarch64/docker-{version}.tgz", "darwin_arm64_sha256": "9dae125282116146b06eb777c2125ddda6c0468c0b9ad6c72a82edbc6783a77b", - "linux_x86_64": f"https://download.docker.com/linux/static/stable/x86_64/docker-{version}.tgz", - "linux_x86_64_sha256": "9b4f6fe406e50f9085ee474c451e2bb5adb119a03591f467922d3b4e2ddf31d3", } version_buildx = "v0.22.0" @@ -123,8 +121,6 @@ def install_global() -> None: "darwin_x86_64_sha256": "5221ad6b8acd2283f8fbbeebc79ae4b657e83519ca1c1e4cfbb9405230b3d933", "darwin_arm64": f"https://github.com/docker/buildx/releases/download/{version_buildx}/buildx-{version_buildx}.darwin-arm64", "darwin_arm64_sha256": "5898c338abb1f673107bc087997dc3cb63b4ea66d304ce4223472f57bd8d616e", - "linux_x86_64": f"https://github.com/docker/buildx/releases/download/{version_buildx}/buildx-{version_buildx}.linux-amd64", - "linux_x86_64_sha256": "805195386fba0cea5a1487cf0d47da82a145ea0a792bd3fb477583e2dbcdcc2f", } binroot = f"{root}/bin" diff --git a/devenv/lib/limactl.py b/devenv/lib/limactl.py index a0d9edd..dba5476 100644 --- a/devenv/lib/limactl.py +++ b/devenv/lib/limactl.py @@ -37,10 +37,7 @@ def _install(url: str, sha256: str, into: str) -> None: archive_file = archive.download(url, sha256, dest=f"{tmpd}/download") # the archive from homebrew has a lima/version prefix - if url.startswith("https://ghcr.io/v2/homebrew"): - archive.unpack_strip_n(archive_file, tmpd, n=2) - else: - archive.unpack(archive_file, tmpd) + archive.unpack_strip_n(archive_file, tmpd, n=2) # the archive was atomically placed into tmpd so # these are on the same fs and can be atomically moved too @@ -75,10 +72,6 @@ def install_global() -> None: # arm64_sonoma "darwin_arm64": "https://ghcr.io/v2/homebrew/core/lima/blobs/sha256:8aeb0a3b7295f0c3e0c2a7a92a798a44397936e5bb732db825aee6da5e762d7a", "darwin_arm64_sha256": "8aeb0a3b7295f0c3e0c2a7a92a798a44397936e5bb732db825aee6da5e762d7a", - # on linux we use github releases since most people are probably not using - # linuxbrew and the go binary in homebrew links to linuxbrew's ld.so - "linux_x86_64": f"https://github.com/lima-vm/lima/releases/download/v{version}/lima-{version}-Linux-x86_64.tar.gz", - "linux_x86_64_sha256": "b109cac29569a4aacab01c588f922ea6c7e2ef06ce9260bbc4c382e475bc3b98", } binroot = f"{root}/bin" diff --git a/devenv/update.py b/devenv/update.py index 556ba01..09a922f 100644 --- a/devenv/update.py +++ b/devenv/update.py @@ -36,11 +36,16 @@ def main(context: Context, argv: Sequence[str] | None = None) -> int: """ ) os.makedirs(f"{constants.root}/bin", exist_ok=True) - brew.install() - docker.install_global() + + if constants.DARWIN: + # we only install brew and colima-related stuff on macos + brew.install() + docker.install_global() + colima.install_global() + limactl.install_global() + direnv.install() - colima.install_global() - limactl.install_global() + return 0 is_global_devenv = sys.executable.startswith( diff --git a/tests/doctor/test_load_checks.py b/tests/doctor/test_load_checks.py index 60b5017..33af123 100644 --- a/tests/doctor/test_load_checks.py +++ b/tests/doctor/test_load_checks.py @@ -9,12 +9,15 @@ def test_load_checks_no_checks() -> None: - assert doctor.load_checks(Repository("not a real repository"), set()) == [] + assert ( + doctor.load_checks(Repository("not a real repository"), set(), set()) + == [] + ) def test_load_checks_test_checks(capsys: pytest.CaptureFixture[str]) -> None: loaded_checks = doctor.load_checks( - Repository(os.path.join(os.path.dirname(__file__))), set() + Repository(os.path.join(os.path.dirname(__file__))), set(), set() ) loaded_check_names = [check.name for check in loaded_checks] assert len(loaded_check_names) == 5 @@ -37,7 +40,7 @@ def test_load_checks_test_checks(capsys: pytest.CaptureFixture[str]) -> None: def test_load_checks_only_passing_tag() -> None: loaded_checks = doctor.load_checks( - Repository(os.path.join(os.path.dirname(__file__))), {"pass"} + Repository(os.path.join(os.path.dirname(__file__))), {"pass"}, set() ) loaded_check_names = [check.name for check in loaded_checks] assert len(loaded_check_names) == 1 @@ -46,7 +49,7 @@ def test_load_checks_only_passing_tag() -> None: def test_load_checks_only_failing_tag() -> None: loaded_checks = doctor.load_checks( - Repository(os.path.join(os.path.dirname(__file__))), {"fail"} + Repository(os.path.join(os.path.dirname(__file__))), {"fail"}, set() ) loaded_check_names = [check.name for check in loaded_checks] assert len(loaded_check_names) == 2 @@ -56,7 +59,9 @@ def test_load_checks_only_failing_tag() -> None: def test_load_checks_passing_and_failing_tag() -> None: loaded_checks = doctor.load_checks( - Repository(os.path.join(os.path.dirname(__file__))), {"pass", "fail"} + Repository(os.path.join(os.path.dirname(__file__))), + {"pass", "fail"}, + set(), ) loaded_check_names = [check.name for check in loaded_checks] assert len(loaded_check_names) == 0 @@ -64,7 +69,7 @@ def test_load_checks_passing_and_failing_tag() -> None: def test_load_checks_test_tag() -> None: loaded_checks = doctor.load_checks( - Repository(os.path.join(os.path.dirname(__file__))), {"test"} + Repository(os.path.join(os.path.dirname(__file__))), {"test"}, set() ) loaded_check_names = [check.name for check in loaded_checks] assert len(loaded_check_names) == 5 @@ -73,3 +78,25 @@ def test_load_checks_test_tag() -> None: assert "failing check with msg" in loaded_check_names assert "broken check" in loaded_check_names assert "broken fix" in loaded_check_names + + +def test_load_checks_exclude_passing_tag() -> None: + loaded_checks = doctor.load_checks( + Repository(os.path.join(os.path.dirname(__file__))), {"test"}, {"pass"} + ) + loaded_check_names = [check.name for check in loaded_checks] + assert len(loaded_check_names) == 4 + assert "passing check" not in loaded_check_names + assert "failing check" in loaded_check_names + assert "failing check with msg" in loaded_check_names + + +def test_load_checks_exclude_failing_tag() -> None: + loaded_checks = doctor.load_checks( + Repository(os.path.join(os.path.dirname(__file__))), {"test"}, {"fail"} + ) + loaded_check_names = [check.name for check in loaded_checks] + assert len(loaded_check_names) == 3 + assert "passing check" in loaded_check_names + assert "failing check" not in loaded_check_names + assert "failing check with msg" not in loaded_check_names