diff --git a/setup/ubuntu/packages.json b/setup/ubuntu/packages.json index 88a32450142d..fedc00472e7c 100644 --- a/setup/ubuntu/packages.json +++ b/setup/ubuntu/packages.json @@ -3,8 +3,13 @@ "type": "download_deb", "name": "bazelisk", "version": "1.29.0", - "arches": ["amd64"], - "codenames": ["noble", "resolute"], + "arches": [ + "amd64" + ], + "codenames": [ + "noble", + "resolute" + ], "sha256": "186d78a20e1a64f59ba08987791a989892d142c9d3a9f9cc0c5c35e201f53924", "urls": [ "https://github.com/bazelbuild/bazelisk/releases/download/v1.29.0/bazelisk-amd64.deb" @@ -14,8 +19,13 @@ "type": "download_deb", "name": "bazelisk", "version": "1.29.0", - "arches": ["arm64"], - "codenames": ["noble", "resolute"], + "arches": [ + "arm64" + ], + "codenames": [ + "noble", + "resolute" + ], "sha256": "db8ada89c841afd2cb33db7d13aa98ea4fb14612579a8bae84722250caa84272", "urls": [ "https://github.com/bazelbuild/bazelisk/releases/download/v1.29.0/bazelisk-arm64.deb" @@ -25,12 +35,16 @@ "type": "download_deb", "name": "kcov", "version": "43+dfsg-1", - "arches": ["amd64"], - "codenames": ["noble"], + "arches": [ + "amd64" + ], + "codenames": [ + "noble" + ], "sha256": "d192fd3cfd0d63e95f13a1b2120d0603a31b3a034b82c618d5e18205517d5cbb", "urls": [ "https://drake-mirror.csail.mit.edu/ubuntu/pool/universe/k/kcov/kcov_43%2Bdfsg-1_amd64.deb" ], "note": "Because Noble does not offer kcov natively, this file was mirrored from Ubuntu 25.04 Plucky at https://packages.ubuntu.com/plucky/kcov." } -] +] \ No newline at end of file diff --git a/tools/workspace/bazelisk_internal/BUILD.bazel b/tools/workspace/bazelisk_internal/BUILD.bazel index c127006b88d7..afcc9fdd066c 100644 --- a/tools/workspace/bazelisk_internal/BUILD.bazel +++ b/tools/workspace/bazelisk_internal/BUILD.bazel @@ -1,5 +1,19 @@ load("//tools/lint:lint.bzl", "add_lint_tests") -load("//tools/skylark:drake_py.bzl", "drake_py_unittest") +load("//tools/skylark:drake_py.bzl", "drake_py_binary", "drake_py_unittest") + +drake_py_binary( + name = "upgrade", + srcs = ["upgrade.py"], + data = [ + "@bazelisk_internal//:LICENSE", + "@bazelisk_internal//:bazelisk.py", + ], + env = { + "DRAKE_BAZELISK_LICENSE_PATH": "$(rlocationpath @bazelisk_internal//:LICENSE)", + "DRAKE_BAZELISK_PY_PATH": "$(rlocationpath @bazelisk_internal//:bazelisk.py)", + }, + deps = ["@rules_python//python/runfiles"], +) drake_py_unittest( name = "lint_test", diff --git a/tools/workspace/bazelisk_internal/repository.bzl b/tools/workspace/bazelisk_internal/repository.bzl index fc89e0915cdc..577bf46b02cf 100644 --- a/tools/workspace/bazelisk_internal/repository.bzl +++ b/tools/workspace/bazelisk_internal/repository.bzl @@ -7,25 +7,17 @@ def bazelisk_internal_repository( name = name, repository = "bazelbuild/bazelisk", upgrade_advice = """ - When updating, the following additional steps (run in the Drake source - tree) must also be performed: - - $ bazel build @bazelisk_internal//:* - $ cp -t third_party/com_github_bazelbuild_bazelisk/ \\ - bazel-drake/external/+internal_repositories+bazelisk_internal/LICENSE \\ - bazel-drake/external/+internal_repositories+bazelisk_internal/bazelisk.py - - Additionally, you must manually update the version numbers in - setup/ubuntu/packages.json - and adjust the expected checksums accordingly. - To calculate a new checksum, download the deb file specifed in the json - and use: - shasum -a 256 'xxx.deb' - - To fully test, a Linux uprovisioned job must be launched from the - pull request. + When upgrading, most Linux uprovisioned jobs should be launched from + the pull request. See the jenkins-jobs-experimental branch on + RobotLocomotion/drake for job lists. """, # noqa upgrade_type = "release", + post_upgrade_script = "upgrade.py", + # Our upgrade.py modifies things outside of the current directory. + extra_upgrade_paths = [ + "setup/ubuntu", + "third_party/com_github_bazelbuild_bazelisk", + ], commit = "v1.29.0", sha256 = "7e4c7b8ade016052e63c1553cb4fbe0c4fe921e1e66913d49eef074ed894e933", # noqa build_file = ":package.BUILD.bazel", diff --git a/tools/workspace/bazelisk_internal/upgrade.py b/tools/workspace/bazelisk_internal/upgrade.py new file mode 100755 index 000000000000..d6d1d6aeb1b2 --- /dev/null +++ b/tools/workspace/bazelisk_internal/upgrade.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +""" +upgrade.py - Upgrades Drake's version of bazelisk. + +This program is only tested / supported on Ubuntu. +""" + +# noqa: shebang +# We suppress shebang lint checking because we have a magic trampoline atop our +# main function that allows us to re-execute ourselves using Bazel. + +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import tempfile +from urllib.parse import urlparse +import urllib.request + + +def _get_url_sha256(url: str) -> str: + hasher = hashlib.sha256() + with tempfile.TemporaryDirectory("drake-bazelisk") as tmp: + tmp_file = Path(tmp) / os.path.basename(urlparse(url).path) + with ( + tmp_file.open("wb") as f, + urllib.request.urlopen(url=url, timeout=30) as response, + ): + while True: + data = response.read(4096) + if not data: + break + hasher.update(data) + f.write(data) + return hasher.hexdigest() + + +def main(): + bazelisk_license_path = os.environ.get("DRAKE_BAZELISK_LICENSE_PATH") + if bazelisk_license_path is None: + # Operate relative to the root of the Drake source tree. + os.chdir(Path(__file__).resolve().parents[3]) + os.execvp( + "bazel", + ["bazel", "run", "//tools/workspace/bazelisk_internal:upgrade"], + ) + + # This import only works when run via Bazel, so must come after the re-exec. + from python import runfiles + + manifest = runfiles.Create() + + mydir = ( + Path(os.environ["BUILD_WORKSPACE_DIRECTORY"]) + / "tools/workspace/bazelisk_internal" + ) + + # Find out which version we are pinned to (new_release has already upgraded + # our repository.bzl). + new_version = None + my_version_re = re.compile(r'commit\s*=\s*["\']([^"\']+)["\']') + repo_bzl_lines = ( + (mydir / "repository.bzl").read_text(encoding="utf-8").splitlines() + ) + for line in repo_bzl_lines: + m = my_version_re.search(line) + if m: + new_version = m.group(1) + break + assert new_version + + # Upgrade setup/ubuntu/packages.json. + setup_dir = ( + Path(os.environ["BUILD_WORKSPACE_DIRECTORY"]) / "setup" / "ubuntu" + ) + packages = json.loads( + (setup_dir / "packages.json").read_text(encoding="utf-8") + ) + for package in packages: + if package["name"] != "bazelisk": + continue + for i, url in enumerate(package["urls"]): + package["urls"][i] = re.sub(r"v\d+\.\d+\.\d+", new_version, url) + package["sha256"] = _get_url_sha256(package["urls"][0]) + (setup_dir / "packages.json").write_text(json.dumps(packages, indent=4)) + + # Upgrade our third_party copy. + bazelisk_py_path = os.environ.get("DRAKE_BAZELISK_PY_PATH") + bazelisk_files = { + manifest.Rlocation(bazelisk_license_path), + manifest.Rlocation(bazelisk_py_path), + } + third_party_dir = ( + Path(os.environ["BUILD_WORKSPACE_DIRECTORY"]) + / "third_party" + / "com_github_bazelbuild_bazelisk" + ) + for file in bazelisk_files: + shutil.copy2(file, third_party_dir) + + +if __name__ == "__main__": + main() diff --git a/tools/workspace/github.bzl b/tools/workspace/github.bzl index 480d50455c17..a3214a6a416a 100644 --- a/tools/workspace/github.bzl +++ b/tools/workspace/github.bzl @@ -22,6 +22,8 @@ def github_archive( local_repository_override = None, mirrors = None, upgrade_advice = "", + post_upgrade_script = "", + extra_upgrade_paths = None, **kwargs): """A macro to be called in the MODULE.bazel that adds an external from GitHub using a workspace rule. @@ -82,6 +84,13 @@ def github_archive( upgrade_advice: optional string that describes extra steps that should be taken when upgrading to a new version. Used by //tools/workspace:new_release. + post_upgrade_script: optional string describing a path to an upgrade + script to be run after the automated upgrade, relative to the + package. + Used by //tools/workspace:new_release. + extra_upgrade_paths: optional list of directories that are expected + to (at least potentially) be changed when upgrading. + Used by //tools/workspace:new_release. """ if repository == None: fail("Missing repository=") @@ -131,6 +140,8 @@ def github_archive( extra_strip_prefix = extra_strip_prefix, mirrors = mirrors, upgrade_advice = upgrade_advice, + post_upgrade_script = post_upgrade_script, + extra_upgrade_paths = extra_upgrade_paths, **kwargs ) @@ -204,6 +215,12 @@ _github_archive_real = repository_rule( "upgrade_advice": attr.string( default = "", ), + "post_upgrade_script": attr.string( + default = "", + ), + "extra_upgrade_paths": attr.string_list( + default = [], + ), }, ) """This is a rule() formulation of the github_archive() macro. It is identical @@ -236,6 +253,8 @@ def setup_github_repository(repository_ctx): sha256 = repository_ctx.attr.sha256, extra_strip_prefix = repository_ctx.attr.extra_strip_prefix, upgrade_advice = getattr(repository_ctx.attr, "upgrade_advice", ""), + post_upgrade_script = getattr(repository_ctx.attr, "post_upgrade_script", ""), + extra_upgrade_paths = getattr(repository_ctx.attr, "extra_upgrade_paths", None), ) # Optionally apply source patches, using Bazel's utility helper. Here we @@ -267,6 +286,8 @@ def github_download_and_extract( sha256 = "0" * 64, extra_strip_prefix = "", upgrade_advice = "", + post_upgrade_script = "", + extra_upgrade_paths = None, upgrade_cooldown_days = None, commit_pin = None): """Download an archive of the provided GitHub repository and commit to the @@ -304,6 +325,13 @@ def github_download_and_extract( upgrade_advice: optional string that describes extra steps that should be taken when upgrading to a new version. Used by //tools/workspace:new_release. + post_upgrade_script: optional string describing a path to an upgrade + script to be run after the automated upgrade, relative to the + package. + Used by //tools/workspace:new_release. + extra_upgrade_paths: optional list of directories that are expected + to (at least potentially) be changed when upgrading. + Used by //tools/workspace:new_release. """ urls = _urls( repository = repository, @@ -338,10 +366,12 @@ def github_download_and_extract( if upgrade_type not in ["release", "tag"] and exclude_tags_pattern: fail("exclude_tags_pattern can only be used for releases or tags") + repository_rule_type = "github_script" if post_upgrade_script != "" else "github" + # Create a summary file for Drake maintainers. generate_repository_metadata( repository_ctx, - repository_rule_type = "github", + repository_rule_type = repository_rule_type, repository = repository, upgrade_type = upgrade_type, upgrade_cooldown_days = upgrade_cooldown_days, @@ -353,6 +383,8 @@ def github_download_and_extract( urls = urls, strip_prefix = strip_prefix, upgrade_advice = upgrade_advice, + post_upgrade_script = post_upgrade_script, + extra_upgrade_paths = extra_upgrade_paths, ) def _sha256(sha256): diff --git a/tools/workspace/new_release.py b/tools/workspace/new_release.py index 758dce689dbc..1af7234e1f9a 100644 --- a/tools/workspace/new_release.py +++ b/tools/workspace/new_release.py @@ -113,10 +113,16 @@ class RuleType(Enum): GITHUB_RELEASE_ATTACHMENTS = "github_release_attachments" # Repository rule that uses an external upgrade script. SCRIPTED = "scripted" + # Repository rule that does both of the above. + GITHUB_WITH_SCRIPT = "github_script" @property def is_github(self) -> bool: - return self in {RuleType.GITHUB, RuleType.GITHUB_RELEASE_ATTACHMENTS} + return self in { + RuleType.GITHUB, + RuleType.GITHUB_RELEASE_ATTACHMENTS, + RuleType.GITHUB_WITH_SCRIPT, + } class UpgradeType(Enum): @@ -522,7 +528,11 @@ def _do_upgrade_github_release_attachments( def _do_upgrade_scripted( - *, local_drake_checkout: git.Repo, workspace_root: str, script: str + *, + local_drake_checkout: git.Repo, + workspace_root: str, + script: str, + extra_modified_paths: list[str] | None = None, ) -> set[str]: """Performs a scripted upgrade and returns the set of files modified.""" # Run the upgrade script. @@ -530,7 +540,11 @@ def _do_upgrade_scripted( subprocess.check_call([os.path.join(repo_root, workspace_root, script)]) # Look for modified paths. - return _modified_paths(local_drake_checkout, workspace_root) + modified_paths = _modified_paths(local_drake_checkout, workspace_root) + if extra_modified_paths: + for path in extra_modified_paths: + modified_paths.update(_modified_paths(local_drake_checkout, path)) + return modified_paths def _do_upgrade( @@ -548,7 +562,8 @@ def _do_upgrade( data = metadata[workspace_name] rule_type = RuleType(data["repository_rule_type"]) - bzl_filename = f"tools/workspace/{workspace_name}/repository.bzl" + workspace_root = f"tools/workspace/{workspace_name}/" + bzl_filename = f"{workspace_root}repository.bzl" if workspace_name in _OTHER_REPOSITORIES + _CHECK_ONLY_REPOSITORIES: upgrade_advice = data.get("upgrade_advice", "") @@ -566,7 +581,6 @@ def _do_upgrade( if rule_type == RuleType.SCRIPTED: # Determine if we should and can commit the changes made. - workspace_root = f"tools/workspace/{workspace_name}/" can_commit = _is_unmodified(local_drake_checkout, workspace_root) if commit and not can_commit: warn(f"{workspace_root} has local changes.") @@ -578,6 +592,7 @@ def _do_upgrade( local_drake_checkout=local_drake_checkout, workspace_root=workspace_root, script=data["upgrade_script"], + extra_modified_paths=data["extra_upgrade_paths"] or None, ) if not len(modified_paths): return UpgradeResult(False) @@ -602,8 +617,9 @@ def _do_upgrade( warn(f"Changes made for {workspace_name} will NOT be committed.") # Do the upgrade. + modified_paths = set() upgrade_type = UpgradeType(data["upgrade_type"]) - if rule_type == RuleType.GITHUB: + if rule_type in {RuleType.GITHUB, RuleType.GITHUB_WITH_SCRIPT}: _do_upgrade_github_archive( temp_dir=temp_dir, upgrade_type=upgrade_type, @@ -612,8 +628,9 @@ def _do_upgrade( bzl_filename=bzl_filename, repository=data["repository"], ) - else: - assert rule_type == RuleType.GITHUB_RELEASE_ATTACHMENTS + modified_paths.add(bzl_filename) + + if rule_type == RuleType.GITHUB_RELEASE_ATTACHMENTS: _do_upgrade_github_release_attachments( temp_dir=temp_dir, old_commit=old_commit, @@ -622,9 +639,19 @@ def _do_upgrade( repository=data["repository"], old_attachments=data["attachments"], ) + modified_paths.add(bzl_filename) + + if rule_type == RuleType.GITHUB_WITH_SCRIPT: + modified_paths.update( + _do_upgrade_scripted( + local_drake_checkout=local_drake_checkout, + workspace_root=workspace_root, + script=data["post_upgrade_script"], + extra_modified_paths=data["extra_upgrade_paths"] or None, + ) + ) # Finalize the result field(s). - modified_paths = {bzl_filename} if upgrade_type == UpgradeType.COMMIT: message = f"Update dependency {workspace_name} to latest commit" else: