diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 03727776bd4a9a..99aae4164eeabd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -214,14 +214,14 @@ jobs: name: diff_report_${{ github.event.number }} path: selfdrive/test/process_replay/diff_report.txt - name: Checkout ci-artifacts - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 with: - repository: commaai/ci-artifacts + repository: elkoled/ci-artifacts ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} path: ${{ github.workspace }}/ci-artifacts - name: Push refs - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' working-directory: ${{ github.workspace }}/ci-artifacts run: | git checkout --orphan process-replay diff --git a/pyproject.toml b/pyproject.toml index 8b0f4004d68e34..b1295ef81379ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,8 +97,6 @@ testing = [ dev = [ "av", - "azure-identity", - "azure-storage-blob", "dictdiffer", "matplotlib", "opencv-python-headless", diff --git a/selfdrive/test/process_replay/regen.py b/selfdrive/test/process_replay/regen.py index c501a4b25083c1..6e17217c3b7a03 100755 --- a/selfdrive/test/process_replay/regen.py +++ b/selfdrive/test/process_replay/regen.py @@ -9,7 +9,6 @@ from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, ProcessConfig, replay_process, get_process_config, \ check_openpilot_enabled, check_most_messages_valid, get_custom_params_from_lr -from openpilot.selfdrive.test.update_ci_routes import upload_route from openpilot.tools.lib.framereader import FrameReader from openpilot.tools.lib.logreader import LogReader, LogIterable, save_log from openpilot.tools.lib.openpilotci import get_url @@ -52,7 +51,7 @@ def setup_data_readers( def regen_and_save( route: str, sidx: int, processes: str | Iterable[str] = "all", outdir: str = FAKEDATA, - upload: bool = False, disable_tqdm: bool = False, dummy_driver_cam: bool = False + disable_tqdm: bool = False, dummy_driver_cam: bool = False ) -> str: if not isinstance(processes, str) and not hasattr(processes, "__iter__"): raise ValueError("whitelist_proc must be a string or iterable") @@ -90,9 +89,6 @@ def regen_and_save( if not check_most_messages_valid(output_logs): raise Exception("Route has too many invalid messages") - if upload: - upload_route(rel_log_dir) - return rel_log_dir @@ -102,7 +98,6 @@ def comma_separated_list(string): all_procs = [p.proc_name for p in CONFIGS] parser = argparse.ArgumentParser(description="Generate new segments from old ones") - parser.add_argument("--upload", action="store_true", help="Upload the new segment to the CI bucket") parser.add_argument("--outdir", help="log output dir", default=FAKEDATA) parser.add_argument("--dummy-dcamera", action='store_true', help="Use dummy blank driver camera") parser.add_argument("--whitelist-procs", type=comma_separated_list, default=all_procs, @@ -115,4 +110,4 @@ def comma_separated_list(string): blacklist_set = set(args.blacklist_procs) processes = [p for p in args.whitelist_procs if p not in blacklist_set] - regen_and_save(args.route, args.seg, processes=processes, upload=args.upload, outdir=args.outdir, dummy_driver_cam=args.dummy_dcamera) + regen_and_save(args.route, args.seg, processes=processes, outdir=args.outdir, dummy_driver_cam=args.dummy_dcamera) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 7c3318b6a11a44..54af0239610000 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -67,7 +67,7 @@ # dashcamOnly makes don't need to be tested until a full port is done excluded_interfaces = ["mock", "body", "psa"] -BASE_URL = "https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/process-replay/" +BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"} @@ -188,10 +188,6 @@ def test_process(cfg, lr, segment, ref_log_msgs, new_log_path, ignore_fields=Non if cfg.proc_name not in tested_procs: continue - # to speed things up, we only test all segments on card - if cfg.proc_name not in ('card', 'controlsd', 'lagd') and car_brand not in ('HYUNDAI', 'TOYOTA'): - continue - cur_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{cur_commit}.zst".replace("|", "_")) if args.update_refs: # reference logs will not exist if routes were just regenerated ref_log_path = get_url(*segment.rsplit("--", 1,), "rlog.zst") diff --git a/selfdrive/test/update_ci_routes.py b/selfdrive/test/update_ci_routes.py deleted file mode 100755 index 54e1c88718f9f8..00000000000000 --- a/selfdrive/test/update_ci_routes.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -import os -import re -import subprocess -import sys -from collections.abc import Iterable - -from tqdm import tqdm - -from opendbc.car.tests.routes import routes as test_car_models_routes -from openpilot.selfdrive.test.process_replay.test_processes import source_segments as replay_segments -from openpilot.tools.lib.azure_container import AzureContainer -from openpilot.tools.lib.openpilotcontainers import DataCIContainer, DataProdContainer, OpenpilotCIContainer - -SOURCES: list[AzureContainer] = [ - DataProdContainer, - DataCIContainer -] - -DEST = OpenpilotCIContainer - -def upload_route(path: str, exclude_patterns: Iterable[str] | None = None) -> None: - if exclude_patterns is None: - exclude_patterns = [r'dcamera\.hevc'] - - r, n = path.rsplit("--", 1) - r = '/'.join(r.split('/')[-2:]) # strip out anything extra in the path - destpath = f"{r}/{n}" - for file in os.listdir(path): - if any(re.search(pattern, file) for pattern in exclude_patterns): - continue - DEST.upload_file(os.path.join(path, file), f"{destpath}/{file}") - - -def sync_to_ci_public(route: str) -> bool: - dest_container, dest_key = DEST.get_client_and_key() - key_prefix = route.replace('|', '/') - dongle_id = key_prefix.split('/')[0] - - if next(dest_container.list_blob_names(name_starts_with=key_prefix), None) is not None: - return True - - print(f"Uploading {route}") - for source_container in SOURCES: - # assumes az login has been run - print(f"Trying {source_container.ACCOUNT}/{source_container.CONTAINER}") - _, source_key = source_container.get_client_and_key() - cmd = [ - "azcopy", - "copy", - f"{source_container.BASE_URL}{key_prefix}?{source_key}", - f"{DEST.BASE_URL}{dongle_id}?{dest_key}", - "--recursive=true", - "--overwrite=false", - "--exclude-pattern=*/dcamera.hevc", - ] - - try: - result = subprocess.call(cmd, stdout=subprocess.DEVNULL) - if result == 0: - print("Success") - return True - except subprocess.CalledProcessError: - print("Failed") - - return False - - -if __name__ == "__main__": - failed_routes = [] - - to_sync = sys.argv[1:] - - if not len(to_sync): - # sync routes from the car tests routes and process replay - to_sync.extend([rt.route for rt in test_car_models_routes]) - to_sync.extend([s[1].rsplit('--', 1)[0] for s in replay_segments]) - - for r in tqdm(to_sync): - if not sync_to_ci_public(r): - failed_routes.append(r) - - if len(failed_routes): - print("failed routes:", failed_routes) diff --git a/tools/lib/azure_container.py b/tools/lib/azure_container.py deleted file mode 100644 index 7ba2e1515fca7c..00000000000000 --- a/tools/lib/azure_container.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -from datetime import datetime, timedelta, UTC -from functools import lru_cache -from pathlib import Path -from typing import IO - - -TOKEN_PATH = Path("/data/azure_token") - -@lru_cache -def get_azure_credential(): - if "AZURE_TOKEN" in os.environ: - return os.environ["AZURE_TOKEN"] - elif TOKEN_PATH.is_file(): - return TOKEN_PATH.read_text().strip() - else: - from azure.identity import AzureCliCredential - return AzureCliCredential() - -@lru_cache -def get_container_sas(account_name: str, container_name: str): - from azure.storage.blob import BlobServiceClient, ContainerSasPermissions, generate_container_sas - start_time = datetime.now(UTC).replace(tzinfo=None) - expiry_time = start_time + timedelta(hours=1) - blob_service = BlobServiceClient( - account_url=f"https://{account_name}.blob.core.windows.net", - credential=get_azure_credential(), - ) - return generate_container_sas( - account_name, - container_name, - user_delegation_key=blob_service.get_user_delegation_key(start_time, expiry_time), - permission=ContainerSasPermissions(read=True, write=True, list=True), - expiry=expiry_time, - ) - -class AzureContainer: - def __init__(self, account, container): - self.ACCOUNT = account - self.CONTAINER = container - - @property - def ACCOUNT_URL(self) -> str: - return f"https://{self.ACCOUNT}.blob.core.windows.net" - - @property - def BASE_URL(self) -> str: - return f"{self.ACCOUNT_URL}/{self.CONTAINER}/" - - def get_client_and_key(self): - from azure.storage.blob import ContainerClient - client = ContainerClient(self.ACCOUNT_URL, self.CONTAINER, credential=get_azure_credential()) - key = get_container_sas(self.ACCOUNT, self.CONTAINER) - return client, key - - def get_url(self, route_name: str, segment_num: str, filename: str) -> str: - return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{filename}" - - def upload_bytes(self, data: bytes | IO, blob_name: str, overwrite=False) -> str: - from azure.storage.blob import BlobClient - blob = BlobClient( - account_url=self.ACCOUNT_URL, - container_name=self.CONTAINER, - blob_name=blob_name, - credential=get_azure_credential(), - overwrite=overwrite, - ) - blob.upload_blob(data, overwrite=overwrite) - return self.BASE_URL + blob_name - - def upload_file(self, path: str | os.PathLike, blob_name: str, overwrite=False) -> str: - with open(path, "rb") as f: - return self.upload_bytes(f, blob_name, overwrite) diff --git a/tools/lib/openpilotci.py b/tools/lib/openpilotci.py index 1c1e1f171b7858..6b484dfdadab0a 100644 --- a/tools/lib/openpilotci.py +++ b/tools/lib/openpilotci.py @@ -1,12 +1,4 @@ -from openpilot.tools.lib.openpilotcontainers import OpenpilotCIContainer +BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" -def get_url(*args, **kwargs): - return OpenpilotCIContainer.get_url(*args, **kwargs) - -def upload_file(*args, **kwargs): - return OpenpilotCIContainer.upload_file(*args, **kwargs) - -def upload_bytes(*args, **kwargs): - return OpenpilotCIContainer.upload_bytes(*args, **kwargs) - -BASE_URL = OpenpilotCIContainer.BASE_URL +def get_url(route_name: str, segment_num, filename: str) -> str: + return BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{filename}" diff --git a/tools/lib/openpilotcontainers.py b/tools/lib/openpilotcontainers.py deleted file mode 100755 index 988246111189c4..00000000000000 --- a/tools/lib/openpilotcontainers.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python3 -from openpilot.tools.lib.azure_container import AzureContainer - -OpenpilotCIContainer = AzureContainer("commadataci", "openpilotci") -DataCIContainer = AzureContainer("commadataci", "commadataci") -DataProdContainer = AzureContainer("commadata2", "commadata2") diff --git a/uv.lock b/uv.lock index 81e2d91360276c..6975baff25f218 100644 --- a/uv.lock +++ b/uv.lock @@ -119,50 +119,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/ff/48fa68888b8d5bae36d915556ff18f9e5fdc6b5ff5ae23dc4904c9713168/av-13.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:5ea0deab0e6a739cb742fba2a3983d8102f7516a3cdf3c46669f3cac0ed1f351", size = 25781343, upload-time = "2024-10-06T04:53:29.577Z" }, ] -[[package]] -name = "azure-core" -version = "1.38.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/1b/e503e08e755ea94e7d3419c9242315f888fc664211c90d032e40479022bf/azure_core-1.38.0.tar.gz", hash = "sha256:8194d2682245a3e4e3151a667c686464c3786fed7918b394d035bdcd61bb5993", size = 363033, upload-time = "2026-01-12T17:03:05.535Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/d8/b8fcba9464f02b121f39de2db2bf57f0b216fe11d014513d666e8634380d/azure_core-1.38.0-py3-none-any.whl", hash = "sha256:ab0c9b2cd71fecb1842d52c965c95285d3cfb38902f6766e4a471f1cd8905335", size = 217825, upload-time = "2026-01-12T17:03:07.291Z" }, -] - -[[package]] -name = "azure-identity" -version = "1.25.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "msal" }, - { name = "msal-extensions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/8d/1a6c41c28a37eab26dc85ab6c86992c700cd3f4a597d9ed174b0e9c69489/azure_identity-1.25.1.tar.gz", hash = "sha256:87ca8328883de6036443e1c37b40e8dc8fb74898240f61071e09d2e369361456", size = 279826, upload-time = "2025-10-06T20:30:02.194Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/7b/5652771e24fff12da9dde4c20ecf4682e606b104f26419d139758cc935a6/azure_identity-1.25.1-py3-none-any.whl", hash = "sha256:e9edd720af03dff020223cd269fa3a61e8f345ea75443858273bcb44844ab651", size = 191317, upload-time = "2025-10-06T20:30:04.251Z" }, -] - -[[package]] -name = "azure-storage-blob" -version = "12.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "isodate" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/71/24/072ba8e27b0e2d8fec401e9969b429d4f5fc4c8d4f0f05f4661e11f7234a/azure_storage_blob-12.28.0.tar.gz", hash = "sha256:e7d98ea108258d29aa0efbfd591b2e2075fa1722a2fae8699f0b3c9de11eff41", size = 604225, upload-time = "2026-01-06T23:48:57.282Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/3a/6ef2047a072e54e1142718d433d50e9514c999a58f51abfff7902f3a72f8/azure_storage_blob-12.28.0-py3-none-any.whl", hash = "sha256:00fb1db28bf6a7b7ecaa48e3b1d5c83bfadacc5a678b77826081304bd87d6461", size = 431499, upload-time = "2026-01-06T23:48:58.995Z" }, -] - [[package]] name = "casadi" version = "3.7.2" @@ -595,15 +551,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/94/040a0d9c81f018c39bd887b7b825013b024deb0a6c795f9524797e2cd41b/inputs-0.5-py2.py3-none-any.whl", hash = "sha256:13f894564e52134cf1e3862b1811da034875eb1f2b62e6021e3776e9669a96ec", size = 33630, upload-time = "2018-10-05T22:38:28.28Z" }, ] -[[package]] -name = "isodate" -version = "0.7.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, -] - [[package]] name = "jeepney" version = "0.9.0" @@ -903,32 +850,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, ] -[[package]] -name = "msal" -version = "1.34.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cf/0e/c857c46d653e104019a84f22d4494f2119b4fe9f896c92b4b864b3b045cc/msal-1.34.0.tar.gz", hash = "sha256:76ba83b716ea5a6d75b0279c0ac353a0e05b820ca1f6682c0eb7f45190c43c2f", size = 153961, upload-time = "2025-09-22T23:05:48.989Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/dc/18d48843499e278538890dc709e9ee3dea8375f8be8e82682851df1b48b5/msal-1.34.0-py3-none-any.whl", hash = "sha256:f669b1644e4950115da7a176441b0e13ec2975c29528d8b9e81316023676d6e1", size = 116987, upload-time = "2025-09-22T23:05:47.294Z" }, -] - -[[package]] -name = "msal-extensions" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "msal" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, -] - [[package]] name = "multidict" version = "6.7.1" @@ -1058,8 +979,6 @@ dependencies = [ [package.optional-dependencies] dev = [ { name = "av" }, - { name = "azure-identity" }, - { name = "azure-storage-blob" }, { name = "dictdiffer" }, { name = "matplotlib" }, { name = "opencv-python-headless" }, @@ -1097,8 +1016,6 @@ requires-dist = [ { name = "aiohttp" }, { name = "aiortc" }, { name = "av", marker = "extra == 'dev'" }, - { name = "azure-identity", marker = "extra == 'dev'" }, - { name = "azure-storage-blob", marker = "extra == 'dev'" }, { name = "casadi", specifier = ">=3.6.6" }, { name = "cffi" }, { name = "codespell", marker = "extra == 'testing'" }, @@ -1448,11 +1365,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997, upload-time = "2024-11-28T03:43:27.893Z" }, ] -[package.optional-dependencies] -crypto = [ - { name = "cryptography" }, -] - [[package]] name = "pylibsrtp" version = "1.0.0"