E2E tests for ONNX and RVC4 NNArchive metric consistency with LuxonisTrain

.github/workflows/onnx-nnarchive-regression.yml

@@ -0,0 +1,52 @@
+name: ONNX NNArchive Regression
+
+on:
+  push:
+    branches:
+      - main
+      - "develop*"
+      - "develop/*"
+      - "develop/**"
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  onnx-nnarchive-regression:
+    if: |
+      (github.event_name == 'push' &&
+        (github.ref_name == 'main' || startsWith(github.ref_name, 'develop'))) ||
+      (github.event_name == 'pull_request' &&
+        (github.head_ref == 'main' || startsWith(github.head_ref, 'develop')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_CREDENTIALS }}
+          create_credentials_file: true
+          export_environment_variables: true
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install gcsfs
+          python -m pip install -e ".[dev]"
+
+      - name: Run ONNX NNArchive regression tests
+        env:
+          LUXONIS_TELEMETRY_ENABLED: "false"
+        run: |
+          pytest tests/test_onnx_nnarchive_regression.py -q

.github/workflows/rvc4-nnarchive-regression.yml

@@ -0,0 +1,127 @@
+name: RVC4 NNArchive Regression
+
+on:
+  workflow_dispatch:
+    inputs:
+      reservation_name:
+        description: "Reservation name"
+        required: false
+        type: string
+      os_version:
+        description: "Optional camera OS version to install before the run"
+        required: false
+        type: string
+      hold_reservation:
+        description: "Hold testbed reservation"
+        required: false
+        type: boolean
+      depthai_version:
+        description: "DepthAI version to test against"
+        required: false
+        default: "3.8.0"
+        type: string
+      additional_options:
+        description: "Additional hil_runner options"
+        required: false
+        type: string
+  push:
+    branches:
+      - main
+      - "develop*"
+      - "develop/*"
+      - "develop/**"
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  rvc4-nnarchive-regression:
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'push' &&
+        (github.ref_name == 'main' || startsWith(github.ref_name, 'develop'))) ||
+      (github.event_name == 'pull_request' &&
+        (github.head_ref == 'main' || startsWith(github.head_ref, 'develop')))
+    runs-on: ["self-hosted", "testbed-runner"]
+    permissions:
+      contents: read
+
+    env:
+      LUXONIS_TELEMETRY_ENABLED: "false"
+      HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Write Google credentials
+        shell: bash
+        run: |
+          mkdir -p .ci
+          cat <<'EOF' > .ci/gcp-credentials.json
+          ${{ secrets.GCP_CREDENTIALS }}
+          EOF
+
+      - name: Install HIL runner
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade \
+            --index-url "https://__token__:${HIL_FRAMEWORK_TOKEN}@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" \
+            hil-framework
+
+      - name: Run RVC4 NNArchive regression tests
+        shell: bash
+        env:
+          DEPTHAI_VERSION_INPUT: ${{ github.event.inputs.depthai_version }}
+          RESERVATION_NAME_INPUT: ${{ github.event.inputs.reservation_name }}
+          HOLD_RESERVATION_INPUT: ${{ github.event.inputs.hold_reservation }}
+          OS_VERSION_INPUT: ${{ github.event.inputs.os_version }}
+          ADDITIONAL_OPTIONS_INPUT: ${{ github.event.inputs.additional_options }}
+        run: |
+          DEPTHAI_VERSION_VALUE="${DEPTHAI_VERSION_INPUT:-3.8.0}"
+          if [[ ! "$DEPTHAI_VERSION_VALUE" =~ ^3\.(8|9|[1-9][0-9])\.[0-9]{1,2}$ ]]; then
+            echo "DEPTHAI_VERSION must be within >=3.8.0,<4.0.0; got: $DEPTHAI_VERSION_VALUE" >&2
+            exit 1
+          fi
+
+          if [[ -n "$RESERVATION_NAME_INPUT" ]]; then
+            RESERVATION_OPTION="--reservation-name $RESERVATION_NAME_INPUT"
+          else
+            RESERVATION_OPTION="--reservation-name https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#luxonis-eval-rvc4"
+          fi
+
+          HOLD_RESERVATION_OPTION=""
+          if [[ "$HOLD_RESERVATION_INPUT" == "true" ]]; then
+            HOLD_RESERVATION_OPTION="--hold-reservation"
+          fi
+
+          OS_VERSION_OPTION=""
+          if [[ -n "$OS_VERSION_INPUT" ]]; then
+            OS_VERSION_OPTION="--os-version $OS_VERSION_INPUT"
+          fi
+
+          ADDITIONAL_OPTIONS_VALUE=""
+          if [[ -n "$ADDITIONAL_OPTIONS_INPUT" ]]; then
+            ADDITIONAL_OPTIONS_VALUE="$ADDITIONAL_OPTIONS_INPUT"
+          fi
+
+          REMOTE_HIL_FRAMEWORK_TOKEN="$(printf '%q' "$HIL_FRAMEWORK_TOKEN")"
+          REMOTE_DEPTHAI_VERSION="$(printf '%q' "$DEPTHAI_VERSION_VALUE")"
+          REMOTE_CMD="cd /tmp/luxonis-eval && \
+            export GOOGLE_APPLICATION_CREDENTIALS=/tmp/luxonis-eval/.ci/gcp-credentials.json && \
+            export LUXONIS_TELEMETRY_ENABLED=false && \
+            export HIL_FRAMEWORK_TOKEN=$REMOTE_HIL_FRAMEWORK_TOKEN && \
+            export DEPTHAI_VERSION=$REMOTE_DEPTHAI_VERSION && \
+            bash ./tests/run_rvc4_nnarchive_hil_tests.sh"
+
+          exec hil_runner \
+            --platforms rvc4 \
+            $HOLD_RESERVATION_OPTION \
+            $ADDITIONAL_OPTIONS_VALUE \
+            --wait \
+            $OS_VERSION_OPTION \
+            $RESERVATION_OPTION \
+            --sync-workspace --rsync-args="--exclude=venv" \
+            --commands "$REMOTE_CMD"

luxonis_eval/core/runtime.py

@@ -154,22 +154,23 @@ def resolve_luxonis_loader_class_mapping(
     ldf_class_map = {
         v: k for k, v in dataset_classes[loader_task_name].items()
     }
+    dataset_identifier = _get_dataset_identifier(dataloader)
 
-    if "imagenet" in dataloader.dataset.dataset_name:
+    if "imagenet" in dataset_identifier:
         native_class_map = get_dataset_class_mapping("imagenet")
-    elif "coco" in dataloader.dataset.dataset_name:
+    elif "coco" in dataset_identifier:
         native_class_map = get_dataset_class_mapping("coco")
     else:
         native_class_map = class_mapping
         if native_class_map:
             logger.info(
-                f"Dataset '{dataloader.dataset.dataset_name}' does not match "
+                f"Dataset '{dataset_identifier}' does not match "
                 "known datasets for automatic class mapping. Using the "
                 "provided 'loader.params.class_mapping' argument."
             )
         else:
             logger.warning(
-                f"Dataset '{dataloader.dataset.dataset_name}' does not match "
+                f"Dataset '{dataset_identifier}' does not match "
                 "known datasets for automatic class mapping and no "
                 "'loader.params.class_mapping' was provided. Falling back to "
                 "the dataset's LDF class order as the native class mapping."
@@ -215,3 +216,23 @@ def get_class_index_mapping(
             raise ValueError(f"Label {v} not found in native class map.")
 
     return ldf_to_native_index_map
+
+
+def _get_dataset_identifier(dataloader: LuxonisLoader) -> str:
+    dataset = dataloader.dataset
+    identifier = getattr(dataset, "identifier", None)
+    if isinstance(identifier, str) and identifier:
+        return identifier
+
+    dataset_name = getattr(dataset, "dataset_name", None)
+    if isinstance(dataset_name, str) and dataset_name:
+        return dataset_name
+
+    private_name = getattr(dataset, "_dataset_name", None)
+    if isinstance(private_name, str) and private_name:
+        return private_name
+
+    raise AttributeError(
+        "Unable to resolve a dataset identifier from the LuxonisLoader "
+        "dataset instance."
+    )

requirements-dev.txt

@@ -1 +1,2 @@
-pre-commit==3.2.1
+pytest>=8.0.0,<9.0.0
+pre-commit>=3.2.1,<4.0.0

sitecustomize.py

@@ -0,0 +1,12 @@
+import os
+import sys
+from pathlib import Path
+
+
+def _is_pytest_process() -> bool:
+    executable_name = Path(sys.argv[0]).name.lower()
+    return "pytest" in executable_name
+
+
+if _is_pytest_process():
+    os.environ.setdefault("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1")

tests/conftest.py

@@ -0,0 +1,111 @@
+import os
+from pathlib import Path
+
+import pytest
+from luxonis_ml.utils import LuxonisFileSystem
+
+os.environ.setdefault("LUXONIS_TELEMETRY_ENABLED", "false")
+
+NORMALIZED_TESTDATA_REMOTE_DIR = (
+    "gs://luxonis-test-bucket/luxonis-eval-test-data/NormalizedTestData"
+)
+LOCAL_TESTDATA_ENV_VAR = "LUXONIS_EVAL_TESTDATA_DIR"
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--device-ip",
+        action="store",
+        default=None,
+        help="RVC4 device IP or MXID. If omitted, uses RVC4_IP or USB.",
+    )
+    parser.addoption(
+        "--testbed-name",
+        action="store",
+        default=None,
+        help="Optional HIL testbed name used to resolve an RVC4 device.",
+    )
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line(
+        "markers", "device: mark test as requiring a physical device"
+    )
+    config.addinivalue_line(
+        "markers", "rvc4: mark test as requiring an RVC4 device"
+    )
+
+
+@pytest.fixture(scope="session")
+def nnarchive_testdata_root(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    configured_dir = os.environ.get(LOCAL_TESTDATA_ENV_VAR)
+    if configured_dir:
+        return Path(configured_dir)
+
+    data_dir = tmp_path_factory.mktemp("nnarchive_testdata")
+    return LuxonisFileSystem.download(
+        NORMALIZED_TESTDATA_REMOTE_DIR,
+        dest=data_dir,
+    )
+
+
+@pytest.fixture(scope="session")
+def rvc4_device_ip(request: pytest.FixtureRequest) -> str | None:
+    configured_device_ip = request.config.getoption("--device-ip")
+    if configured_device_ip:
+        return configured_device_ip
+
+    env_device_ip = os.environ.get("RVC4_IP")
+    if env_device_ip:
+        return env_device_ip
+
+    testbed_name = request.config.getoption("--testbed-name") or os.environ.get(
+        "HIL_TESTBED"
+    )
+    if not testbed_name:
+        return None
+
+    return _resolve_rvc4_device_ip_from_testbed(testbed_name)
+
+
+def _resolve_rvc4_device_ip_from_testbed(testbed_name: str) -> str:
+    try:
+        from hil_framework.lib_testbed.config.Config import Config
+        from hil_framework.lib_testbed.utils.Testbed import Testbed
+    except ImportError as exc:
+        pytest.exit(
+            "hil_framework is required when --testbed-name or HIL_TESTBED is used.",
+            returncode=1,
+        )
+        raise exc
+
+    testbed = Testbed(Config(testbed_name))
+    target_matches = [
+        camera
+        for camera in testbed.cameras
+        if str(getattr(camera, "platform", "")).lower() == "rvc4"
+    ]
+    if len(target_matches) == 1:
+        hostname = getattr(target_matches[0], "hostname", None)
+        if not hostname:
+            pytest.exit(
+                f"Selected RVC4 camera in testbed {testbed_name!r} does not expose a hostname.",
+                returncode=1,
+            )
+        return str(hostname)
+
+    available_cameras = ", ".join(
+        f"{camera.name}:{getattr(camera, 'hostname', None)}:{getattr(camera, 'platform', None)}"
+        for camera in testbed.cameras
+    )
+    if not target_matches:
+        pytest.exit(
+            f"No RVC4 camera found in testbed {testbed_name!r}. Available cameras: {available_cameras}",
+            returncode=1,
+        )
+
+    pytest.exit(
+        "Unable to select a unique RVC4 camera from testbed "
+        f"{testbed_name!r}. Available cameras: {available_cameras}",
+        returncode=1,
+    )