diff --git a/.github/workflows/onnx-nnarchive-regression.yml b/.github/workflows/onnx-nnarchive-regression.yml new file mode 100644 index 0000000..6d5c131 --- /dev/null +++ b/.github/workflows/onnx-nnarchive-regression.yml @@ -0,0 +1,52 @@ +name: ONNX NNArchive Regression + +on: + push: + branches: + - main + - "develop*" + - "develop/*" + - "develop/**" + pull_request: + types: [opened, synchronize, reopened] + +jobs: + onnx-nnarchive-regression: + if: | + (github.event_name == 'push' && + (github.ref_name == 'main' || startsWith(github.ref_name, 'develop'))) || + (github.event_name == 'pull_request' && + (github.head_ref == 'main' || startsWith(github.head_ref, 'develop'))) + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_CREDENTIALS }} + create_credentials_file: true + export_environment_variables: true + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install gcsfs + python -m pip install -e ".[dev]" + + - name: Run ONNX NNArchive regression tests + env: + LUXONIS_TELEMETRY_ENABLED: "false" + run: | + pytest tests/test_onnx_nnarchive_regression.py -q diff --git a/.github/workflows/rvc4-nnarchive-regression.yml b/.github/workflows/rvc4-nnarchive-regression.yml new file mode 100644 index 0000000..9ffdf05 --- /dev/null +++ b/.github/workflows/rvc4-nnarchive-regression.yml @@ -0,0 +1,127 @@ +name: RVC4 NNArchive Regression + +on: + workflow_dispatch: + inputs: + reservation_name: + description: "Reservation name" + required: false + type: string + os_version: + description: "Optional camera OS version to install before the run" + required: false + type: string + hold_reservation: + description: "Hold testbed reservation" + required: false + type: boolean + depthai_version: + description: "DepthAI version to test against" + required: false + default: "3.8.0" + type: string + additional_options: + description: "Additional hil_runner options" + required: false + type: string + push: + branches: + - main + - "develop*" + - "develop/*" + - "develop/**" + pull_request: + types: [opened, synchronize, reopened] + +jobs: + rvc4-nnarchive-regression: + if: | + github.event_name == 'workflow_dispatch' || + (github.event_name == 'push' && + (github.ref_name == 'main' || startsWith(github.ref_name, 'develop'))) || + (github.event_name == 'pull_request' && + (github.head_ref == 'main' || startsWith(github.head_ref, 'develop'))) + runs-on: ["self-hosted", "testbed-runner"] + permissions: + contents: read + + env: + LUXONIS_TELEMETRY_ENABLED: "false" + HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Write Google credentials + shell: bash + run: | + mkdir -p .ci + cat <<'EOF' > .ci/gcp-credentials.json + ${{ secrets.GCP_CREDENTIALS }} + EOF + + - name: Install HIL runner + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade \ + --index-url "https://__token__:${HIL_FRAMEWORK_TOKEN}@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" \ + hil-framework + + - name: Run RVC4 NNArchive regression tests + shell: bash + env: + DEPTHAI_VERSION_INPUT: ${{ github.event.inputs.depthai_version }} + RESERVATION_NAME_INPUT: ${{ github.event.inputs.reservation_name }} + HOLD_RESERVATION_INPUT: ${{ github.event.inputs.hold_reservation }} + OS_VERSION_INPUT: ${{ github.event.inputs.os_version }} + ADDITIONAL_OPTIONS_INPUT: ${{ github.event.inputs.additional_options }} + run: | + DEPTHAI_VERSION_VALUE="${DEPTHAI_VERSION_INPUT:-3.8.0}" + if [[ ! "$DEPTHAI_VERSION_VALUE" =~ ^3\.(8|9|[1-9][0-9])\.[0-9]{1,2}$ ]]; then + echo "DEPTHAI_VERSION must be within >=3.8.0,<4.0.0; got: $DEPTHAI_VERSION_VALUE" >&2 + exit 1 + fi + + if [[ -n "$RESERVATION_NAME_INPUT" ]]; then + RESERVATION_OPTION="--reservation-name $RESERVATION_NAME_INPUT" + else + RESERVATION_OPTION="--reservation-name https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#luxonis-eval-rvc4" + fi + + HOLD_RESERVATION_OPTION="" + if [[ "$HOLD_RESERVATION_INPUT" == "true" ]]; then + HOLD_RESERVATION_OPTION="--hold-reservation" + fi + + OS_VERSION_OPTION="" + if [[ -n "$OS_VERSION_INPUT" ]]; then + OS_VERSION_OPTION="--os-version $OS_VERSION_INPUT" + fi + + ADDITIONAL_OPTIONS_VALUE="" + if [[ -n "$ADDITIONAL_OPTIONS_INPUT" ]]; then + ADDITIONAL_OPTIONS_VALUE="$ADDITIONAL_OPTIONS_INPUT" + fi + + REMOTE_HIL_FRAMEWORK_TOKEN="$(printf '%q' "$HIL_FRAMEWORK_TOKEN")" + REMOTE_DEPTHAI_VERSION="$(printf '%q' "$DEPTHAI_VERSION_VALUE")" + REMOTE_CMD="cd /tmp/luxonis-eval && \ + export GOOGLE_APPLICATION_CREDENTIALS=/tmp/luxonis-eval/.ci/gcp-credentials.json && \ + export LUXONIS_TELEMETRY_ENABLED=false && \ + export HIL_FRAMEWORK_TOKEN=$REMOTE_HIL_FRAMEWORK_TOKEN && \ + export DEPTHAI_VERSION=$REMOTE_DEPTHAI_VERSION && \ + bash ./tests/run_rvc4_nnarchive_hil_tests.sh" + + exec hil_runner \ + --platforms rvc4 \ + $HOLD_RESERVATION_OPTION \ + $ADDITIONAL_OPTIONS_VALUE \ + --wait \ + $OS_VERSION_OPTION \ + $RESERVATION_OPTION \ + --sync-workspace --rsync-args="--exclude=venv" \ + --commands "$REMOTE_CMD" diff --git a/luxonis_eval/core/runtime.py b/luxonis_eval/core/runtime.py index 7b7148d..e27981a 100644 --- a/luxonis_eval/core/runtime.py +++ b/luxonis_eval/core/runtime.py @@ -154,22 +154,23 @@ def resolve_luxonis_loader_class_mapping( ldf_class_map = { v: k for k, v in dataset_classes[loader_task_name].items() } + dataset_identifier = _get_dataset_identifier(dataloader) - if "imagenet" in dataloader.dataset.dataset_name: + if "imagenet" in dataset_identifier: native_class_map = get_dataset_class_mapping("imagenet") - elif "coco" in dataloader.dataset.dataset_name: + elif "coco" in dataset_identifier: native_class_map = get_dataset_class_mapping("coco") else: native_class_map = class_mapping if native_class_map: logger.info( - f"Dataset '{dataloader.dataset.dataset_name}' does not match " + f"Dataset '{dataset_identifier}' does not match " "known datasets for automatic class mapping. Using the " "provided 'loader.params.class_mapping' argument." ) else: logger.warning( - f"Dataset '{dataloader.dataset.dataset_name}' does not match " + f"Dataset '{dataset_identifier}' does not match " "known datasets for automatic class mapping and no " "'loader.params.class_mapping' was provided. Falling back to " "the dataset's LDF class order as the native class mapping." @@ -215,3 +216,23 @@ def get_class_index_mapping( raise ValueError(f"Label {v} not found in native class map.") return ldf_to_native_index_map + + +def _get_dataset_identifier(dataloader: LuxonisLoader) -> str: + dataset = dataloader.dataset + identifier = getattr(dataset, "identifier", None) + if isinstance(identifier, str) and identifier: + return identifier + + dataset_name = getattr(dataset, "dataset_name", None) + if isinstance(dataset_name, str) and dataset_name: + return dataset_name + + private_name = getattr(dataset, "_dataset_name", None) + if isinstance(private_name, str) and private_name: + return private_name + + raise AttributeError( + "Unable to resolve a dataset identifier from the LuxonisLoader " + "dataset instance." + ) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0aabd6c..99d4205 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,2 @@ -pre-commit==3.2.1 +pytest>=8.0.0,<9.0.0 +pre-commit>=3.2.1,<4.0.0 diff --git a/sitecustomize.py b/sitecustomize.py new file mode 100644 index 0000000..8ef2d16 --- /dev/null +++ b/sitecustomize.py @@ -0,0 +1,12 @@ +import os +import sys +from pathlib import Path + + +def _is_pytest_process() -> bool: + executable_name = Path(sys.argv[0]).name.lower() + return "pytest" in executable_name + + +if _is_pytest_process(): + os.environ.setdefault("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..267f8ca --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,111 @@ +import os +from pathlib import Path + +import pytest +from luxonis_ml.utils import LuxonisFileSystem + +os.environ.setdefault("LUXONIS_TELEMETRY_ENABLED", "false") + +NORMALIZED_TESTDATA_REMOTE_DIR = ( + "gs://luxonis-test-bucket/luxonis-eval-test-data/NormalizedTestData" +) +LOCAL_TESTDATA_ENV_VAR = "LUXONIS_EVAL_TESTDATA_DIR" + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--device-ip", + action="store", + default=None, + help="RVC4 device IP or MXID. If omitted, uses RVC4_IP or USB.", + ) + parser.addoption( + "--testbed-name", + action="store", + default=None, + help="Optional HIL testbed name used to resolve an RVC4 device.", + ) + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", "device: mark test as requiring a physical device" + ) + config.addinivalue_line( + "markers", "rvc4: mark test as requiring an RVC4 device" + ) + + +@pytest.fixture(scope="session") +def nnarchive_testdata_root(tmp_path_factory: pytest.TempPathFactory) -> Path: + configured_dir = os.environ.get(LOCAL_TESTDATA_ENV_VAR) + if configured_dir: + return Path(configured_dir) + + data_dir = tmp_path_factory.mktemp("nnarchive_testdata") + return LuxonisFileSystem.download( + NORMALIZED_TESTDATA_REMOTE_DIR, + dest=data_dir, + ) + + +@pytest.fixture(scope="session") +def rvc4_device_ip(request: pytest.FixtureRequest) -> str | None: + configured_device_ip = request.config.getoption("--device-ip") + if configured_device_ip: + return configured_device_ip + + env_device_ip = os.environ.get("RVC4_IP") + if env_device_ip: + return env_device_ip + + testbed_name = request.config.getoption("--testbed-name") or os.environ.get( + "HIL_TESTBED" + ) + if not testbed_name: + return None + + return _resolve_rvc4_device_ip_from_testbed(testbed_name) + + +def _resolve_rvc4_device_ip_from_testbed(testbed_name: str) -> str: + try: + from hil_framework.lib_testbed.config.Config import Config + from hil_framework.lib_testbed.utils.Testbed import Testbed + except ImportError as exc: + pytest.exit( + "hil_framework is required when --testbed-name or HIL_TESTBED is used.", + returncode=1, + ) + raise exc + + testbed = Testbed(Config(testbed_name)) + target_matches = [ + camera + for camera in testbed.cameras + if str(getattr(camera, "platform", "")).lower() == "rvc4" + ] + if len(target_matches) == 1: + hostname = getattr(target_matches[0], "hostname", None) + if not hostname: + pytest.exit( + f"Selected RVC4 camera in testbed {testbed_name!r} does not expose a hostname.", + returncode=1, + ) + return str(hostname) + + available_cameras = ", ".join( + f"{camera.name}:{getattr(camera, 'hostname', None)}:{getattr(camera, 'platform', None)}" + for camera in testbed.cameras + ) + if not target_matches: + pytest.exit( + f"No RVC4 camera found in testbed {testbed_name!r}. Available cameras: {available_cameras}", + returncode=1, + ) + + pytest.exit( + "Unable to select a unique RVC4 camera from testbed " + f"{testbed_name!r}. Available cameras: {available_cameras}", + returncode=1, + ) diff --git a/tests/nnarchive_regression_common.py b/tests/nnarchive_regression_common.py new file mode 100644 index 0000000..bea5701 --- /dev/null +++ b/tests/nnarchive_regression_common.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +from luxonis_ml.data import BucketStorage, LuxonisParser +from luxonis_ml.enums import DatasetType + +RELATIVE_TOLERANCE = 0.02 + + +@dataclass(frozen=True) +class RegressionCase: + name: str + + +CASES = [ + RegressionCase(name="detection_shapes_ldf_native"), + RegressionCase( + name="instance_segmentation_multiclass_squares_ldf_native_empty_task" + ), + RegressionCase(name="keypoint_squares_ldf_native_empty_task"), + RegressionCase( + name="semantic_segmentation_squares_ldf_native_empty_task" + ), +] + + +def case_dir(nnarchive_testdata_root: Path, case: RegressionCase) -> Path: + return nnarchive_testdata_root / case.name + + +def parse_dataset(case: RegressionCase, nnarchive_testdata_root: Path) -> None: + resolved_case_dir = case_dir(nnarchive_testdata_root, case) + LuxonisParser( + str(resolved_case_dir / case.name), + dataset_name=case.name, + dataset_type=DatasetType.NATIVE, + bucket_storage=BucketStorage.LOCAL, + delete_local=True, + ).parse() + + +def load_expected_metrics( + case: RegressionCase, nnarchive_testdata_root: Path +) -> dict[str, float]: + resolved_case_dir = case_dir(nnarchive_testdata_root, case) + return { + key: float(value) + for key, value in json.loads( + (resolved_case_dir / "metrics_summary.json").read_text( + encoding="utf-8" + ) + ).items() + } + + +def flatten_metrics( + metrics: list[tuple[str, dict[str, float]]], +) -> dict[str, float]: + flattened: dict[str, float] = {} + for metric_name, metric_values in metrics: + for metric_key, metric_value in metric_values.items(): + flattened[f"{metric_name}.{metric_key}"] = float(metric_value) + return flattened + + +def assert_within_relative_tolerance( + metric_key: str, *, actual: float, expected: float +) -> None: + if expected == 0: + assert actual == expected, ( + f"{metric_key} mismatch: actual={actual:.6f}, " + f"expected={expected:.6f}." + ) + return + + relative_error = abs(actual - expected) / abs(expected) + if relative_error <= RELATIVE_TOLERANCE: + return + + raise AssertionError( + f"{metric_key} mismatch: actual={actual:.6f}, " + f"expected={expected:.6f}, rel_error={relative_error:.2%}, " + f"allowed={RELATIVE_TOLERANCE:.2%}." + ) diff --git a/tests/run_rvc4_nnarchive_hil_tests.sh b/tests/run_rvc4_nnarchive_hil_tests.sh new file mode 100644 index 0000000..e8e4758 --- /dev/null +++ b/tests/run_rvc4_nnarchive_hil_tests.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +python3 -m venv venv +source venv/bin/activate + +python -m pip install --upgrade pip +python -m pip install gcsfs +python -m pip install -e ".[dev]" + +if [[ -n "${DEPTHAI_VERSION:-}" ]]; then + python -m pip install "depthai==${DEPTHAI_VERSION}" +fi + +if [[ -n "${HIL_FRAMEWORK_TOKEN:-}" && -n "${HIL_TESTBED:-}" ]]; then + python -m pip install --upgrade \ + --index-url "https://__token__:${HIL_FRAMEWORK_TOKEN}@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" \ + hil-framework +fi + +export LUXONIS_TELEMETRY_ENABLED="${LUXONIS_TELEMETRY_ENABLED:-false}" + +pytest tests/test_rvc4_nnarchive_regression.py -q "$@" diff --git a/tests/test_onnx_nnarchive_regression.py b/tests/test_onnx_nnarchive_regression.py new file mode 100644 index 0000000..848a718 --- /dev/null +++ b/tests/test_onnx_nnarchive_regression.py @@ -0,0 +1,55 @@ +from pathlib import Path + +import pytest + +from luxonis_eval.__main__ import quality_run +from tests.nnarchive_regression_common import ( + CASES, + RegressionCase, + assert_within_relative_tolerance, + case_dir, + flatten_metrics, + load_expected_metrics, + parse_dataset, +) + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.name) +def test_onnx_nnarchive_regression( + case: RegressionCase, + monkeypatch, + nnarchive_testdata_root: Path, + tmp_path: Path, +) -> None: + monkeypatch.setenv("LUXONISML_BASE_PATH", str(tmp_path / "luxonis_ml")) + monkeypatch.setenv("LUXONISML_TEAM_ID", "pytest") + + resolved_case_dir = case_dir(nnarchive_testdata_root, case) + + parse_dataset(case, nnarchive_testdata_root) + expected_metrics = load_expected_metrics(case, nnarchive_testdata_root) + + result = quality_run( + resolved_case_dir / "onnx_eval.yaml", + { + "pipeline.engine.model_path": str( + resolved_case_dir / "onnx_model.onnx.tar.xz" + ), + "runtime.logging.level": "WARNING", + "runtime.logging.use_rich": False, + }, + ) + actual_metrics = flatten_metrics(result["metrics"]) + + missing_metrics = sorted(set(expected_metrics) - set(actual_metrics)) + assert not missing_metrics, ( + f"{case.name} is missing expected metrics: {missing_metrics}. " + f"Available metrics: {sorted(actual_metrics)}" + ) + + for metric_key, expected_value in expected_metrics.items(): + assert_within_relative_tolerance( + metric_key, + actual=actual_metrics[metric_key], + expected=expected_value, + ) diff --git a/tests/test_rvc4_nnarchive_regression.py b/tests/test_rvc4_nnarchive_regression.py new file mode 100644 index 0000000..6859207 --- /dev/null +++ b/tests/test_rvc4_nnarchive_regression.py @@ -0,0 +1,59 @@ +from pathlib import Path + +import pytest + +from luxonis_eval.__main__ import quality_run +from tests.nnarchive_regression_common import ( + CASES, + RegressionCase, + assert_within_relative_tolerance, + case_dir, + flatten_metrics, + load_expected_metrics, + parse_dataset, +) + + +@pytest.mark.device +@pytest.mark.rvc4 +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.name) +def test_rvc4_nnarchive_regression( + case: RegressionCase, + monkeypatch: pytest.MonkeyPatch, + nnarchive_testdata_root: Path, + rvc4_device_ip: str | None, + tmp_path: Path, +) -> None: + monkeypatch.setenv("LUXONISML_BASE_PATH", str(tmp_path / "luxonis_ml")) + monkeypatch.setenv("LUXONISML_TEAM_ID", "pytest") + + resolved_case_dir = case_dir(nnarchive_testdata_root, case) + + parse_dataset(case, nnarchive_testdata_root) + expected_metrics = load_expected_metrics(case, nnarchive_testdata_root) + + result = quality_run( + resolved_case_dir / "depthai_eval.yaml", + { + "pipeline.engine.model_path": str( + resolved_case_dir / "rvc4_model.rvc4.tar.xz" + ), + "pipeline.engine.params.device_ip": rvc4_device_ip or "", + "runtime.logging.level": "WARNING", + "runtime.logging.use_rich": False, + }, + ) + actual_metrics = flatten_metrics(result["metrics"]) + + missing_metrics = sorted(set(expected_metrics) - set(actual_metrics)) + assert not missing_metrics, ( + f"{case.name} is missing expected metrics: {missing_metrics}. " + f"Available metrics: {sorted(actual_metrics)}" + ) + + for metric_key, expected_value in expected_metrics.items(): + assert_within_relative_tolerance( + metric_key, + actual=actual_metrics[metric_key], + expected=expected_value, + )