diff --git a/.github/workflows/webgpu-native.yml b/.github/workflows/webgpu-native.yml index e3f980e..5f56ead 100644 --- a/.github/workflows/webgpu-native.yml +++ b/.github/workflows/webgpu-native.yml @@ -42,6 +42,7 @@ jobs: tests.python.test_webgpu_runtime tests.python.test_webgpu_models tests.python.test_npm_release + tests.python.test_npm_webgpu_release tests.python.test_webgpu_qualification tests.python.test_webgpu_report_review - name: Validate the checked-in real-device report pair diff --git a/tests/python/npm_release_fixtures.py b/tests/python/npm_release_fixtures.py new file mode 100644 index 0000000..9520ff6 --- /dev/null +++ b/tests/python/npm_release_fixtures.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import platform as host_platform + +from tests.python.webgpu_runtime_fixtures import create_fake_packages +from tools import npm_release + + +def release_metadata(root: Path) -> Path: + metadata = root / "metadata" + (metadata / "licenses").mkdir(parents=True) + (metadata / "licenses" / "notice.txt").write_text("notice\n", "utf-8") + (metadata / "license-inventory.json").write_text( + '{"schemaVersion":"1.0","files":[]}\n', "utf-8" + ) + (metadata / "sbom.spdx.json").write_text( + '{"spdxVersion":"SPDX-2.3"}\n', "utf-8" + ) + return metadata + + +def stage_cpu_native_packages(root: Path) -> Path: + build_dir = root / "build" + binaries = build_dir / "bin" + binaries.mkdir(parents=True) + (binaries / "light_ocr_node.node").write_bytes(b"native-addon") + for platform in npm_release.PLATFORMS.values(): + (binaries / platform["runtime"]).write_bytes(platform["runtime"].encode()) + + metadata = release_metadata(root) + native_root = root / "native" + for platform_id in npm_release.PLATFORMS: + npm_release.stage_native( + argparse.Namespace( + platform_id=platform_id, + build_dir=build_dir, + metadata_dir=metadata, + output_dir=native_root / platform_id, + ) + ) + return native_root + + +def model_bundle(root: Path) -> Path: + bundle = root / "bundle" + bundle.mkdir() + (bundle / "manifest.json").write_text( + json.dumps( + { + "schemaVersion": "1.2", + "bundleId": npm_release.BUNDLE_ID, + "normalizedConfigPath": "normalized-config.json", + "providers": { + "apple": { + "schemaVersion": "1.1", + "devicePolicy": "open-macos", + "architectures": ["arm64", "x86_64"], + "validatedDeviceFamilies": ["Apple M4"], + }, + "webgpu": { + "schemaVersion": "1.0", + "conversionId": "onnxruntime-float16-1.24.4-20260719.1", + "precision": "fp16", + "graphOptimizationLevel": "extended", + "cpuPartition": "allow-required", + "requiredCpuOperators": ["Concat", "Gather", "Slice"], + }, + }, + } + ) + + "\n", + "utf-8", + ) + (bundle / "normalized-config.json").write_text( + json.dumps( + { + "schemaVersion": "1.2", + "runtimeProfiles": {"tiled": {"contractVersion": "tiled-v1"}}, + } + ) + + "\n", + "utf-8", + ) + return bundle + + +def webgpu_stage_inputs( + root: Path, lock: dict[str, object] +) -> tuple[dict[str, Path], Path, Path]: + packages = create_fake_packages(root, lock) + build_dir = root / "build" + binaries = build_dir / "bin" + binaries.mkdir(parents=True) + (binaries / "light_ocr_node.node").write_bytes(b"addon") + return packages, build_dir, release_metadata(root) + + +def current_platform_id() -> str | None: + machine = host_platform.machine().lower() + system = host_platform.system() + if system == "Darwin" and machine in {"arm64", "aarch64"}: + return "macos-arm64" + if system == "Darwin" and machine in {"x86_64", "amd64"}: + return "macos-x64" + if system == "Linux" and machine in {"x86_64", "amd64"}: + return "linux-x64" + if system == "Windows" and machine in {"x86_64", "amd64"}: + return "windows-x64" + return None diff --git a/tests/python/test_bootstrap_dependencies.py b/tests/python/test_bootstrap_dependencies.py index a82471f..22c204f 100644 --- a/tests/python/test_bootstrap_dependencies.py +++ b/tests/python/test_bootstrap_dependencies.py @@ -5,6 +5,7 @@ import json from pathlib import Path import stat +import tarfile import tempfile import unittest from unittest import mock @@ -63,6 +64,28 @@ def locked(data: bytes) -> dict[str, object]: class BootstrapDependenciesTest(unittest.TestCase): + def test_archive_inspection_rejects_traversal_and_links(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + traversal = root / "traversal.zip" + with zipfile.ZipFile(traversal, "w") as archive: + archive.writestr("../escape", "unsafe") + link = root / "link.tar" + with tarfile.open(link, "w") as archive: + member = tarfile.TarInfo("link") + member.type = tarfile.SYMTYPE + member.linkname = "target" + archive.addfile(member) + + for name, path, error in ( + ("traversal", traversal, "unsafe archive member path"), + ("link", link, "unsupported archive member"), + ): + with self.subTest(name=name), self.assertRaisesRegex( + RuntimeError, error + ): + bootstrap_dependencies.inspect_archive(path) + def test_selects_common_and_one_matching_runtime(self) -> None: lock = { "dependencies": [ diff --git a/tests/python/test_npm_release.py b/tests/python/test_npm_release.py index 808596d..089afe1 100644 --- a/tests/python/test_npm_release.py +++ b/tests/python/test_npm_release.py @@ -1,19 +1,20 @@ from __future__ import annotations import argparse -import copy import json from pathlib import Path -import platform as host_platform import shutil import subprocess import tempfile import unittest from unittest import mock +from tests.python.npm_release_fixtures import ( + current_platform_id, + model_bundle, + stage_cpu_native_packages, +) from tools import npm_release -from tools.webgpu import build_runtime -from tests.python.test_webgpu_runtime import create_fake_packages, locked class NpmReleaseTests(unittest.TestCase): @@ -53,17 +54,6 @@ def test_single_config_generator_ignores_stale_configuration_types(self) -> None binary, ) - def test_rejects_a_pre_tiled_package_version(self) -> None: - with self.assertRaisesRegex(RuntimeError, "0.2.0 or newer"): - npm_release.assemble( - argparse.Namespace( - version="0.1.1", - bundle=Path("unused"), - native_root=Path("unused"), - output_dir=Path("unused"), - ) - ) - def test_rejects_a_version_that_does_not_match_the_source(self) -> None: with self.assertRaisesRegex(RuntimeError, "does not match source version"): npm_release.assemble( @@ -106,32 +96,7 @@ def test_stages_and_deterministically_packs_six_packages(self) -> None: self.skipTest("npm is unavailable") with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) - build = root / "build" / "bin" - build.mkdir(parents=True) - (build / "light_ocr_node.node").write_bytes(b"native-addon") - for platform in npm_release.PLATFORMS.values(): - (build / platform["runtime"]).write_bytes(platform["runtime"].encode()) - - metadata = root / "metadata" - (metadata / "licenses").mkdir(parents=True) - (metadata / "licenses" / "dependency.txt").write_text("license\n", "utf-8") - (metadata / "license-inventory.json").write_text( - '{"schemaVersion":"1.0","files":[]}\n', "utf-8" - ) - (metadata / "sbom.spdx.json").write_text( - '{"spdxVersion":"SPDX-2.3"}\n', "utf-8" - ) - - native_root = root / "native" - for platform_id in npm_release.PLATFORMS: - npm_release.stage_native( - argparse.Namespace( - platform_id=platform_id, - build_dir=build.parent, - metadata_dir=metadata, - output_dir=native_root / platform_id, - ) - ) + native_root = stage_cpu_native_packages(root) for platform_id in ("macos-arm64", "macos-x64"): descriptor = json.loads( ( @@ -149,51 +114,11 @@ def test_stages_and_deterministically_packs_six_packages(self) -> None: {"apple", "cpu"} if platform_id == "macos-arm64" else {"cpu"}, ) - bundle = root / "bundle" - bundle.mkdir() - (bundle / "manifest.json").write_text( - json.dumps( - { - "schemaVersion": "1.2", - "bundleId": npm_release.BUNDLE_ID, - "normalizedConfigPath": "normalized-config.json", - "providers": { - "apple": { - "schemaVersion": "1.1", - "devicePolicy": "open-macos", - "architectures": ["arm64", "x86_64"], - "validatedDeviceFamilies": ["Apple M4"], - }, - "webgpu": { - "schemaVersion": "1.0", - "conversionId": "onnxruntime-float16-1.24.4-20260719.1", - "precision": "fp16", - "graphOptimizationLevel": "extended", - "cpuPartition": "allow-required", - "requiredCpuOperators": ["Concat", "Gather", "Slice"], - }, - }, - } - ) - + "\n", - "utf-8", - ) - (bundle / "normalized-config.json").write_text( - json.dumps( - { - "schemaVersion": "1.2", - "runtimeProfiles": {"tiled": {"contractVersion": "tiled-v1"}}, - } - ) - + "\n", - "utf-8", - ) - staging = root / "staging" npm_release.assemble( argparse.Namespace( version="0.3.0", - bundle=bundle, + bundle=model_bundle(root), native_root=native_root, output_dir=staging, ) @@ -221,16 +146,8 @@ def test_stages_and_deterministically_packs_six_packages(self) -> None: self.assertEqual(len(release["packages"]), 6) self.assertEqual(len(list(tarballs.glob("*.tgz"))), 6) - machine = host_platform.machine().lower() - if host_platform.system() == "Darwin" and machine in {"arm64", "aarch64"}: - platform_id = "macos-arm64" - elif host_platform.system() == "Darwin" and machine in {"x86_64", "amd64"}: - platform_id = "macos-x64" - elif host_platform.system() == "Linux" and machine in {"x86_64", "amd64"}: - platform_id = "linux-x64" - elif host_platform.system() == "Windows" and machine in {"x86_64", "amd64"}: - platform_id = "windows-x64" - else: + platform_id = current_platform_id() + if platform_id is None: return filenames = { record["name"]: record["filename"] for record in release["packages"] @@ -319,199 +236,5 @@ def test_runtime_descriptor_rejects_mutated_payload_and_qualification_release( descriptor, root, require_released=True ) - def test_rejects_pending_webgpu_release_but_stages_both_qualification_packages( - self, - ) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - lock = copy.deepcopy(locked()) - qualification = lock["qualification"] - qualification["status"] = "development-pending-device-validation" - qualification["providerGatePassed"] = False - qualification["productionArtifactQualified"] = False - qualification["qualifiedArtifactSetSha256"] = { - "linux-x64": None, - "windows-x64": None, - } - qualification["qualificationReportSha256"] = { - "linux-x64": None, - "windows-x64": None, - } - packages = create_fake_packages(root, lock) - build = root / "build" / "bin" - build.mkdir(parents=True) - (build / "light_ocr_node.node").write_bytes(b"addon") - - metadata = root / "metadata" - (metadata / "licenses").mkdir(parents=True) - (metadata / "licenses" / "notice.txt").write_text("notice", "utf-8") - (metadata / "license-inventory.json").write_text("{}\n", "utf-8") - (metadata / "sbom.spdx.json").write_text("{}\n", "utf-8") - - for platform_id, expected_names in ( - ( - "linux-x64", - { - "libonnxruntime.so.1", - "libonnxruntime_providers_webgpu.so", - }, - ), - ( - "windows-x64", - { - "onnxruntime.dll", - "onnxruntime_providers_webgpu.dll", - "dxcompiler.dll", - "dxil.dll", - }, - ), - ): - with self.subTest(platform=platform_id): - sdk = root / f"sdk-{platform_id}" - manifest_path = build_runtime.stage_runtime( - lock, platform_id, packages, sdk - ) - build_runtime.validate_sdk(sdk, lock) - output = root / f"output-{platform_id}" - arguments = argparse.Namespace( - platform_id=platform_id, - build_dir=build.parent, - metadata_dir=metadata, - output_dir=output, - runtime_flavor="webgpu", - webgpu_artifact_manifest=manifest_path, - qualification_build=False, - ) - with mock.patch( - "tools.npm_release.webgpu_runtime.load_lock", - return_value=lock, - ): - with self.assertRaisesRegex( - RuntimeError, "accepted Linux and Windows Provider Gates" - ): - npm_release.stage_native(arguments) - self.assertFalse(output.exists()) - - arguments.qualification_build = True - with mock.patch( - "tools.npm_release.webgpu_runtime.load_lock", - return_value=lock, - ): - npm_release.stage_native(arguments) - descriptor = json.loads( - (output / "native" / "runtime-descriptor.json").read_text( - "utf-8" - ) - ) - self.assertEqual(descriptor["schemaVersion"], "2.0") - self.assertTrue(descriptor["qualificationOnly"]) - self.assertFalse(descriptor["released"]) - self.assertEqual( - descriptor["autoPolicy"]["providers"], ["webgpu", "cpu"] - ) - self.assertEqual( - { - Path(record["path"]).name - for record in descriptor["runtime"]["artifacts"] - }, - expected_names, - ) - provider = descriptor["providers"]["webgpu"] - self.assertEqual(provider["providerVersion"], "0.1.0") - self.assertIn(provider["providerLibrary"], provider["artifacts"]) - self.assertEqual( - provider["providerLibrary"]["path"], - ( - "native/onnxruntime_providers_webgpu.dll" - if platform_id == "windows-x64" - else "native/libonnxruntime_providers_webgpu.so" - ), - ) - with self.assertRaisesRegex( - RuntimeError, "cannot enter npm release" - ): - npm_release.validate_runtime_descriptor( - descriptor, output, require_released=True - ) - - provider_path = output / provider["providerLibrary"]["path"] - provider_path.write_bytes(b"tampered") - with self.assertRaisesRegex( - RuntimeError, "(?:byte count|hash) mismatch" - ): - npm_release.validate_runtime_descriptor(descriptor, output) - - def test_stages_production_webgpu_only_after_both_platforms_are_bound( - self, - ) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - lock = locked() - packages = create_fake_packages(root, lock) - artifact_hashes: dict[str, str] = {} - for platform_id in ("linux-x64", "windows-x64"): - sdk = root / f"pending-{platform_id}" - build_runtime.stage_runtime(lock, platform_id, packages, sdk) - manifest = build_runtime.validate_sdk(sdk, lock) - artifact_hashes[platform_id] = manifest["artifacts"][ - "artifactSetSha256" - ] - - qualification = lock["qualification"] - qualification["status"] = "production-qualified" - qualification["providerGatePassed"] = True - qualification["productionArtifactQualified"] = True - qualification["qualifiedArtifactSetSha256"] = artifact_hashes - qualification["qualificationReportSha256"] = { - "linux-x64": "3" * 64, - "windows-x64": "4" * 64, - } - build_runtime.validate_lock(lock) - - build = root / "build" / "bin" - build.mkdir(parents=True) - (build / "light_ocr_node.node").write_bytes(b"addon") - metadata = root / "metadata" - (metadata / "licenses").mkdir(parents=True) - (metadata / "licenses" / "notice.txt").write_text("notice", "utf-8") - (metadata / "license-inventory.json").write_text("{}\n", "utf-8") - (metadata / "sbom.spdx.json").write_text("{}\n", "utf-8") - - for platform_id in ("linux-x64", "windows-x64"): - with self.subTest(platform=platform_id): - sdk = root / f"qualified-{platform_id}" - manifest_path = build_runtime.stage_runtime( - lock, platform_id, packages, sdk - ) - output = root / f"release-{platform_id}" - arguments = argparse.Namespace( - platform_id=platform_id, - build_dir=build.parent, - metadata_dir=metadata, - output_dir=output, - runtime_flavor="webgpu", - webgpu_artifact_manifest=manifest_path, - qualification_build=False, - ) - with mock.patch( - "tools.npm_release.webgpu_runtime.load_lock", - return_value=lock, - ): - npm_release.stage_native(arguments) - descriptor = json.loads( - (output / "native" / "runtime-descriptor.json").read_text( - "utf-8" - ) - ) - self.assertFalse(descriptor["qualificationOnly"]) - self.assertTrue(descriptor["released"]) - npm_release.validate_runtime_descriptor( - descriptor, - output, - platform_id=platform_id, - require_released=True, - ) - - if __name__ == "__main__": unittest.main() diff --git a/tests/python/test_npm_webgpu_release.py b/tests/python/test_npm_webgpu_release.py new file mode 100644 index 0000000..233ccee --- /dev/null +++ b/tests/python/test_npm_webgpu_release.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import argparse +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from tests.python.npm_release_fixtures import webgpu_stage_inputs +from tests.python.webgpu_runtime_fixtures import locked +from tools import npm_release +from tools.webgpu import build_runtime + + +class NpmWebGpuReleaseTests(unittest.TestCase): + def test_rejects_pending_release_but_stages_qualification_packages(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + lock = copy.deepcopy(locked()) + qualification = lock["qualification"] + qualification["status"] = "development-pending-device-validation" + qualification["providerGatePassed"] = False + qualification["productionArtifactQualified"] = False + qualification["qualifiedArtifactSetSha256"] = { + "linux-x64": None, + "windows-x64": None, + } + qualification["qualificationReportSha256"] = { + "linux-x64": None, + "windows-x64": None, + } + packages, build_dir, metadata = webgpu_stage_inputs(root, lock) + + for platform_id, expected_names in ( + ( + "linux-x64", + {"libonnxruntime.so.1", "libonnxruntime_providers_webgpu.so"}, + ), + ( + "windows-x64", + { + "onnxruntime.dll", + "onnxruntime_providers_webgpu.dll", + "dxcompiler.dll", + "dxil.dll", + }, + ), + ): + with self.subTest(platform=platform_id): + sdk = root / f"sdk-{platform_id}" + manifest_path = build_runtime.stage_runtime( + lock, platform_id, packages, sdk + ) + build_runtime.validate_sdk(sdk, lock) + output = root / f"output-{platform_id}" + arguments = argparse.Namespace( + platform_id=platform_id, + build_dir=build_dir, + metadata_dir=metadata, + output_dir=output, + runtime_flavor="webgpu", + webgpu_artifact_manifest=manifest_path, + qualification_build=False, + ) + with mock.patch( + "tools.npm_release.webgpu_runtime.load_lock", + return_value=lock, + ): + with self.assertRaisesRegex( + RuntimeError, "accepted Linux and Windows Provider Gates" + ): + npm_release.stage_native(arguments) + self.assertFalse(output.exists()) + + arguments.qualification_build = True + with mock.patch( + "tools.npm_release.webgpu_runtime.load_lock", + return_value=lock, + ): + npm_release.stage_native(arguments) + descriptor = json.loads( + (output / "native" / "runtime-descriptor.json").read_text( + "utf-8" + ) + ) + self.assertEqual(descriptor["schemaVersion"], "2.0") + self.assertTrue(descriptor["qualificationOnly"]) + self.assertFalse(descriptor["released"]) + self.assertEqual( + descriptor["autoPolicy"]["providers"], ["webgpu", "cpu"] + ) + self.assertEqual( + { + Path(record["path"]).name + for record in descriptor["runtime"]["artifacts"] + }, + expected_names, + ) + provider = descriptor["providers"]["webgpu"] + self.assertEqual(provider["providerVersion"], "0.1.0") + self.assertIn(provider["providerLibrary"], provider["artifacts"]) + expected_library = ( + "native/onnxruntime_providers_webgpu.dll" + if platform_id == "windows-x64" + else "native/libonnxruntime_providers_webgpu.so" + ) + self.assertEqual( + provider["providerLibrary"]["path"], expected_library + ) + with self.assertRaisesRegex( + RuntimeError, "cannot enter npm release" + ): + npm_release.validate_runtime_descriptor( + descriptor, output, require_released=True + ) + + provider_path = output / provider["providerLibrary"]["path"] + provider_path.write_bytes(b"tampered") + with self.assertRaisesRegex( + RuntimeError, "(?:byte count|hash) mismatch" + ): + npm_release.validate_runtime_descriptor(descriptor, output) + + def test_stages_production_only_after_both_platforms_are_bound(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + lock = locked() + packages, build_dir, metadata = webgpu_stage_inputs(root, lock) + artifact_hashes: dict[str, str] = {} + for platform_id in ("linux-x64", "windows-x64"): + sdk = root / f"pending-{platform_id}" + build_runtime.stage_runtime(lock, platform_id, packages, sdk) + manifest = build_runtime.validate_sdk(sdk, lock) + artifact_hashes[platform_id] = manifest["artifacts"][ + "artifactSetSha256" + ] + + qualification = lock["qualification"] + qualification["status"] = "production-qualified" + qualification["providerGatePassed"] = True + qualification["productionArtifactQualified"] = True + qualification["qualifiedArtifactSetSha256"] = artifact_hashes + qualification["qualificationReportSha256"] = { + "linux-x64": "3" * 64, + "windows-x64": "4" * 64, + } + build_runtime.validate_lock(lock) + + for platform_id in ("linux-x64", "windows-x64"): + with self.subTest(platform=platform_id): + sdk = root / f"qualified-{platform_id}" + manifest_path = build_runtime.stage_runtime( + lock, platform_id, packages, sdk + ) + output = root / f"release-{platform_id}" + arguments = argparse.Namespace( + platform_id=platform_id, + build_dir=build_dir, + metadata_dir=metadata, + output_dir=output, + runtime_flavor="webgpu", + webgpu_artifact_manifest=manifest_path, + qualification_build=False, + ) + with mock.patch( + "tools.npm_release.webgpu_runtime.load_lock", + return_value=lock, + ): + npm_release.stage_native(arguments) + descriptor = json.loads( + (output / "native" / "runtime-descriptor.json").read_text( + "utf-8" + ) + ) + self.assertFalse(descriptor["qualificationOnly"]) + self.assertTrue(descriptor["released"]) + npm_release.validate_runtime_descriptor( + descriptor, + output, + platform_id=platform_id, + require_released=True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_tiled_release_gate.py b/tests/python/test_tiled_release_gate.py index d6c390b..fe46e31 100644 --- a/tests/python/test_tiled_release_gate.py +++ b/tests/python/test_tiled_release_gate.py @@ -89,28 +89,6 @@ def test_node_bootstrap_observes_cross_process_deltas(self) -> None: ) self.assertEqual(observed, (1.1, 1.15, 60 * 1024 * 1024)) - def test_node_bootstrap_does_not_gate_non_interleaved_latency(self) -> None: - node = {"latencyUs": {"median": 111, "p95": 120}} - core = {"latencyUs": {"median": 100, "p95": 120}} - - observed = tiled_release_gate.observe_node_against_core( - node, core, 600 * 1024 * 1024, 540 * 1024 * 1024, "fixture" - ) - self.assertEqual(observed[0], 1.11) - - def test_node_bootstrap_does_not_gate_process_baseline_delta(self) -> None: - node = {"latencyUs": {"median": 100, "p95": 120}} - core = {"latencyUs": {"median": 100, "p95": 120}} - - observed = tiled_release_gate.observe_node_against_core( - node, - core, - 605 * 1024 * 1024, - 540 * 1024 * 1024, - "fixture", - ) - self.assertEqual(observed[2], 65 * 1024 * 1024) - def test_node_bootstrap_rejects_absolute_peak(self) -> None: node = {"latencyUs": {"median": 100, "p95": 120}} core = {"latencyUs": {"median": 100, "p95": 120}} diff --git a/tests/python/test_webgpu_models.py b/tests/python/test_webgpu_models.py index 7a8dfae..68b5cd2 100644 --- a/tests/python/test_webgpu_models.py +++ b/tests/python/test_webgpu_models.py @@ -1,60 +1,33 @@ from __future__ import annotations -import hashlib import json from pathlib import Path import tempfile import unittest +from unittest import mock from tools.webgpu import package_bundle class WebGpuModelsTest(unittest.TestCase): - def test_tracked_fp16_artifact_matches_lock_and_provenance(self) -> None: + def test_tracked_fp16_artifact_satisfies_the_packaging_contract(self) -> None: + bundle, artifact = package_bundle.locked_artifact() + package_bundle.validate_locked_artifact(bundle, artifact) + + def test_packaging_contract_rejects_semantic_provenance_drift(self) -> None: bundle, artifact = package_bundle.locked_artifact() root = package_bundle.ROOT / artifact["directory"] - provenance_bytes = package_bundle.verify_file( - root / artifact["provenance"]["path"], - artifact["provenance"], - "provenance", - ) - provenance = json.loads(provenance_bytes) - self.assertEqual(provenance["artifactId"], artifact["artifactId"]) - self.assertEqual(provenance["conversionId"], artifact["conversionId"]) - self.assertEqual( - provenance["runtimeContract"], - { - "precision": "fp16", - "graphOptimizationLevel": "extended", - "cpuPartition": "allow-required", - "requiredCpuOperators": ["Concat", "Gather", "Slice"], - }, + provenance = json.loads( + (root / artifact["provenance"]["path"]).read_text("utf-8") ) - source_artifacts = { - record["name"]: record for record in bundle["artifacts"] - } - for kind in ("detection", "recognition"): - data = package_bundle.verify_file( - root / artifact[kind]["path"], artifact[kind], kind - ) - self.assertEqual(hashlib.sha256(data).hexdigest(), artifact[kind]["sha256"]) - self.assertEqual(provenance["models"][kind]["output"], artifact[kind]) - self.assertEqual( - provenance["models"][kind]["source"], - { - "path": ( - "det/inference.onnx" - if kind == "detection" - else "rec/inference.onnx" - ), - "bytes": source_artifacts[kind]["members"]["inference.onnx"][ - "bytes" - ], - "sha256": source_artifacts[kind]["members"]["inference.onnx"][ - "sha256" - ], - }, - ) + provenance["models"]["detection"]["output"]["sha256"] = "0" * 64 + with mock.patch.object( + package_bundle, + "verify_file", + side_effect=[json.dumps(provenance).encode(), b"det", b"rec"], + ): + with self.assertRaisesRegex(RuntimeError, "detection provenance"): + package_bundle.validate_locked_artifact(bundle, artifact) def test_locked_file_verification_rejects_tampering(self) -> None: with tempfile.TemporaryDirectory() as directory: diff --git a/tests/python/test_webgpu_qualification.py b/tests/python/test_webgpu_qualification.py index ad16c1f..b2d8aab 100644 --- a/tests/python/test_webgpu_qualification.py +++ b/tests/python/test_webgpu_qualification.py @@ -1,6 +1,5 @@ from __future__ import annotations -import importlib.util import json from pathlib import Path import subprocess @@ -8,76 +7,15 @@ import unittest from unittest import mock - -ROOT = Path(__file__).resolve().parents[2] -MODULE_PATH = ROOT / "tools" / "webgpu" / "qualify.py" -SPEC = importlib.util.spec_from_file_location("webgpu_qualify", MODULE_PATH) -assert SPEC is not None and SPEC.loader is not None -qualify = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(qualify) - - -def line(*, text: str = "HELLO 123", confidence: float = 0.95) -> dict[str, object]: - return { - "text": text, - "confidence": confidence, - "box": [[1.0, 2.0], [20.0, 2.0], [20.0, 12.0], [1.0, 12.0]], - } - - -def report( - mode: str, chain: list[str], *, lifecycle: bool = False -) -> dict[str, object]: - cpu = mode == "cpu" - value: dict[str, object] = { - "schemaVersion": "1.1", - "ok": True, - "result": { - "lines": [line()], - "deterministic": True, - "sha256": "1" * 64, - }, - "engine": { - "executionProvider": ( - "CPUExecutionProvider" if mode == "cpu" else "WebGpuExecutionProvider" - ), - "execution": { - "sessions": { - "detection": { - "actualProviderChain": chain, - "precision": "fp16" if mode == "allow" else "fp32", - }, - "recognition": { - "actualProviderChain": chain, - "precision": "fp16" if mode == "allow" else "fp32", - }, - } - }, - }, - "latencyUs": { - "minimum": 80 if not cpu else 120, - "p50": 100 if not cpu else 160, - "p95": 120 if not cpu else 160, - "maximum": 140 if not cpu else 180, - }, - "warmup": 2, - "iterations": 10, - "cycles": 3, - "engineInitializationUs": { - "minimum": 1000, - "p50": 1000, - "maximum": 1000, - "values": [1000, 1000, 1000], - }, - "firstPredictionUs": 2000, - "firstPredictionUsByCycle": [2000, 2000, 2000], - "lifecycle": { - "residentMinimumBytes": 100 * 1024 * 1024, - "residentMaximumBytes": 110 * 1024 * 1024, - "retainedGrowthBytes": 1024 if lifecycle else 0, - }, - } - return value +from tests.python.webgpu_report_fixtures import ( + LIFECYCLE_RSS_MIB, + WEBGPU_CHAIN, + ocr_line, + qualification_case, + qualification_layout, + synthetic_gate_inputs, +) +from tools.webgpu import qualify class WebGpuQualificationTest(unittest.TestCase): @@ -136,13 +74,13 @@ def test_profile_summary_records_provider_placement_and_hashes(self) -> None: ) def test_quality_gate_accepts_tolerance_and_rejects_invalid_results(self) -> None: - cpu = {"result": {"lines": [line()]}} - close = {"result": {"lines": [line(confidence=0.93)]}} + cpu = {"result": {"lines": [ocr_line()]}} + close = {"result": {"lines": [ocr_line(confidence=0.93)]}} self.assertTrue(qualify.quality_matches(cpu, close)[0]) - wrong_text = {"result": {"lines": [line(text="HELLO 124")]}} + wrong_text = {"result": {"lines": [ocr_line(text="HELLO 124")]}} self.assertFalse(qualify.quality_matches(cpu, wrong_text)[0]) - non_finite = {"result": {"lines": [line(confidence=float("nan"))]}} + non_finite = {"result": {"lines": [ocr_line(confidence=float("nan"))]}} self.assertEqual( qualify.quality_matches(cpu, non_finite), (False, "confidence is not finite"), @@ -151,111 +89,8 @@ def test_quality_gate_accepts_tolerance_and_rejects_invalid_results(self) -> Non def test_collect_evidence_passes_a_complete_synthetic_gate(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - sdk = root / "sdk" - native = root / "native-package" - (native / "native").mkdir(parents=True) - sdk.mkdir() - (sdk / "artifact-manifest.json").write_text( - json.dumps( - { - "contractId": "native-webgpu-plugin-0.1.0-ort-1.24.4-v1", - "artifacts": {"artifactSetSha256": "2" * 64}, - "qualification": {"evidenceId": "synthetic-evidence"}, - } - ), - "utf-8", - ) - (native / "native" / "runtime-descriptor.json").write_text( - json.dumps({"runtime": {"kind": "onnxruntime-plugin-webgpu"}}), - "utf-8", - ) - cpu = report("cpu", ["CPUExecutionProvider"]) - allow = report( - "allow", - ["WebGpuExecutionProvider", "CPUExecutionProvider"], - lifecycle=True, - ) - fp32 = report( - "fp32", ["WebGpuExecutionProvider", "CPUExecutionProvider"] - ) - strict = { - "schemaVersion": "1.1", - "ok": True, - "expectedRejection": True, - "error": { - "code": "unsupported_capability", - "message": "The WebGPU model requires a bounded CPU operator partition", - "detail": "required operators: Concat, Gather, Slice", - }, - } - auto = report("auto", ["WebGpuExecutionProvider", "CPUExecutionProvider"]) - auto["host"] = {"platform": "linux", "architecture": "x64"} - auto["engine"]["execution"]["selectionTrace"] = { - "orderedCandidates": ["webgpu", "cpu"], - "selectedProvider": "webgpu", - } - cases = { - "generated-hello-123:cpu": cpu, - "generated-hello-123:fp32": fp32, - "generated-hello-123:allow": allow, - "generated-hello-123:strict": strict, - "generated-hello-123:auto": auto, - "generated-hello-123:lifecycle": report( - "allow", - ["WebGpuExecutionProvider", "CPUExecutionProvider"], - lifecycle=True, - ), - "native-cpp:auto": { - "ok": True, - "engineInitializationUs": 1000, - "firstPredictionUs": 2000, - "memoryBytes": {"peakResident": 120 * 1024 * 1024}, - "execution": { - "requestedProvider": "auto", - "selectionTrace": { - "orderedCandidates": ["webgpu", "cpu"], - "selectedProvider": "webgpu", - }, - "detection": { - "actualProviderChain": [ - "WebGpuExecutionProvider", - "CPUExecutionProvider", - ] - }, - "recognition": { - "actualProviderChain": [ - "WebGpuExecutionProvider", - "CPUExecutionProvider", - ] - }, - }, - }, - } - cases["generated-hello-123:lifecycle"]["cycles"] = 20 - cases["native-cpp:auto"]["warmup"] = 1 - cases["native-cpp:auto"]["iterations"] = 10 - profiles = { - "generated-hello-123:fp32": { - "files": ["fp32.json"], - "nodeCounts": {"WebGpuExecutionProvider": 10}, - }, - "generated-hello-123:allow": { - "files": ["allow.json"], - "nodeCounts": {"WebGpuExecutionProvider": 10}, - }, - "generated-hello-123:auto": { - "files": ["auto.json"], - "nodeCounts": {"WebGpuExecutionProvider": 10}, - }, - "native-cpp:auto": { - "files": ["native-cpp-auto.json"], - "nodeCounts": {"WebGpuExecutionProvider": 10}, - }, - "generated-hello-123:lifecycle": { - "files": ["lifecycle.json"], - "nodeCounts": {"WebGpuExecutionProvider": 10}, - }, - } + sdk, native = qualification_layout(root) + cases, profiles = synthetic_gate_inputs() evidence = qualify.collect_evidence( platform_id="linux-x64", sdk=sdk, @@ -307,54 +142,13 @@ def test_lifecycle_gate_uses_warmup_aware_baseline_when_rss_samples_present(self # warmup) as a leak and fails a healthy run. with tempfile.TemporaryDirectory() as directory: root = Path(directory) - sdk = root / "sdk" - native = root / "native-package" - (native / "native").mkdir(parents=True) - sdk.mkdir() - (sdk / "artifact-manifest.json").write_text( - json.dumps( - { - "contractId": "native-webgpu-plugin-0.1.0-ort-1.24.4-v1", - "artifacts": {"artifactSetSha256": "2" * 64}, - "runtime": {"kind": "onnxruntime-plugin-webgpu"}, - "qualification": {"evidenceId": "test-eid"}, - } - ), - "utf-8", - ) - (native / "native" / "runtime-descriptor.json").write_text( - json.dumps({"runtime": {"kind": "onnxruntime-plugin-webgpu"}}), - "utf-8", - ) + sdk, native = qualification_layout(root) mib = 1024 * 1024 # 20 cycles * 2 samples each, mirroring the observed WebGPU run: # ramp 270 -> 466 in first 5 cycles, then plateaus around 380-460. - rss_bytes = [ - 270 * mib, 288 * mib, # cycle 1 - 322 * mib, 352 * mib, # cycle 2 - 374 * mib, 391 * mib, # cycle 3 - 406 * mib, 413 * mib, # cycle 4 - 429 * mib, 406 * mib, # cycle 5 (warmup ends here) - 414 * mib, 410 * mib, # cycle 6 (measured region starts) - 423 * mib, 419 * mib, # cycle 7 - 442 * mib, 419 * mib, # cycle 8 - 450 * mib, 445 * mib, # cycle 9 - 448 * mib, 427 * mib, # cycle 10 - 455 * mib, 451 * mib, # cycle 11 - 464 * mib, 460 * mib, # cycle 12 - 458 * mib, 432 * mib, # cycle 13 - 440 * mib, 428 * mib, # cycle 14 - 435 * mib, 422 * mib, # cycle 15 - 430 * mib, 425 * mib, # cycle 16 - 428 * mib, 432 * mib, # cycle 17 - 430 * mib, 428 * mib, # cycle 18 - 429 * mib, 431 * mib, # cycle 19 - 430 * mib, 432 * mib, # cycle 20 - ] - lifecycle_case = report( - "allow", - ["WebGpuExecutionProvider", "CPUExecutionProvider"], - lifecycle=True, + rss_bytes = [value * mib for value in LIFECYCLE_RSS_MIB] + lifecycle_case = qualification_case( + "allow", WEBGPU_CHAIN, lifecycle=True ) lifecycle_case["cycles"] = 20 lifecycle_case["lifecycle"]["rssBytes"] = rss_bytes diff --git a/tests/python/test_webgpu_report_review.py b/tests/python/test_webgpu_report_review.py index 0b44e85..3e5df83 100644 --- a/tests/python/test_webgpu_report_review.py +++ b/tests/python/test_webgpu_report_review.py @@ -1,361 +1,26 @@ from __future__ import annotations -import copy -import hashlib import json -from pathlib import Path, PurePosixPath +from pathlib import Path import tempfile import unittest -from tools.webgpu import build_runtime, qualify, review_reports - - -def pending_lock() -> dict[str, object]: - lock = build_runtime.load_lock() - qualification = lock["qualification"] - qualification["status"] = "development-pending-device-validation" - qualification["providerGatePassed"] = False - qualification["productionArtifactQualified"] = False - qualification["qualifiedArtifactSetSha256"] = { - "linux-x64": None, - "windows-x64": None, - } - qualification["qualificationReportSha256"] = { - "linux-x64": None, - "windows-x64": None, - } - return lock - - -def line() -> dict[str, object]: - return { - "text": "HELLO 123", - "confidence": 0.95, - "box": [[1.0, 2.0], [20.0, 2.0], [20.0, 12.0], [1.0, 12.0]], - } - - -def node_case(mode: str, chain: list[str]) -> dict[str, object]: - cpu = mode == "cpu" - return { - "schemaVersion": "1.1", - "ok": True, - "result": { - "lines": [line()], - "deterministic": True, - "sha256": "1" * 64, - }, - "engine": { - "executionProvider": ( - "CPUExecutionProvider" if cpu else "WebGpuExecutionProvider" - ), - "execution": { - "sessions": { - "detection": { - "actualProviderChain": chain, - "precision": "fp32", - }, - "recognition": { - "actualProviderChain": chain, - "precision": "fp32", - }, - } - }, - }, - "latencyUs": { - "minimum": 80 if not cpu else 120, - "p50": 100 if not cpu else 160, - "p95": 120 if not cpu else 160, - "maximum": 140 if not cpu else 180, - }, - "warmup": 2, - "iterations": 10, - "cycles": 3, - "engineInitializationUs": { - "minimum": 1000, - "p50": 1000, - "maximum": 1000, - "values": [1000, 1000, 1000], - }, - "firstPredictionUs": 2000, - "firstPredictionUsByCycle": [2000, 2000, 2000], - "processCpuUs": 1000, - "measuredWallUs": 2000, - "averageProcessCpuCores": 0.5, - "lifecycle": { - "residentMinimumBytes": 100 * 1024 * 1024, - "residentMaximumBytes": 110 * 1024 * 1024, - "retainedGrowthBytes": 1024, - }, - } - - -def cases_and_profiles() -> tuple[dict[str, dict], dict[str, dict]]: - cases: dict[str, dict] = {} - profiles: dict[str, dict] = {} - for fixture in qualify.DEFAULT_FIXTURES: - cases[f"{fixture}:cpu"] = node_case("cpu", ["CPUExecutionProvider"]) - cases[f"{fixture}:allow"] = node_case( - "allow", ["WebGpuExecutionProvider", "CPUExecutionProvider"] - ) - cases[f"{fixture}:strict"] = { - "schemaVersion": "1.1", - "ok": True, - "expectedRejection": True, - "error": { - "code": "unsupported_capability", - "message": "The WebGPU model requires a bounded CPU operator partition", - "detail": "required operators: Concat, Gather, Slice", - }, - } - profiles[f"{fixture}:allow"] = { - "files": [f"{fixture}-allow.json"], - "fileSha256": {f"{fixture}-allow.json": "2" * 64}, - "nodeCounts": {"WebGpuExecutionProvider": 10}, - "operators": {}, - } - canary = qualify.DEFAULT_FIXTURES[0] - auto = node_case("auto", ["WebGpuExecutionProvider", "CPUExecutionProvider"]) - auto["host"] = {"platform": "test", "architecture": "x64"} - auto["engine"]["execution"]["selectionTrace"] = { - "orderedCandidates": ["webgpu", "cpu"], - "selectedProvider": "webgpu", - } - cases[f"{canary}:auto"] = auto - lifecycle = node_case("allow", ["WebGpuExecutionProvider", "CPUExecutionProvider"]) - lifecycle["warmup"] = 0 - lifecycle["iterations"] = 1 - lifecycle["cycles"] = 20 - cases[f"{canary}:lifecycle"] = lifecycle - cases["native-cpp:auto"] = { - "ok": True, - "engineInitializationUs": 1000, - "firstPredictionUs": 2000, - "warmup": 1, - "iterations": 10, - "memoryBytes": {"peakResident": 120 * 1024 * 1024}, - "execution": { - "requestedProvider": "auto", - "selectionTrace": { - "orderedCandidates": ["webgpu", "cpu"], - "selectedProvider": "webgpu", - }, - "detection": { - "actualProviderChain": [ - "WebGpuExecutionProvider", - "CPUExecutionProvider", - ] - }, - "recognition": { - "actualProviderChain": [ - "WebGpuExecutionProvider", - "CPUExecutionProvider", - ] - }, - }, - } - for key in (f"{canary}:auto", f"{canary}:lifecycle", "native-cpp:auto"): - profiles[key] = { - "files": [key.replace(":", "-") + ".json"], - "fileSha256": {key.replace(":", "-") + ".json": "4" * 64}, - "nodeCounts": {"WebGpuExecutionProvider": 10}, - "operators": {}, - } - return cases, profiles - - -def sdk_manifest(lock: dict[str, object], platform_id: str) -> dict[str, object]: - records = [] - for spec in build_runtime.artifact_plan(lock, platform_id): - records.append( - { - "path": spec["outputPath"], - "bytes": 17, - "sha256": hashlib.sha256(spec["outputPath"].encode()).hexdigest(), - "role": spec["role"], - "sourcePackage": spec["package"], - "sourcePath": spec["sourcePath"], - } - ) - platform = lock["platforms"][platform_id] - platform_identity = { - "id": platform_id, - "operatingSystem": platform["operatingSystem"], - "architecture": platform["architecture"], - } - if "libc" in platform: - platform_identity["libc"] = platform["libc"] - runtime_paths = [record["outputPath"] for record in platform["runtimeFiles"]] - provider_path = next( - record["outputPath"] - for record in platform["runtimeFiles"] - if record["role"] == "webgpu-plugin" - ) - package_records = [] - for name in ("onnxruntime", "webgpu"): - package = lock["packages"][name] - package_records.append( - { - "name": name, - **{ - key: package[key] - for key in ( - "id", - "version", - "source", - "catalog", - "bytes", - "sha512", - ) - }, - } - ) - return { - "schemaVersion": 2, - "contractId": lock["contractId"], - "platform": platform_identity, - "runtime": build_runtime.runtime_identity(lock, platform_id), - "artifacts": { - "linkLibrary": platform["linkLibrary"]["outputPath"], - "providerLibrary": provider_path, - "runtimeFiles": runtime_paths, - "files": sorted(records, key=lambda value: value["path"]), - "artifactSetSha256": build_runtime.artifact_set_digest(records), - }, - "headers": { - "directory": "include", - "onnxruntimeVersion": "1.24.4", - "files": [record for record in records if record["role"] == "header"], - }, - "packages": package_records, - "sessionOptions": copy.deepcopy(lock["sessionOptions"]), - "qualification": copy.deepcopy(lock["qualification"]), - } - - -def runtime_descriptor( - manifest: dict[str, object], platform_id: str -) -> dict[str, object]: - records = {record["path"]: record for record in manifest["artifacts"]["files"]} - runtime_records = [ - { - "path": f"native/{PurePosixPath(relative).name}", - "bytes": records[relative]["bytes"], - "sha256": records[relative]["sha256"], - } - for relative in manifest["artifacts"]["runtimeFiles"] - ] - addon = {"path": "native/light_ocr_node.node", "bytes": 23, "sha256": "5" * 64} - platform = { - "id": platform_id, - "os": "linux" if platform_id == "linux-x64" else "win32", - "architecture": "x86_64", - } - if platform_id == "linux-x64": - platform["libc"] = "glibc" - return { - "schemaVersion": "2.0", - "platform": platform, - "runtime": { - "flavor": "webgpu", - "kind": "onnxruntime-plugin-webgpu", - "version": "1.24.4", - "abi": "onnxruntime-c-api-24-plugin-ep-0.1", - "artifacts": runtime_records, - }, - "qualificationOnly": True, - "released": False, - "autoPolicy": { - "id": f"{platform_id}-v1", - "version": 1, - "providers": ["webgpu", "cpu"], - }, - "providers": { - "webgpu": { - "runtimeProvider": "WebGpuExecutionProvider", - "providerVersion": "0.1.0", - "qualificationId": manifest["qualification"]["evidenceId"], - "providerLibrary": runtime_records[1], - "artifacts": runtime_records[1:], - }, - "cpu": { - "runtimeProvider": "CPUExecutionProvider", - "qualificationId": "cpu-baseline-v1", - "artifacts": [runtime_records[0]], - }, - }, - "addon": addon, - } - - -def write_json(path: Path, value: object) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", "utf-8") - - -def write_platform_report( - root: Path, platform_id: str, lock: dict[str, object] -) -> None: - directory = root / platform_id - artifacts = directory / "artifacts" - artifacts.mkdir(parents=True) - manifest = sdk_manifest(lock, platform_id) - manifest_path = artifacts / "sdk-artifact-manifest.json" - write_json(manifest_path, manifest) - descriptor = runtime_descriptor(manifest, platform_id) - descriptor_path = artifacts / "native-runtime-descriptor.json" - write_json(descriptor_path, descriptor) - - sdk = directory / "sdk" - native = directory / "native-package" / "native" - sdk.mkdir() - native.mkdir(parents=True) - write_json(sdk / "artifact-manifest.json", manifest) - write_json(native / "runtime-descriptor.json", descriptor) - payload_bytes = sum( - record["bytes"] - for record in [descriptor["addon"], *descriptor["runtime"]["artifacts"]] - ) - with (native / "payload.bin").open("wb") as stream: - stream.truncate(payload_bytes) - cases, profiles = cases_and_profiles() - graphics = { - "source": "synthetic", - "adapters": [{"driver": "test", "driverVersion": "1.0"}], - } - report = qualify.collect_evidence( - platform_id=platform_id, - sdk=sdk, - native=directory / "native-package", - cases=cases, - profiles=profiles, - graphics=graphics, - rebuilt_from_source=True, - ) - report_path = directory / "qualification-report.json" - write_json(report_path, report) - (directory / "qualification-report.sha256").write_text( - f"{review_reports.sha256(report_path)} qualification-report.json\n", - "utf-8", - ) +from tests.python.webgpu_report_fixtures import ( + create_report_pair, + pending_lock, + write_json, +) +from tools.webgpu import review_reports class WebGpuReportReviewTest(unittest.TestCase): def setUp(self) -> None: self.revision = review_reports.current_revision() - def create_pair(self, root: Path) -> Path: - lock = pending_lock() - lock_path = root / "runtime-lock.json" - write_json(lock_path, lock) - for platform_id in review_reports.PLATFORMS: - write_platform_report(root, platform_id, lock) - return lock_path - def test_collects_intact_pair_as_manual_review_candidate(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - lock_path = self.create_pair(root) + lock_path = create_report_pair(root) candidate = review_reports.collect_pair( root, expected_revision=self.revision, lock_path=lock_path ) @@ -369,7 +34,7 @@ def test_collects_intact_pair_as_manual_review_candidate(self) -> None: def test_rejects_report_changed_without_sidecar_update(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - lock_path = self.create_pair(root) + lock_path = create_report_pair(root) report_path = root / "linux-x64" / "qualification-report.json" report = json.loads(report_path.read_text("utf-8")) report["passed"] = False @@ -382,7 +47,7 @@ def test_rejects_report_changed_without_sidecar_update(self) -> None: def test_rejects_rehashed_report_with_missing_gate(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - lock_path = self.create_pair(root) + lock_path = create_report_pair(root) report_path = root / "linux-x64" / "qualification-report.json" report = json.loads(report_path.read_text("utf-8")) report["gates"].pop() @@ -399,7 +64,7 @@ def test_rejects_rehashed_report_with_missing_gate(self) -> None: def test_rejects_cross_revision_report_pair(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - lock_path = self.create_pair(root) + lock_path = create_report_pair(root) with self.assertRaisesRegex(RuntimeError, "report identity"): review_reports.collect_pair( root, expected_revision="a" * 40, lock_path=lock_path @@ -408,7 +73,7 @@ def test_rejects_cross_revision_report_pair(self) -> None: def test_collects_staggered_platform_revisions_without_override(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - lock_path = self.create_pair(root) + lock_path = create_report_pair(root) report_path = root / "windows-x64" / "qualification-report.json" report = json.loads(report_path.read_text("utf-8")) report["sourceRevision"] = "b" * 40 @@ -426,7 +91,7 @@ def test_collects_staggered_platform_revisions_without_override(self) -> None: def test_rejects_tampered_copied_descriptor(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - lock_path = self.create_pair(root) + lock_path = create_report_pair(root) descriptor_path = ( root / "windows-x64" / "artifacts" / "native-runtime-descriptor.json" ) @@ -441,7 +106,7 @@ def test_rejects_tampered_copied_descriptor(self) -> None: def test_production_lock_must_bind_the_reviewed_pair(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - self.create_pair(root) + create_report_pair(root) lock = pending_lock() qualification = lock["qualification"] qualification["status"] = "production-qualified" diff --git a/tests/python/test_webgpu_runtime.py b/tests/python/test_webgpu_runtime.py index 2b6b13c..127e248 100644 --- a/tests/python/test_webgpu_runtime.py +++ b/tests/python/test_webgpu_runtime.py @@ -1,96 +1,34 @@ from __future__ import annotations import copy -import hashlib -import importlib.util import json import os from pathlib import Path +import shutil +import subprocess import tempfile import unittest import zipfile +from tests.python.webgpu_runtime_fixtures import ( + create_fake_packages, + locked, + package_identity, + pending_lock, +) +from tools.webgpu import build_runtime ROOT = Path(__file__).resolve().parents[2] -MODULE_PATH = ROOT / "tools" / "webgpu" / "build_runtime.py" -SPEC = importlib.util.spec_from_file_location("webgpu_build_runtime", MODULE_PATH) -assert SPEC is not None and SPEC.loader is not None -build_runtime = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(build_runtime) - - -def locked() -> dict[str, object]: - return json.loads( - (ROOT / "tools" / "webgpu" / "runtime-lock.json").read_text("utf-8") - ) - - -def pending_locked() -> dict[str, object]: - lock = locked() - qualification = lock["qualification"] - qualification["status"] = "development-pending-device-validation" - qualification["providerGatePassed"] = False - qualification["productionArtifactQualified"] = False - qualification["qualifiedArtifactSetSha256"] = { - "linux-x64": None, - "windows-x64": None, - } - qualification["qualificationReportSha256"] = { - "linux-x64": None, - "windows-x64": None, - } - return lock - - -def package_members(lock: dict[str, object], package_name: str) -> set[str]: - members: set[str] = set() - for platform_id in ("linux-x64", "windows-x64"): - for spec in build_runtime.artifact_plan(lock, platform_id): - if spec["package"] == package_name: - members.add(spec["sourcePath"]) - return members - - -def create_fake_packages( - root: Path, - lock: dict[str, object], - *, - omit: tuple[str, str] | None = None, -) -> dict[str, Path]: - paths: dict[str, Path] = {} - for package_name in ("onnxruntime", "webgpu"): - path = root / f"{package_name}.nupkg" - with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive: - for member in sorted(package_members(lock, package_name)): - if omit == (package_name, member): - continue - archive.writestr(member, f"{package_name}:{member}\n".encode()) - paths[package_name] = path - return paths - - -def package_identity(path: Path, name: str) -> dict[str, object]: - data = path.read_bytes() - return { - "id": name, - "filename": path.name, - "source": "https://example.invalid/package.nupkg", - "bytes": len(data), - "sha512": hashlib.sha512(data).hexdigest(), - } class WebGpuRuntimeContractTest(unittest.TestCase): - def test_committed_lock_is_valid(self) -> None: - build_runtime.validate_lock(locked()) - def test_complete_production_qualification_state_is_valid(self) -> None: lock = locked() build_runtime.validate_lock(lock) self.assertEqual(lock["qualification"]["status"], "production-qualified") def test_complete_pending_qualification_state_is_valid(self) -> None: - build_runtime.validate_lock(pending_locked()) + build_runtime.validate_lock(pending_lock()) def test_production_qualification_requires_both_platform_reports(self) -> None: lock = locked() @@ -395,28 +333,49 @@ def test_archive_member_rejects_paths_outside_the_archive_contract(self) -> None ): build_runtime.archive_member(archive, "../safe", "test") - def test_cmake_freezes_plugin_runtime_release_boundary(self) -> None: - sources = { - name: (ROOT / "cmake" / name).read_text("utf-8") - for name in ("Dependencies.cmake", "WebGpuRuntime.cmake") - } - dependencies = "\n".join(sources.values()) - required = [ - "LIGHT_OCR_ONNXRUNTIME_FLAVOR", - "LIGHT_OCR_WEBGPU_SDK_DIR", - "LIGHT_OCR_WEBGPU_QUALIFICATION_BUILD", - "native-webgpu-plugin-0.1.0-ort-1.24.4-v1", - "onnxruntime-plugin-webgpu", - "productionArtifactQualified", - "providerGatePassed", - "LIGHT_OCR_HAS_WEBGPU=1", - ] - for token in required: - with self.subTest(token=token): - self.assertIn(token, dependencies) - webgpu_runtime = sources["WebGpuRuntime.cmake"] - self.assertIn("_qualification_hash_length EQUAL 64", webgpu_runtime) - self.assertNotIn("[0-9a-f]{64}", webgpu_runtime) + def test_cmake_rejects_a_malformed_production_qualification_hash(self) -> None: + cmake = shutil.which("cmake") + if cmake is None: + self.skipTest("cmake is unavailable") + platform_id = "windows-x64" if os.name == "nt" else "linux-x64" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + lock = locked() + sdk = root / "sdk" + manifest_path = build_runtime.stage_runtime( + lock, platform_id, create_fake_packages(root, lock), sdk + ) + manifest = json.loads(manifest_path.read_text("utf-8")) + qualification = manifest["qualification"] + qualification["qualifiedArtifactSetSha256"][platform_id] = manifest[ + "artifacts" + ]["artifactSetSha256"] + manifest_path.write_text(json.dumps(manifest), "utf-8") + + command = [ + cmake, + "-DCMAKE_SIZEOF_VOID_P=8", + f"-DCMAKE_SYSTEM_NAME={'Windows' if os.name == 'nt' else 'Linux'}", + "-DCMAKE_SYSTEM_PROCESSOR=x86_64", + "-DLIGHT_OCR_WEBGPU_VALIDATE_ONLY=ON", + f"-DLIGHT_OCR_WEBGPU_SDK_DIR={sdk}", + "-DLIGHT_OCR_WEBGPU_QUALIFICATION_BUILD=OFF", + ] + if os.name != "nt": + command.append("-DLIGHT_OCR_TARGET_LIBC=glibc") + command.extend(["-P", str(ROOT / "cmake" / "WebGpuRuntime.cmake")]) + + accepted = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(accepted.returncode, 0, accepted.stderr) + + qualification["qualificationReportSha256"][platform_id] = "g" * 64 + manifest_path.write_text(json.dumps(manifest), "utf-8") + rejected = subprocess.run(command, capture_output=True, text=True) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn( + "requires accepted Linux and Windows Provider Gates", + rejected.stdout + rejected.stderr, + ) if __name__ == "__main__": diff --git a/tests/python/webgpu_report_fixtures.py b/tests/python/webgpu_report_fixtures.py new file mode 100644 index 0000000..c9e7b71 --- /dev/null +++ b/tests/python/webgpu_report_fixtures.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import argparse +from contextlib import redirect_stdout +import io +import json +from pathlib import Path +from unittest import mock + +from tests.python.npm_release_fixtures import release_metadata +from tests.python.webgpu_runtime_fixtures import create_fake_packages, pending_lock +from tools import npm_release +from tools.webgpu import build_runtime, qualify, review_reports + + +WEBGPU_CHAIN = ["WebGpuExecutionProvider", "CPUExecutionProvider"] +LIFECYCLE_RSS_MIB = ( + 270, 288, 322, 352, 374, 391, 406, 413, 429, 406, + 414, 410, 423, 419, 442, 419, 450, 445, 448, 427, + 455, 451, 464, 460, 458, 432, 440, 428, 435, 422, + 430, 425, 428, 432, 430, 428, 429, 431, 430, 432, +) + + +def ocr_line( + *, text: str = "HELLO 123", confidence: float = 0.95 +) -> dict[str, object]: + return { + "text": text, + "confidence": confidence, + "box": [[1.0, 2.0], [20.0, 2.0], [20.0, 12.0], [1.0, 12.0]], + } + + +def _node_case( + mode: str, + chain: list[str], + *, + precision: str, + retained_growth: int, +) -> dict[str, object]: + cpu = mode == "cpu" + return { + "schemaVersion": "1.1", + "ok": True, + "result": { + "lines": [ocr_line()], + "deterministic": True, + "sha256": "1" * 64, + }, + "engine": { + "executionProvider": ( + "CPUExecutionProvider" if cpu else "WebGpuExecutionProvider" + ), + "execution": { + "sessions": { + stage: {"actualProviderChain": chain, "precision": precision} + for stage in ("detection", "recognition") + } + }, + }, + "latencyUs": { + "minimum": 120 if cpu else 80, + "p50": 160 if cpu else 100, + "p95": 160 if cpu else 120, + "maximum": 180 if cpu else 140, + }, + "warmup": 2, + "iterations": 10, + "cycles": 3, + "engineInitializationUs": { + "minimum": 1000, + "p50": 1000, + "maximum": 1000, + "values": [1000, 1000, 1000], + }, + "firstPredictionUs": 2000, + "firstPredictionUsByCycle": [2000, 2000, 2000], + "lifecycle": { + "residentMinimumBytes": 100 * 1024 * 1024, + "residentMaximumBytes": 110 * 1024 * 1024, + "retainedGrowthBytes": retained_growth, + }, + } + + +def qualification_case( + mode: str, chain: list[str], *, lifecycle: bool = False +) -> dict[str, object]: + return _node_case( + mode, + chain, + precision="fp16" if mode == "allow" else "fp32", + retained_growth=1024 if lifecycle else 0, + ) + + +def review_node_case(mode: str, chain: list[str]) -> dict[str, object]: + value = _node_case(mode, chain, precision="fp32", retained_growth=1024) + value.update( + {"processCpuUs": 1000, "measuredWallUs": 2000, "averageProcessCpuCores": 0.5} + ) + return value + + +def qualification_layout(root: Path) -> tuple[Path, Path]: + sdk = root / "sdk" + native = root / "native-package" + sdk.mkdir() + (native / "native").mkdir(parents=True) + write_json( + sdk / "artifact-manifest.json", + { + "contractId": "native-webgpu-plugin-0.1.0-ort-1.24.4-v1", + "artifacts": {"artifactSetSha256": "2" * 64}, + "runtime": {"kind": "onnxruntime-plugin-webgpu"}, + "qualification": {"evidenceId": "synthetic-evidence"}, + }, + ) + write_json( + native / "native" / "runtime-descriptor.json", + {"runtime": {"kind": "onnxruntime-plugin-webgpu"}}, + ) + return sdk, native + + +def synthetic_gate_inputs() -> tuple[dict[str, dict], dict[str, dict]]: + cpu = qualification_case("cpu", ["CPUExecutionProvider"]) + allow = qualification_case("allow", WEBGPU_CHAIN, lifecycle=True) + auto = qualification_case("auto", WEBGPU_CHAIN) + auto["host"] = {"platform": "linux", "architecture": "x64"} + auto["engine"]["execution"]["selectionTrace"] = { + "orderedCandidates": ["webgpu", "cpu"], + "selectedProvider": "webgpu", + } + cases = { + "generated-hello-123:cpu": cpu, + "generated-hello-123:fp32": qualification_case("fp32", WEBGPU_CHAIN), + "generated-hello-123:allow": allow, + "generated-hello-123:strict": { + "schemaVersion": "1.1", + "ok": True, + "expectedRejection": True, + "error": { + "code": "unsupported_capability", + "message": "The WebGPU model requires a bounded CPU operator partition", + "detail": "required operators: Concat, Gather, Slice", + }, + }, + "generated-hello-123:auto": auto, + "generated-hello-123:lifecycle": qualification_case( + "allow", WEBGPU_CHAIN, lifecycle=True + ), + "native-cpp:auto": native_auto_case(), + } + cases["generated-hello-123:lifecycle"]["cycles"] = 20 + profiles = { + key: { + "files": [f"{label}.json"], + "nodeCounts": {"WebGpuExecutionProvider": 10}, + } + for key, label in ( + ("generated-hello-123:fp32", "fp32"), + ("generated-hello-123:allow", "allow"), + ("generated-hello-123:auto", "auto"), + ("native-cpp:auto", "native-cpp-auto"), + ("generated-hello-123:lifecycle", "lifecycle"), + ) + } + return cases, profiles + + +def native_auto_case() -> dict[str, object]: + sessions = { + stage: {"actualProviderChain": WEBGPU_CHAIN} + for stage in ("detection", "recognition") + } + return { + "ok": True, + "engineInitializationUs": 1000, + "firstPredictionUs": 2000, + "memoryBytes": {"peakResident": 120 * 1024 * 1024}, + "execution": { + "requestedProvider": "auto", + "selectionTrace": { + "orderedCandidates": ["webgpu", "cpu"], + "selectedProvider": "webgpu", + }, + **sessions, + }, + "warmup": 1, + "iterations": 10, + } + + +def review_cases_and_profiles() -> tuple[dict[str, dict], dict[str, dict]]: + cases: dict[str, dict] = {} + profiles: dict[str, dict] = {} + for fixture in qualify.DEFAULT_FIXTURES: + cases[f"{fixture}:cpu"] = review_node_case("cpu", ["CPUExecutionProvider"]) + cases[f"{fixture}:allow"] = review_node_case("allow", WEBGPU_CHAIN) + cases[f"{fixture}:strict"] = { + "schemaVersion": "1.1", + "ok": True, + "expectedRejection": True, + "error": { + "code": "unsupported_capability", + "message": "The WebGPU model requires a bounded CPU operator partition", + "detail": "required operators: Concat, Gather, Slice", + }, + } + profiles[f"{fixture}:allow"] = profile(f"{fixture}-allow", "2") + canary = qualify.DEFAULT_FIXTURES[0] + auto = review_node_case("auto", WEBGPU_CHAIN) + auto["host"] = {"platform": "test", "architecture": "x64"} + auto["engine"]["execution"]["selectionTrace"] = { + "orderedCandidates": ["webgpu", "cpu"], + "selectedProvider": "webgpu", + } + cases[f"{canary}:auto"] = auto + lifecycle = review_node_case("allow", WEBGPU_CHAIN) + lifecycle.update({"warmup": 0, "iterations": 1, "cycles": 20}) + cases[f"{canary}:lifecycle"] = lifecycle + cases["native-cpp:auto"] = native_auto_case() + for key in (f"{canary}:auto", f"{canary}:lifecycle", "native-cpp:auto"): + profiles[key] = profile(key.replace(":", "-"), "4") + return cases, profiles + + +def profile(name: str, digest_character: str) -> dict[str, object]: + filename = f"{name}.json" + return { + "files": [filename], + "fileSha256": {filename: digest_character * 64}, + "nodeCounts": {"WebGpuExecutionProvider": 10}, + "operators": {}, + } + + +def write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", "utf-8") + + +def write_platform_report( + root: Path, platform_id: str, lock: dict[str, object] +) -> None: + directory = root / platform_id + artifacts = directory / "artifacts" + artifacts.mkdir(parents=True) + packages = create_fake_packages(directory, lock) + sdk = directory / "sdk" + manifest_path = build_runtime.stage_runtime(lock, platform_id, packages, sdk) + manifest = build_runtime.validate_sdk(sdk, lock) + (artifacts / "sdk-artifact-manifest.json").write_bytes(manifest_path.read_bytes()) + + build_dir = directory / "build" + binaries = build_dir / "bin" + binaries.mkdir(parents=True) + (binaries / "light_ocr_node.node").write_bytes(b"addon") + native_package = directory / "native-package" + arguments = argparse.Namespace( + platform_id=platform_id, + build_dir=build_dir, + metadata_dir=release_metadata(directory), + output_dir=native_package, + runtime_flavor="webgpu", + webgpu_artifact_manifest=manifest_path, + qualification_build=True, + ) + with mock.patch( + "tools.npm_release.webgpu_runtime.load_lock", return_value=lock + ), redirect_stdout(io.StringIO()): + npm_release.stage_native(arguments) + descriptor_path = native_package / "native" / "runtime-descriptor.json" + (artifacts / "native-runtime-descriptor.json").write_bytes( + descriptor_path.read_bytes() + ) + + cases, profiles = review_cases_and_profiles() + report = qualify.collect_evidence( + platform_id=platform_id, + sdk=sdk, + native=native_package, + cases=cases, + profiles=profiles, + graphics={ + "source": "synthetic", + "adapters": [{"driver": "test", "driverVersion": "1.0"}], + }, + rebuilt_from_source=True, + ) + report_path = directory / "qualification-report.json" + write_json(report_path, report) + (directory / "qualification-report.sha256").write_text( + f"{review_reports.sha256(report_path)} qualification-report.json\n", + "utf-8", + ) + + +def create_report_pair(root: Path) -> Path: + lock = pending_lock() + lock_path = root / "runtime-lock.json" + write_json(lock_path, lock) + for platform_id in review_reports.PLATFORMS: + write_platform_report(root, platform_id, lock) + return lock_path diff --git a/tests/python/webgpu_runtime_fixtures.py b/tests/python/webgpu_runtime_fixtures.py new file mode 100644 index 0000000..a0cf2b6 --- /dev/null +++ b/tests/python/webgpu_runtime_fixtures.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import zipfile + +from tools.webgpu import build_runtime + + +ROOT = Path(__file__).resolve().parents[2] + + +def locked() -> dict[str, object]: + return json.loads( + (ROOT / "tools" / "webgpu" / "runtime-lock.json").read_text("utf-8") + ) + + +def pending_lock() -> dict[str, object]: + lock = locked() + qualification = lock["qualification"] + qualification["status"] = "development-pending-device-validation" + qualification["providerGatePassed"] = False + qualification["productionArtifactQualified"] = False + qualification["qualifiedArtifactSetSha256"] = { + "linux-x64": None, + "windows-x64": None, + } + qualification["qualificationReportSha256"] = { + "linux-x64": None, + "windows-x64": None, + } + return lock + + +def package_members(lock: dict[str, object], package_name: str) -> set[str]: + return { + spec["sourcePath"] + for platform_id in ("linux-x64", "windows-x64") + for spec in build_runtime.artifact_plan(lock, platform_id) + if spec["package"] == package_name + } + + +def create_fake_packages( + root: Path, + lock: dict[str, object], + *, + omit: tuple[str, str] | None = None, +) -> dict[str, Path]: + paths: dict[str, Path] = {} + for package_name in ("onnxruntime", "webgpu"): + path = root / f"{package_name}.nupkg" + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for member in sorted(package_members(lock, package_name)): + if omit != (package_name, member): + archive.writestr(member, f"{package_name}:{member}\n".encode()) + paths[package_name] = path + return paths + + +def package_identity(path: Path, name: str) -> dict[str, object]: + data = path.read_bytes() + return { + "id": name, + "filename": path.name, + "source": "https://example.invalid/package.nupkg", + "bytes": len(data), + "sha512": hashlib.sha512(data).hexdigest(), + } diff --git a/tools/npm_release.py b/tools/npm_release.py index 088225e..91c6591 100644 --- a/tools/npm_release.py +++ b/tools/npm_release.py @@ -757,8 +757,6 @@ def assemble(arguments: argparse.Namespace) -> None: ): raise RuntimeError("version must be a plain stable SemVer value") version = arguments.version - if tuple(int(part) for part in version.split(".")) < (0, 2, 0): - raise RuntimeError("tiled-v1 packages require version 0.2.0 or newer") if version != SOURCE_VERSION: raise RuntimeError( f"release version {version} does not match source version {SOURCE_VERSION}" diff --git a/tools/webgpu/README.md b/tools/webgpu/README.md index b033f39..6f24ee7 100644 --- a/tools/webgpu/README.md +++ b/tools/webgpu/README.md @@ -118,6 +118,7 @@ native superset bundle stays reproducible; this does not expose WebGPU FP16. python3 -m unittest \ tests.python.test_webgpu_runtime \ tests.python.test_npm_release \ + tests.python.test_npm_webgpu_release \ tests.python.test_webgpu_qualification ``` diff --git a/tools/webgpu/package_bundle.py b/tools/webgpu/package_bundle.py index 4eef47d..ae14cdf 100644 --- a/tools/webgpu/package_bundle.py +++ b/tools/webgpu/package_bundle.py @@ -70,8 +70,9 @@ def locked_artifact() -> tuple[dict[str, object], dict[str, object]]: return bundle, bundle["providerArtifacts"]["webgpuFp16"] -def package_bundle(base: Path, output: Path) -> None: - bundle_lock, artifact = locked_artifact() +def validate_locked_artifact( + bundle_lock: dict[str, object], artifact: dict[str, object] +) -> tuple[bytes, dict[str, bytes]]: derived = ROOT / str(artifact["directory"]) provenance_bytes = verify_file( derived / str(artifact["provenance"]["path"]), @@ -92,14 +93,40 @@ def package_bundle(base: Path, output: Path) -> None: ): raise RuntimeError("WebGPU FP16 provenance contract differs from its lock") - model_bytes = { - kind: verify_file( + source_artifacts = { + record["name"]: record for record in bundle_lock["artifacts"] + } + provenance_models = provenance.get("models") + if not isinstance(provenance_models, dict): + raise RuntimeError("WebGPU FP16 provenance model inventory is invalid") + model_bytes: dict[str, bytes] = {} + for kind, short in (("detection", "det"), ("recognition", "rec")): + model_bytes[kind] = verify_file( derived / str(artifact[kind]["path"]), artifact[kind], f"WebGPU FP16 {kind} model", ) - for kind in ("detection", "recognition") - } + expected_source = source_artifacts[kind]["members"]["inference.onnx"] + provenance_model = provenance_models.get(kind) + if ( + not isinstance(provenance_model, dict) + or provenance_model.get("output") != artifact[kind] + or provenance_model.get("source") != { + "path": f"{short}/inference.onnx", + "bytes": expected_source["bytes"], + "sha256": expected_source["sha256"], + } + ): + raise RuntimeError( + f"WebGPU FP16 {kind} provenance differs from the locked source and output" + ) + return provenance_bytes, model_bytes + + +def package_bundle(base: Path, output: Path) -> None: + bundle_lock, artifact = locked_artifact() + provenance_bytes, model_bytes = validate_locked_artifact(bundle_lock, artifact) + provenance = json.loads(provenance_bytes) base_manifest = json.loads((base / "manifest.json").read_text("utf-8")) if ( base_manifest.get("bundleId") != bundle_lock["bundleId"]