diff --git a/.dockerignore b/.dockerignore index 86e2a9c..bd02464 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,9 @@ LivingCity/*.csv LivingCity/berkeley_2018/* .git +.pytest_cache +.ruff_cache +__pycache__ +build +build-* Makefile diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 524147d..3679264 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,6 +29,8 @@ jobs: make -j$(nproc) 2>&1 | tee build.log if [ -f lpsim ]; then echo "Full build successful" + python3 ../tools/check_cuda_artifacts.py lpsim \ + --require-real 80 --require-virtual 80 elif grep -q "Linking CXX" build.log; then echo "Compilation passed (link may fail without GPU driver stub)" else @@ -39,6 +41,7 @@ jobs: - name: Python tools check run: | + python3 tests/test_cuda_artifacts.py python3 tools/generate_demand.py --network data/networks/sf_bay_area \ --num-trips 100 --model uniform --seed 1 --output /dev/null python3 tools/partition_network.py --network data/networks/sf_bay_area \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 766fa9c..d077edd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,11 +1,52 @@ cmake_minimum_required(VERSION 3.20) + +# Remember whether the caller supplied CMake's standard CUDA architecture +# setting. project() asks nvcc for a host-specific default, which we replace +# below only when the caller did not make an explicit choice. +if(DEFINED CMAKE_CUDA_ARCHITECTURES OR + (DEFINED ENV{CUDAARCHS} AND NOT "$ENV{CUDAARCHS}" STREQUAL "")) + set(LPSIM_CUDA_ARCHITECTURES_EXPLICIT TRUE) +else() + set(LPSIM_CUDA_ARCHITECTURES_EXPLICIT FALSE) +endif() + project(LPSim LANGUAGES CXX CUDA) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CUDA_STANDARD 17) -# GPU architecture (A100=80, H100/H200=90, L40=89) -set(CMAKE_CUDA_ARCHITECTURES "80;89;90" CACHE STRING "CUDA architectures") +# Optional project-specific override. The standard +# -DCMAKE_CUDA_ARCHITECTURES option remains fully supported. +set(LPSIM_CUDA_ARCHITECTURES "" CACHE STRING + "Override LPSim CUDA architectures (for example: 90-real;90-virtual)") + +if(LPSIM_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES "${LPSIM_CUDA_ARCHITECTURES}" CACHE STRING + "CUDA architectures" FORCE) + set(LPSIM_CUDA_ARCHITECTURE_PROFILE "custom") +elseif(NOT LPSIM_CUDA_ARCHITECTURES_EXPLICIT) + if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.8) + # Native cubins: A100, L40, H100/H200, B100/B200. + # PTX: Ampere baseline plus Blackwell for current and future drivers. + set(CMAKE_CUDA_ARCHITECTURES + "80-real;89-real;90-real;100-real;80-virtual;100-virtual" + CACHE STRING "CUDA architectures" FORCE) + set(LPSIM_CUDA_ARCHITECTURE_PROFILE "portable-native") + else() + # CUDA < 12.8 cannot emit an sm_100 cubin. Keep PTX so a sufficiently + # new NVIDIA driver can JIT the kernels on Blackwell GPUs. + set(CMAKE_CUDA_ARCHITECTURES + "80-real;89-real;90-real;80-virtual;90-virtual" + CACHE STRING "CUDA architectures" FORCE) + set(LPSIM_CUDA_ARCHITECTURE_PROFILE "portable-jit") + message(WARNING + "CUDA ${CMAKE_CUDA_COMPILER_VERSION} cannot emit native Blackwell " + "(sm_100) code; B100/B200 will use the embedded PTX fallback. " + "Use CUDA 12.8 or newer for a native Blackwell cubin.") + endif() +else() + set(LPSIM_CUDA_ARCHITECTURE_PROFILE "custom") +endif() find_package(CUDAToolkit REQUIRED) find_package(OpenCV QUIET) @@ -52,6 +93,13 @@ add_executable(lpsim ${SIMULATOR_CUDA} ) +string(REPLACE ";" "," LPSIM_CUDA_ARCHITECTURES_TEXT + "${CMAKE_CUDA_ARCHITECTURES}") +target_compile_definitions(lpsim PRIVATE + LPSIM_CUDA_ARCHITECTURES="${LPSIM_CUDA_ARCHITECTURES_TEXT}" + LPSIM_CUDA_COMPILER_VERSION="${CMAKE_CUDA_COMPILER_VERSION}" +) + target_link_libraries(lpsim PRIVATE CUDA::cudart CUDA::cuda_driver @@ -83,5 +131,7 @@ install(DIRECTORY data/ DESTINATION share/lpsim/data) install(DIRECTORY include/lpsim/ DESTINATION include/lpsim) # Output build info +message(STATUS "CUDA compiler: ${CMAKE_CUDA_COMPILER_VERSION}") +message(STATUS "CUDA architecture profile: ${LPSIM_CUDA_ARCHITECTURE_PROFILE}") message(STATUS "CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}") message(STATUS "Pandana library: ${CHROUTING_LIB}") diff --git a/Dockerfile b/Dockerfile index 2252191..633a973 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,42 @@ -# LPSim: GPU-accelerated multi-GPU traffic microsimulator +# LPSim: GPU-accelerated multi-GPU traffic microsimulator. +# CUDA 12.8+ is required for a native Blackwell (sm_100) cubin. # Build: docker build -t lpsim . # Run: docker run --gpus all lpsim -FROM yibo123/lpsim:cuda12.4 +ARG LPSIM_CUDA_IMAGE=nvidia/cuda:12.9.1-devel-ubuntu22.04 +FROM yibo123/lpsim:cuda12.4 AS lpsim-legacy-dependencies +FROM ${LPSIM_CUDA_IMAGE} -WORKDIR /lpsim -COPY . . +ARG LPSIM_CUDA_ARCHITECTURES="" + +# Pandana/CH routing and Boost 1.59 are not distributed by the CUDA image. Reuse +# the project's existing binary-compatible Ubuntu 22.04 dependency bundle. +COPY --from=lpsim-legacy-dependencies /usr/include/pandana /usr/include/pandana +COPY --from=lpsim-legacy-dependencies /usr/local/boost_1_59_0 /usr/local/boost_1_59_0 + +ENV LD_LIBRARY_PATH=/usr/include/pandana/src:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 RUN apt-get update -qq && \ - apt-get install -y -qq cmake python3-pip && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \ + build-essential cmake python3-pip qtbase5-dev libqt5opengl5-dev \ + libglew-dev libfontconfig1 mesa-common-dev && \ rm -rf /var/lib/apt/lists/* && \ - pip3 install --no-cache-dir -e . && \ - mkdir -p build && cd build && \ - cmake .. -DCMAKE_CUDA_ARCHITECTURES="80;89;90" -DCMAKE_CXX_FLAGS="-w" && \ - make -j$(nproc) + python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel + +WORKDIR /lpsim +COPY . . + +RUN python3 -m pip install --no-cache-dir -e . + +RUN mkdir -p build && cd build && \ + cmake .. -DLPSIM_CUDA_ARCHITECTURES="${LPSIM_CUDA_ARCHITECTURES}" \ + -DCMAKE_CXX_FLAGS="-w" && \ + make -j$(nproc) && \ + if [ -z "${LPSIM_CUDA_ARCHITECTURES}" ]; then \ + python3 ../tools/check_cuda_artifacts.py lpsim \ + --require-real 80 89 90 100 --require-virtual 80 100; \ + else \ + python3 ../tools/check_cuda_artifacts.py lpsim; \ + fi CMD ["build/lpsim"] diff --git a/README.md b/README.md index a2063bf..40a8873 100755 --- a/README.md +++ b/README.md @@ -34,6 +34,17 @@ make -j cd .. && build/lpsim ``` +### GPU architecture compatibility + +Without an explicit architecture option, LPSim builds a portable CUDA binary. +CUDA 12.8+ includes native code for A100, L40/L40S, H100/H200, and B100/B200, +plus PTX fallbacks. Older toolkits retain PTX so Blackwell GPUs can use driver +JIT compilation. A machine-specific build can still use, for example, +`-DCMAKE_CUDA_ARCHITECTURES="90-real;90-virtual"`. + +See [Portable NVIDIA GPU builds](docs/gpu-builds.md) for architecture profiles, +artifact inspection, and native/JIT validation commands. + ## Web Visualizer ```bash diff --git a/docs/gpu-builds.md b/docs/gpu-builds.md new file mode 100644 index 0000000..bbde6dc --- /dev/null +++ b/docs/gpu-builds.md @@ -0,0 +1,76 @@ +# Portable NVIDIA GPU builds + +LPSim can package native CUDA machine code (cubin) and driver-JIT-compatible +PTX in the same executable. At runtime the NVIDIA driver selects the best image +for the installed GPU. + +## Default build profiles + +When no architecture option is supplied, CMake selects a profile based on the +CUDA compiler: + +| CUDA toolkit | Native cubins | Embedded PTX | Blackwell behavior | +| --- | --- | --- | --- | +| 12.8 or newer | `sm_80`, `sm_89`, `sm_90`, `sm_100` | `compute_80`, `compute_100` | Native `sm_100` | +| Older than 12.8 | `sm_80`, `sm_89`, `sm_90` | `compute_80`, `compute_90` | Driver JIT from PTX | + +This covers A100 (8.0), L40/L40S (8.9), H100/H200 (9.0), and B100/B200 +(10.0). A recent NVIDIA driver is required to JIT older PTX on a newer GPU. + +Use the standard CMake option for a machine-specific build: + +```bash +cmake -S . -B build -DCMAKE_CUDA_ARCHITECTURES="90-real;90-virtual" +cmake --build build -j +``` + +Or use the LPSim-specific override, which is convenient in Docker build args: + +```bash +cmake -S . -B build \ + -DLPSIM_CUDA_ARCHITECTURES="80-real;89-real;90-real;100-real;80-virtual;100-virtual" +cmake --build build -j +``` + +The tracked Dockerfile uses CUDA 12.9.1 and the portable-native profile by +default. A smaller GPU-specific image can override the same setting: + +```bash +docker build -t lpsim . +docker build -t lpsim:h100 \ + --build-arg LPSIM_CUDA_ARCHITECTURES="90-real;90-virtual" . +``` + +Native `sm_100` compilation requires CUDA 12.8 or newer. Architecture-specific +targets such as `sm_100a` are intentionally not enabled because their cubins do +not run on every compute-capability 10.0 GPU. + +## Inspect a build + +The artifact checker wraps `cuobjdump` and fails when a required image is absent: + +```bash +python3 tools/check_cuda_artifacts.py build/lpsim \ + --require-real 80 89 90 100 \ + --require-virtual 80 100 +``` + +LPSim also prints the nvcc version, embedded architectures, driver API/runtime +versions, selected GPU names, compute capabilities, memory, SM counts, and P2P +availability during CUDA initialization. + +## Validate both loading paths + +Run a representative regression once with PTX disabled to prove the matching +cubin is present, and once with JIT forced to prove the fallback is usable: + +```bash +CUDA_DISABLE_PTX_JIT=1 tests/run_regression.sh build +CUDA_FORCE_PTX_JIT=1 tests/run_regression.sh build +``` + +The first run should be used for native H100 and B200 validation. The forced-JIT +run is slower on first launch because the driver compiles and caches the PTX. +Floating-point reductions and atomic update order can vary between GPU models, +so cross-GPU tests should compare invariants and tolerances rather than requiring +byte-identical output files. diff --git a/src/simulator/cuda_traffic_sim.cu b/src/simulator/cuda_traffic_sim.cu index 20fa9bb..1f582f5 100644 --- a/src/simulator/cuda_traffic_sim.cu +++ b/src/simulator/cuda_traffic_sim.cu @@ -30,6 +30,14 @@ #include #include +#ifndef LPSIM_CUDA_ARCHITECTURES +#define LPSIM_CUDA_ARCHITECTURES "unknown" +#endif + +#ifndef LPSIM_CUDA_COMPILER_VERSION +#define LPSIM_CUDA_COMPILER_VERSION "unknown" +#endif + #ifndef ushort #define ushort uint16_t #endif @@ -69,6 +77,41 @@ inline void printMemoryUsage() { double used_db = total_db - free_db; printf("GPU memory usage: used = %.0f, free = %.0f MB, total = %.0f MB\n", used_db / 1024.0 / 1024.0, free_db / 1024.0 / 1024.0, total_db / 1024.0 / 1024.0); } + +inline void printCudaEnvironment(int deviceCount, int selectedDeviceCount) { + int driverVersion = 0; + int runtimeVersion = 0; + gpuErrchk(cudaDriverGetVersion(&driverVersion)); + gpuErrchk(cudaRuntimeGetVersion(&runtimeVersion)); + + printf("[LPSim CUDA] nvcc=%s architectures=%s\n", + LPSIM_CUDA_COMPILER_VERSION, LPSIM_CUDA_ARCHITECTURES); + printf("[LPSim CUDA] driver_api=%d.%d runtime=%d.%d visible_devices=%d selected_devices=%d\n", + driverVersion / 1000, (driverVersion % 1000) / 10, + runtimeVersion / 1000, (runtimeVersion % 1000) / 10, + deviceCount, selectedDeviceCount); + + for (int device = 0; device < selectedDeviceCount; ++device) { + cudaDeviceProp properties{}; + gpuErrchk(cudaGetDeviceProperties(&properties, device)); + printf("[LPSim CUDA] device=%d name=\"%s\" cc=%d.%d memory_mib=%zu sms=%d\n", + device, properties.name, properties.major, properties.minor, + properties.totalGlobalMem / (1024 * 1024), + properties.multiProcessorCount); + } + + for (int source = 0; source < selectedDeviceCount; ++source) { + for (int target = 0; target < selectedDeviceCount; ++target) { + if (source == target) { + continue; + } + int canAccessPeer = 0; + gpuErrchk(cudaDeviceCanAccessPeer(&canAccessPeer, source, target)); + printf("[LPSim CUDA] p2p=%d->%d available=%s\n", + source, target, canAccessPeer ? "yes" : "no"); + } + } +} //////////////////////////////// // Multi-GPU device state // @@ -155,13 +198,26 @@ void b18InitCUDA_n( std::vector& numVehPerLinePerTimeInterval, float deltaTime) { ngpus = num_gpus; + if (ngpus <= 0) { + fprintf(stderr, "NUM_GPUS must be positive, got %d\n", ngpus); + exit(1); + } + int maxGpus = 0; - cudaGetDeviceCount(&maxGpus); + cudaError_t deviceCountStatus = cudaGetDeviceCount(&maxGpus); + if (deviceCountStatus != cudaSuccess) { + fprintf(stderr, "Unable to enumerate CUDA devices: %s\n", + cudaGetErrorString(deviceCountStatus)); + exit(deviceCountStatus); + } if(maxGpus=ngpus); + if (firstInitialization) { + printCudaEnvironment(maxGpus, ngpus); + } trafficVehicleVec_d_gpus = new LC::B18TrafficVehicle*[ngpus]; indexPathVec_d = new uint*[ngpus]; edgesData_d = new LC::B18EdgeData*[ngpus]; @@ -396,14 +452,19 @@ void b18InitCUDA_n( } // peer to peer { - int canAccessPeer; for (int i = 0; i < ngpus; i++) { - cudaSetDevice(i); + gpuErrchk(cudaSetDevice(i)); for (int j = 0; j < ngpus; j++) { if (i != j) { - cudaDeviceCanAccessPeer(&canAccessPeer, i, j); + int canAccessPeer = 0; + gpuErrchk(cudaDeviceCanAccessPeer(&canAccessPeer, i, j)); if (canAccessPeer) { - cudaDeviceEnablePeerAccess(j, 0); + cudaError_t peerStatus = cudaDeviceEnablePeerAccess(j, 0); + if (peerStatus == cudaErrorPeerAccessAlreadyEnabled) { + cudaGetLastError(); + } else { + gpuErrchk(peerStatus); + } printf("Peer2Peer support: %d-%d\n",i,j); } } @@ -2394,5 +2455,3 @@ void b18SimulateTrafficCUDA(float currentTime, peopleBench.stopMeasuring(); } - - diff --git a/tests/test_cuda_artifacts.py b/tests/test_cuda_artifacts.py new file mode 100644 index 0000000..4d90017 --- /dev/null +++ b/tests/test_cuda_artifacts.py @@ -0,0 +1,60 @@ +import subprocess +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tools.check_cuda_artifacts import ( # noqa: E402 + inspect_binary, + missing_architectures, + parse_architectures, +) + + +class CudaArtifactTests(unittest.TestCase): + def test_parse_architectures_from_cubin_listing(self): + output = """ + ELF file 1: lpsim.1.sm_80.cubin + ELF file 2: lpsim.2.sm_89.cubin + ELF file 3: lpsim.3.sm_100.cubin + """ + + self.assertEqual(parse_architectures(output), {80, 89, 100}) + + def test_parse_architectures_accepts_compute_targets_and_suffixes(self): + output = "arch = sm_90\n.target compute_100a\n" + + self.assertEqual(parse_architectures(output), {90, 100}) + + @patch("tools.check_cuda_artifacts.subprocess.run") + def test_inspect_binary_queries_native_and_ptx_images(self, run): + run.side_effect = [ + subprocess.CompletedProcess([], 0, "lpsim.sm_90.cubin\n", ""), + subprocess.CompletedProcess( + [], 0, "lpsim.sm_80.ptx\nlpsim.sm_100.ptx\n", "" + ), + ] + + native, ptx = inspect_binary(Path("build/lpsim"), "custom-cuobjdump") + + self.assertEqual(native, {90}) + self.assertEqual(ptx, {80, 100}) + self.assertEqual( + run.call_args_list[0].args[0], + ["custom-cuobjdump", "--list-elf", "build/lpsim"], + ) + self.assertEqual( + run.call_args_list[1].args[0], + ["custom-cuobjdump", "--list-ptx", "build/lpsim"], + ) + + def test_missing_architectures_reports_only_absent_targets(self): + self.assertEqual( + missing_architectures({80, 90}, {80, 89, 90, 100}), {89, 100} + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/check_cuda_artifacts.py b/tools/check_cuda_artifacts.py new file mode 100644 index 0000000..32e6196 --- /dev/null +++ b/tools/check_cuda_artifacts.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Verify the native cubins and PTX images embedded in an LPSim binary.""" + +from __future__ import annotations + +import argparse +import re +import subprocess +from pathlib import Path +from typing import Iterable + + +_ARCHITECTURE_PATTERN = re.compile(r"(?:sm|compute)_([0-9]{2,3})(?:[a-z])?") + + +def parse_architectures(cuobjdump_output: str) -> set[int]: + """Return CUDA architecture numbers found in cuobjdump listing output.""" + return { + int(match.group(1)) + for match in _ARCHITECTURE_PATTERN.finditer(cuobjdump_output) + } + + +def inspect_binary( + binary: Path, cuobjdump: str = "cuobjdump" +) -> tuple[set[int], set[int]]: + """Return (native cubin architectures, PTX architectures) for *binary*.""" + listings = [] + for option in ("--list-elf", "--list-ptx"): + result = subprocess.run( + [cuobjdump, option, str(binary)], + check=True, + capture_output=True, + text=True, + ) + listings.append(parse_architectures(result.stdout + result.stderr)) + return listings[0], listings[1] + + +def missing_architectures(found: Iterable[int], required: Iterable[int]) -> set[int]: + """Return required architectures that were not found.""" + return set(required) - set(found) + + +def _format_architectures(architectures: Iterable[int], prefix: str = "sm") -> str: + values = sorted(architectures) + return ", ".join(f"{prefix}_{value}" for value in values) if values else "none" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path, help="LPSim executable or CUDA object") + parser.add_argument( + "--cuobjdump", + default="cuobjdump", + help="cuobjdump executable (default: cuobjdump)", + ) + parser.add_argument( + "--require-real", + type=int, + nargs="*", + default=[], + metavar="SM", + help="required native cubin architectures, such as 80 89 90 100", + ) + parser.add_argument( + "--require-virtual", + type=int, + nargs="*", + default=[], + metavar="COMPUTE", + help="required embedded PTX architectures, such as 80 100", + ) + args = parser.parse_args() + + if not args.binary.is_file(): + parser.error(f"binary does not exist: {args.binary}") + + try: + native, ptx = inspect_binary(args.binary, args.cuobjdump) + except FileNotFoundError: + parser.error(f"cuobjdump was not found: {args.cuobjdump}") + except subprocess.CalledProcessError as error: + parser.error(f"cuobjdump failed with exit code {error.returncode}") + + print(f"Native cubins: {_format_architectures(native)}") + print(f"Embedded PTX: {_format_architectures(ptx, 'compute')}") + + missing_native = missing_architectures(native, args.require_real) + missing_ptx = missing_architectures(ptx, args.require_virtual) + if missing_native or missing_ptx: + if missing_native: + print(f"Missing native cubins: {_format_architectures(missing_native)}") + if missing_ptx: + print(f"Missing PTX: {_format_architectures(missing_ptx, 'compute')}") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())