Add portable CUDA architecture support

.dockerignore

@@ -1,4 +1,9 @@
 LivingCity/*.csv
 LivingCity/berkeley_2018/*
 .git
+.pytest_cache
+.ruff_cache
+__pycache__
+build
+build-*
 Makefile

.github/workflows/build.yml

@@ -29,6 +29,8 @@ jobs:
           make -j$(nproc) 2>&1 | tee build.log
           if [ -f lpsim ]; then
             echo "Full build successful"
+            python3 ../tools/check_cuda_artifacts.py lpsim \
+              --require-real 80 --require-virtual 80
           elif grep -q "Linking CXX" build.log; then
             echo "Compilation passed (link may fail without GPU driver stub)"
           else
@@ -39,6 +41,7 @@ jobs:
 
       - name: Python tools check
         run: |
+          python3 tests/test_cuda_artifacts.py
           python3 tools/generate_demand.py --network data/networks/sf_bay_area \
             --num-trips 100 --model uniform --seed 1 --output /dev/null
           python3 tools/partition_network.py --network data/networks/sf_bay_area \

CMakeLists.txt

@@ -1,11 +1,52 @@
 cmake_minimum_required(VERSION 3.20)
+
+# Remember whether the caller supplied CMake's standard CUDA architecture
+# setting. project() asks nvcc for a host-specific default, which we replace
+# below only when the caller did not make an explicit choice.
+if(DEFINED CMAKE_CUDA_ARCHITECTURES OR
+   (DEFINED ENV{CUDAARCHS} AND NOT "$ENV{CUDAARCHS}" STREQUAL ""))
+    set(LPSIM_CUDA_ARCHITECTURES_EXPLICIT TRUE)
+else()
+    set(LPSIM_CUDA_ARCHITECTURES_EXPLICIT FALSE)
+endif()
+
 project(LPSim LANGUAGES CXX CUDA)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CUDA_STANDARD 17)
 
-# GPU architecture (A100=80, H100/H200=90, L40=89)
-set(CMAKE_CUDA_ARCHITECTURES "80;89;90" CACHE STRING "CUDA architectures")
+# Optional project-specific override. The standard
+# -DCMAKE_CUDA_ARCHITECTURES option remains fully supported.
+set(LPSIM_CUDA_ARCHITECTURES "" CACHE STRING
+    "Override LPSim CUDA architectures (for example: 90-real;90-virtual)")
+
+if(LPSIM_CUDA_ARCHITECTURES)
+    set(CMAKE_CUDA_ARCHITECTURES "${LPSIM_CUDA_ARCHITECTURES}" CACHE STRING
+        "CUDA architectures" FORCE)
+    set(LPSIM_CUDA_ARCHITECTURE_PROFILE "custom")
+elseif(NOT LPSIM_CUDA_ARCHITECTURES_EXPLICIT)
+    if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.8)
+        # Native cubins: A100, L40, H100/H200, B100/B200.
+        # PTX: Ampere baseline plus Blackwell for current and future drivers.
+        set(CMAKE_CUDA_ARCHITECTURES
+            "80-real;89-real;90-real;100-real;80-virtual;100-virtual"
+            CACHE STRING "CUDA architectures" FORCE)
+        set(LPSIM_CUDA_ARCHITECTURE_PROFILE "portable-native")
+    else()
+        # CUDA < 12.8 cannot emit an sm_100 cubin. Keep PTX so a sufficiently
+        # new NVIDIA driver can JIT the kernels on Blackwell GPUs.
+        set(CMAKE_CUDA_ARCHITECTURES
+            "80-real;89-real;90-real;80-virtual;90-virtual"
+            CACHE STRING "CUDA architectures" FORCE)
+        set(LPSIM_CUDA_ARCHITECTURE_PROFILE "portable-jit")
+        message(WARNING
+            "CUDA ${CMAKE_CUDA_COMPILER_VERSION} cannot emit native Blackwell "
+            "(sm_100) code; B100/B200 will use the embedded PTX fallback. "
+            "Use CUDA 12.8 or newer for a native Blackwell cubin.")
+    endif()
+else()
+    set(LPSIM_CUDA_ARCHITECTURE_PROFILE "custom")
+endif()
 
 find_package(CUDAToolkit REQUIRED)
 find_package(OpenCV QUIET)
@@ -52,6 +93,13 @@ add_executable(lpsim
     ${SIMULATOR_CUDA}
 )
 
+string(REPLACE ";" "," LPSIM_CUDA_ARCHITECTURES_TEXT
+    "${CMAKE_CUDA_ARCHITECTURES}")
+target_compile_definitions(lpsim PRIVATE
+    LPSIM_CUDA_ARCHITECTURES="${LPSIM_CUDA_ARCHITECTURES_TEXT}"
+    LPSIM_CUDA_COMPILER_VERSION="${CMAKE_CUDA_COMPILER_VERSION}"
+)
+
 target_link_libraries(lpsim PRIVATE
     CUDA::cudart
     CUDA::cuda_driver
@@ -83,5 +131,7 @@ install(DIRECTORY data/ DESTINATION share/lpsim/data)
 install(DIRECTORY include/lpsim/ DESTINATION include/lpsim)
 
 # Output build info
+message(STATUS "CUDA compiler: ${CMAKE_CUDA_COMPILER_VERSION}")
+message(STATUS "CUDA architecture profile: ${LPSIM_CUDA_ARCHITECTURE_PROFILE}")
 message(STATUS "CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}")
 message(STATUS "Pandana library: ${CHROUTING_LIB}")

Dockerfile

@@ -1,18 +1,42 @@
-# LPSim: GPU-accelerated multi-GPU traffic microsimulator
+# LPSim: GPU-accelerated multi-GPU traffic microsimulator.
+# CUDA 12.8+ is required for a native Blackwell (sm_100) cubin.
 # Build: docker build -t lpsim .
 # Run:   docker run --gpus all lpsim
 
-FROM yibo123/lpsim:cuda12.4
+ARG LPSIM_CUDA_IMAGE=nvidia/cuda:12.9.1-devel-ubuntu22.04
+FROM yibo123/lpsim:cuda12.4 AS lpsim-legacy-dependencies
+FROM ${LPSIM_CUDA_IMAGE}
 
-WORKDIR /lpsim
-COPY . .
+ARG LPSIM_CUDA_ARCHITECTURES=""
+
+# Pandana/CH routing and Boost 1.59 are not distributed by the CUDA image. Reuse
+# the project's existing binary-compatible Ubuntu 22.04 dependency bundle.
+COPY --from=lpsim-legacy-dependencies /usr/include/pandana /usr/include/pandana
+COPY --from=lpsim-legacy-dependencies /usr/local/boost_1_59_0 /usr/local/boost_1_59_0
+
+ENV LD_LIBRARY_PATH=/usr/include/pandana/src:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
 
 RUN apt-get update -qq && \
-    apt-get install -y -qq cmake python3-pip && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
+        build-essential cmake python3-pip qtbase5-dev libqt5opengl5-dev \
+        libglew-dev libfontconfig1 mesa-common-dev && \
     rm -rf /var/lib/apt/lists/* && \
-    pip3 install --no-cache-dir -e . && \
-    mkdir -p build && cd build && \
-    cmake .. -DCMAKE_CUDA_ARCHITECTURES="80;89;90" -DCMAKE_CXX_FLAGS="-w" && \
-    make -j$(nproc)
+    python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel
+
+WORKDIR /lpsim
+COPY . .
+
+RUN python3 -m pip install --no-cache-dir -e .
+
+RUN mkdir -p build && cd build && \
+    cmake .. -DLPSIM_CUDA_ARCHITECTURES="${LPSIM_CUDA_ARCHITECTURES}" \
+        -DCMAKE_CXX_FLAGS="-w" && \
+    make -j$(nproc) && \
+    if [ -z "${LPSIM_CUDA_ARCHITECTURES}" ]; then \
+        python3 ../tools/check_cuda_artifacts.py lpsim \
+            --require-real 80 89 90 100 --require-virtual 80 100; \
+    else \
+        python3 ../tools/check_cuda_artifacts.py lpsim; \
+    fi
 
 CMD ["build/lpsim"]

README.md

@@ -34,6 +34,17 @@ make -j
 cd .. && build/lpsim
 ```
 
+### GPU architecture compatibility
+
+Without an explicit architecture option, LPSim builds a portable CUDA binary.
+CUDA 12.8+ includes native code for A100, L40/L40S, H100/H200, and B100/B200,
+plus PTX fallbacks. Older toolkits retain PTX so Blackwell GPUs can use driver
+JIT compilation. A machine-specific build can still use, for example,
+`-DCMAKE_CUDA_ARCHITECTURES="90-real;90-virtual"`.
+
+See [Portable NVIDIA GPU builds](docs/gpu-builds.md) for architecture profiles,
+artifact inspection, and native/JIT validation commands.
+
 ## Web Visualizer
 
 ```bash

docs/gpu-builds.md

@@ -0,0 +1,76 @@
+# Portable NVIDIA GPU builds
+
+LPSim can package native CUDA machine code (cubin) and driver-JIT-compatible
+PTX in the same executable. At runtime the NVIDIA driver selects the best image
+for the installed GPU.
+
+## Default build profiles
+
+When no architecture option is supplied, CMake selects a profile based on the
+CUDA compiler:
+
+| CUDA toolkit | Native cubins | Embedded PTX | Blackwell behavior |
+| --- | --- | --- | --- |
+| 12.8 or newer | `sm_80`, `sm_89`, `sm_90`, `sm_100` | `compute_80`, `compute_100` | Native `sm_100` |
+| Older than 12.8 | `sm_80`, `sm_89`, `sm_90` | `compute_80`, `compute_90` | Driver JIT from PTX |
+
+This covers A100 (8.0), L40/L40S (8.9), H100/H200 (9.0), and B100/B200
+(10.0). A recent NVIDIA driver is required to JIT older PTX on a newer GPU.
+
+Use the standard CMake option for a machine-specific build:
+
+```bash
+cmake -S . -B build -DCMAKE_CUDA_ARCHITECTURES="90-real;90-virtual"
+cmake --build build -j
+```
+
+Or use the LPSim-specific override, which is convenient in Docker build args:
+
+```bash
+cmake -S . -B build \
+  -DLPSIM_CUDA_ARCHITECTURES="80-real;89-real;90-real;100-real;80-virtual;100-virtual"
+cmake --build build -j
+```
+
+The tracked Dockerfile uses CUDA 12.9.1 and the portable-native profile by
+default. A smaller GPU-specific image can override the same setting:
+
+```bash
+docker build -t lpsim .
+docker build -t lpsim:h100 \
+  --build-arg LPSIM_CUDA_ARCHITECTURES="90-real;90-virtual" .
+```
+
+Native `sm_100` compilation requires CUDA 12.8 or newer. Architecture-specific
+targets such as `sm_100a` are intentionally not enabled because their cubins do
+not run on every compute-capability 10.0 GPU.
+
+## Inspect a build
+
+The artifact checker wraps `cuobjdump` and fails when a required image is absent:
+
+```bash
+python3 tools/check_cuda_artifacts.py build/lpsim \
+  --require-real 80 89 90 100 \
+  --require-virtual 80 100
+```
+
+LPSim also prints the nvcc version, embedded architectures, driver API/runtime
+versions, selected GPU names, compute capabilities, memory, SM counts, and P2P
+availability during CUDA initialization.
+
+## Validate both loading paths
+
+Run a representative regression once with PTX disabled to prove the matching
+cubin is present, and once with JIT forced to prove the fallback is usable:
+
+```bash
+CUDA_DISABLE_PTX_JIT=1 tests/run_regression.sh build
+CUDA_FORCE_PTX_JIT=1 tests/run_regression.sh build
+```
+
+The first run should be used for native H100 and B200 validation. The forced-JIT
+run is slower on first launch because the driver compiles and caches the PTX.
+Floating-point reductions and atomic update order can vary between GPU models,
+so cross-GPU tests should compare invariants and tolerances rather than requiring
+byte-identical output files.

src/simulator/cuda_traffic_sim.cu

@@ -30,6 +30,14 @@
 #include <ctime>
 #include <atomic>
 
+#ifndef LPSIM_CUDA_ARCHITECTURES
+#define LPSIM_CUDA_ARCHITECTURES "unknown"
+#endif
+
+#ifndef LPSIM_CUDA_COMPILER_VERSION
+#define LPSIM_CUDA_COMPILER_VERSION "unknown"
+#endif
+
 #ifndef ushort
 #define ushort uint16_t
 #endif
@@ -69,6 +77,41 @@ inline void printMemoryUsage() {
   double used_db = total_db - free_db;
   printf("GPU memory usage: used = %.0f, free = %.0f MB, total = %.0f MB\n", used_db / 1024.0 / 1024.0, free_db / 1024.0 / 1024.0, total_db / 1024.0 / 1024.0);
 }
+
+inline void printCudaEnvironment(int deviceCount, int selectedDeviceCount) {
+  int driverVersion = 0;
+  int runtimeVersion = 0;
+  gpuErrchk(cudaDriverGetVersion(&driverVersion));
+  gpuErrchk(cudaRuntimeGetVersion(&runtimeVersion));
+
+  printf("[LPSim CUDA] nvcc=%s architectures=%s\n",
+         LPSIM_CUDA_COMPILER_VERSION, LPSIM_CUDA_ARCHITECTURES);
+  printf("[LPSim CUDA] driver_api=%d.%d runtime=%d.%d visible_devices=%d selected_devices=%d\n",
+         driverVersion / 1000, (driverVersion % 1000) / 10,
+         runtimeVersion / 1000, (runtimeVersion % 1000) / 10,
+         deviceCount, selectedDeviceCount);
+
+  for (int device = 0; device < selectedDeviceCount; ++device) {
+    cudaDeviceProp properties{};
+    gpuErrchk(cudaGetDeviceProperties(&properties, device));
+    printf("[LPSim CUDA] device=%d name=\"%s\" cc=%d.%d memory_mib=%zu sms=%d\n",
+           device, properties.name, properties.major, properties.minor,
+           properties.totalGlobalMem / (1024 * 1024),
+           properties.multiProcessorCount);
+  }
+
+  for (int source = 0; source < selectedDeviceCount; ++source) {
+    for (int target = 0; target < selectedDeviceCount; ++target) {
+      if (source == target) {
+        continue;
+      }
+      int canAccessPeer = 0;
+      gpuErrchk(cudaDeviceCanAccessPeer(&canAccessPeer, source, target));
+      printf("[LPSim CUDA] p2p=%d->%d available=%s\n",
+             source, target, canAccessPeer ? "yes" : "no");
+    }
+  }
+}
 ////////////////////////////////
 // Multi-GPU device state
 //
@@ -155,13 +198,26 @@ void b18InitCUDA_n(
   std::vector<float>& numVehPerLinePerTimeInterval,
   float deltaTime) {
   ngpus = num_gpus;
+  if (ngpus <= 0) {
+    fprintf(stderr, "NUM_GPUS must be positive, got %d\n", ngpus);
+    exit(1);
+  }
+
   int maxGpus = 0;
-  cudaGetDeviceCount(&maxGpus);
+  cudaError_t deviceCountStatus = cudaGetDeviceCount(&maxGpus);
+  if (deviceCountStatus != cudaSuccess) {
+    fprintf(stderr, "Unable to enumerate CUDA devices: %s\n",
+            cudaGetErrorString(deviceCountStatus));
+    exit(deviceCountStatus);
+  }
   if(maxGpus<ngpus){
-    printf("NUM_GPUS is %d but only %d gpus on device\n",ngpus,maxGpus);
+    fprintf(stderr, "NUM_GPUS is %d but only %d GPUs are visible\n",ngpus,maxGpus);
     exit(1);
   }
   assert(maxGpus>=ngpus);
+  if (firstInitialization) {
+    printCudaEnvironment(maxGpus, ngpus);
+  }
   trafficVehicleVec_d_gpus = new LC::B18TrafficVehicle*[ngpus];
   indexPathVec_d = new uint*[ngpus];
   edgesData_d = new LC::B18EdgeData*[ngpus];
@@ -396,14 +452,19 @@ void b18InitCUDA_n(
   }
   // peer to peer
   {
-    int canAccessPeer;
     for (int i = 0; i < ngpus; i++) {
-      cudaSetDevice(i);
+      gpuErrchk(cudaSetDevice(i));
         for (int j = 0; j < ngpus; j++) {
             if (i != j) {
-                cudaDeviceCanAccessPeer(&canAccessPeer, i, j);
+                int canAccessPeer = 0;
+                gpuErrchk(cudaDeviceCanAccessPeer(&canAccessPeer, i, j));
                 if (canAccessPeer) {
-                  cudaDeviceEnablePeerAccess(j, 0);
+                  cudaError_t peerStatus = cudaDeviceEnablePeerAccess(j, 0);
+                  if (peerStatus == cudaErrorPeerAccessAlreadyEnabled) {
+                    cudaGetLastError();
+                  } else {
+                    gpuErrchk(peerStatus);
+                  }
                     printf("Peer2Peer support: %d-%d\n",i,j);
                 }
             }
@@ -2394,5 +2455,3 @@ void b18SimulateTrafficCUDA(float currentTime,
 
   peopleBench.stopMeasuring();
 }
-
-