From 8e5af77b9c8a08396971884832203eaed410e4d2 Mon Sep 17 00:00:00 2001 From: Bruce Wayne Date: Wed, 30 Sep 2026 13:27:44 -0700 Subject: [PATCH 1/2] Stop workers without requeue on fatal GPU health failures --- lib/resource_manager.py | 60 +++++++++++++++++++++++++---------------- worker.py | 6 ++++- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/lib/resource_manager.py b/lib/resource_manager.py index f8c73ac..0bc8e7b 100644 --- a/lib/resource_manager.py +++ b/lib/resource_manager.py @@ -1,6 +1,5 @@ from __future__ import annotations import time -import traceback import types import resource import threading @@ -17,21 +16,35 @@ class ResourceLimitError(Exception): pass +class GPUHealthError(Exception): + pass + + +def check_gpu_health(gpu_bus_ids): + for bus_id in gpu_bus_ids: + try: + gpu_dev = pynvml.nvmlDeviceGetHandleByPciBusId(bus_id) + pynvml.nvmlDeviceGetTemperature(gpu_dev, pynvml.NVML_TEMPERATURE_GPU) + recovery = pynvml.nvmlDeviceGetFieldValues(gpu_dev, [pynvml.NVML_FI_DEV_GET_GPU_RECOVERY_ACTION])[0] + if recovery.nvmlReturn == pynvml.NVML_ERROR_NOT_SUPPORTED: + continue + if recovery.nvmlReturn != pynvml.NVML_SUCCESS: + raise pynvml.NVMLError(recovery.nvmlReturn) + if recovery.value.uiVal != 0: + raise GPUHealthError(f"GPU {bus_id} requires recovery action {recovery.value.uiVal}") + except pynvml.NVMLError as e: + raise GPUHealthError(f"GPU {bus_id} health check failed: {e}") from e + + def check_gpu_status_worker(gpu_bus_ids, output): - pynvml.nvmlInit() while True: try: - for bus_id in gpu_bus_ids: - gpu_dev = pynvml.nvmlDeviceGetHandleByPciBusId(bus_id) - pynvml.nvmlDeviceGetTemperature(gpu_dev, pynvml.NVML_TEMPERATURE_GPU) # query to tell if it's fallen off - output.valid = True - except pynvml.NVMLError: - output.valid = False - except Exception: - traceback.print_exc() - finally: - output.last_reading = time.time() - time.sleep(10) + check_gpu_health(gpu_bus_ids) + except Exception as e: + output.error = str(e) + return + output.last_reading = time.monotonic() + time.sleep(10) @dataclass @@ -55,7 +68,7 @@ class ResourceManager(): def __init__(self, mem_limit_multiplier=0.8, triton_enabled=False): self._cleanup_executor = ThreadPoolExecutor(max_workers=1) if triton_enabled else None self._cleanup_future: Future[None] | None = None - self.gpu_status = types.SimpleNamespace(valid=True, last_reading=time.time()) + self.gpu_status = types.SimpleNamespace(error=None, last_reading=time.monotonic()) self.cpu_totals = self._get_cpu_info_by_node() self.mem_totals = self._get_mem_info_by_node(mem_limit_multiplier) @@ -67,6 +80,8 @@ def __init__(self, mem_limit_multiplier=0.8, triton_enabled=False): self.gpu_locked_job: str | None = None if self.gpus: + check_gpu_health([gpu.bus_id for gpu in self.gpus]) + self.gpu_status.last_reading = time.monotonic() thread = threading.Thread( target=check_gpu_status_worker, args=([gpu.bus_id for gpu in self.gpus], self.gpu_status), @@ -115,16 +130,15 @@ def get_gpu_ids(self, task_uuid: str) -> tuple[int | None, int | None]: def get_limits(self, task_uuid: str) -> Limits: return self._tasks[task_uuid].limits - def consume(self, limits: Limits, job: str, task_uuid: str) -> None: + def check_gpu_status(self) -> None: if self.gpus: - try: - # don't start tasks if the gpus are not responding, or if the gpu status reading is stale - if time.time() > self.gpu_status.last_reading + 20: - raise Exception("waiting for gpu status reading...") - elif not self.gpu_status.valid: - raise Exception("unable to read gpu status") - except Exception as e: - raise ResourceLimitError(str(e)) from e + if self.gpu_status.error is not None: + raise GPUHealthError(self.gpu_status.error) + if time.monotonic() > self.gpu_status.last_reading + 20: + raise GPUHealthError("GPU health check has not completed for 20 seconds") + + def consume(self, limits: Limits, job: str, task_uuid: str) -> None: + self.check_gpu_status() mem_bytes = limits.memory * GB_TO_BYTES small_gpu_mem_bytes = limits.small_gpu_memory * GB_TO_BYTES diff --git a/worker.py b/worker.py index 0136945..0510bdb 100755 --- a/worker.py +++ b/worker.py @@ -35,7 +35,7 @@ cgroup_add_pid, cgroup_kill, cgroup_delete, cgroup_clear_all_children, cgroup_is_populated, ) from miniray.lib.sig_term_handler import SigTermHandler -from miniray.lib.resource_manager import ResourceManager, ResourceLimitError +from miniray.lib.resource_manager import ResourceManager, ResourceLimitError, GPUHealthError from miniray.lib.worker_helpers import ExponentialBackoff from miniray.lib.triton_helpers import TRITON_SERVER_ADDRESS, check_triton_server_health, wait_for_triton_server from miniray.lib.system_helpers import ( @@ -615,6 +615,7 @@ def main(): try: while not sigterm_handler.raised: + rm.check_gpu_status() r_master.set(ACTIVE_KEY, 1, ex=SLEEP_TIME_MAX+1) backoff.sleep() @@ -708,6 +709,9 @@ def main(): task.finish() timings['start_task'] += time.perf_counter() - t0 last_init_timings = task.init_timings + except GPUHealthError: + traceback.print_exc() + sys.exit(1) except Exception as e: fatal_error = e traceback.print_exc() From b80479c461a4af5e780d4e3632c9e610a10466b3 Mon Sep 17 00:00:00 2001 From: Bruce Wayne Date: Wed, 30 Sep 2026 13:47:12 -0700 Subject: [PATCH 2/2] Limit GPU failure handling to existing health checks --- lib/resource_manager.py | 52 +++++++++++++++-------------------------- worker.py | 1 - 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/lib/resource_manager.py b/lib/resource_manager.py index 0bc8e7b..b271ef4 100644 --- a/lib/resource_manager.py +++ b/lib/resource_manager.py @@ -1,5 +1,6 @@ from __future__ import annotations import time +import traceback import types import resource import threading @@ -20,31 +21,21 @@ class GPUHealthError(Exception): pass -def check_gpu_health(gpu_bus_ids): - for bus_id in gpu_bus_ids: - try: - gpu_dev = pynvml.nvmlDeviceGetHandleByPciBusId(bus_id) - pynvml.nvmlDeviceGetTemperature(gpu_dev, pynvml.NVML_TEMPERATURE_GPU) - recovery = pynvml.nvmlDeviceGetFieldValues(gpu_dev, [pynvml.NVML_FI_DEV_GET_GPU_RECOVERY_ACTION])[0] - if recovery.nvmlReturn == pynvml.NVML_ERROR_NOT_SUPPORTED: - continue - if recovery.nvmlReturn != pynvml.NVML_SUCCESS: - raise pynvml.NVMLError(recovery.nvmlReturn) - if recovery.value.uiVal != 0: - raise GPUHealthError(f"GPU {bus_id} requires recovery action {recovery.value.uiVal}") - except pynvml.NVMLError as e: - raise GPUHealthError(f"GPU {bus_id} health check failed: {e}") from e - - def check_gpu_status_worker(gpu_bus_ids, output): + pynvml.nvmlInit() while True: try: - check_gpu_health(gpu_bus_ids) - except Exception as e: - output.error = str(e) - return - output.last_reading = time.monotonic() - time.sleep(10) + for bus_id in gpu_bus_ids: + gpu_dev = pynvml.nvmlDeviceGetHandleByPciBusId(bus_id) + pynvml.nvmlDeviceGetTemperature(gpu_dev, pynvml.NVML_TEMPERATURE_GPU) # query to tell if it's fallen off + output.valid = True + except pynvml.NVMLError: + output.valid = False + except Exception: + traceback.print_exc() + finally: + output.last_reading = time.time() + time.sleep(10) @dataclass @@ -68,7 +59,7 @@ class ResourceManager(): def __init__(self, mem_limit_multiplier=0.8, triton_enabled=False): self._cleanup_executor = ThreadPoolExecutor(max_workers=1) if triton_enabled else None self._cleanup_future: Future[None] | None = None - self.gpu_status = types.SimpleNamespace(error=None, last_reading=time.monotonic()) + self.gpu_status = types.SimpleNamespace(valid=True, last_reading=time.time()) self.cpu_totals = self._get_cpu_info_by_node() self.mem_totals = self._get_mem_info_by_node(mem_limit_multiplier) @@ -80,8 +71,6 @@ def __init__(self, mem_limit_multiplier=0.8, triton_enabled=False): self.gpu_locked_job: str | None = None if self.gpus: - check_gpu_health([gpu.bus_id for gpu in self.gpus]) - self.gpu_status.last_reading = time.monotonic() thread = threading.Thread( target=check_gpu_status_worker, args=([gpu.bus_id for gpu in self.gpus], self.gpu_status), @@ -130,15 +119,12 @@ def get_gpu_ids(self, task_uuid: str) -> tuple[int | None, int | None]: def get_limits(self, task_uuid: str) -> Limits: return self._tasks[task_uuid].limits - def check_gpu_status(self) -> None: - if self.gpus: - if self.gpu_status.error is not None: - raise GPUHealthError(self.gpu_status.error) - if time.monotonic() > self.gpu_status.last_reading + 20: - raise GPUHealthError("GPU health check has not completed for 20 seconds") - def consume(self, limits: Limits, job: str, task_uuid: str) -> None: - self.check_gpu_status() + if self.gpus: + if time.time() > self.gpu_status.last_reading + 20: + raise GPUHealthError("GPU status reading is stale") + if not self.gpu_status.valid: + raise GPUHealthError("Unable to read GPU status") mem_bytes = limits.memory * GB_TO_BYTES small_gpu_mem_bytes = limits.small_gpu_memory * GB_TO_BYTES diff --git a/worker.py b/worker.py index 0510bdb..f555902 100755 --- a/worker.py +++ b/worker.py @@ -615,7 +615,6 @@ def main(): try: while not sigterm_handler.raised: - rm.check_gpu_status() r_master.set(ACTIVE_KEY, 1, ex=SLEEP_TIME_MAX+1) backoff.sleep()