diff --git a/lib/resource_manager.py b/lib/resource_manager.py index f8c73ac..b271ef4 100644 --- a/lib/resource_manager.py +++ b/lib/resource_manager.py @@ -17,6 +17,10 @@ class ResourceLimitError(Exception): pass +class GPUHealthError(Exception): + pass + + def check_gpu_status_worker(gpu_bus_ids, output): pynvml.nvmlInit() while True: @@ -117,14 +121,10 @@ def get_limits(self, task_uuid: str) -> Limits: def consume(self, limits: Limits, job: str, task_uuid: str) -> None: if self.gpus: - try: - # don't start tasks if the gpus are not responding, or if the gpu status reading is stale - if time.time() > self.gpu_status.last_reading + 20: - raise Exception("waiting for gpu status reading...") - elif not self.gpu_status.valid: - raise Exception("unable to read gpu status") - except Exception as e: - raise ResourceLimitError(str(e)) from e + if time.time() > self.gpu_status.last_reading + 20: + raise GPUHealthError("GPU status reading is stale") + if not self.gpu_status.valid: + raise GPUHealthError("Unable to read GPU status") mem_bytes = limits.memory * GB_TO_BYTES small_gpu_mem_bytes = limits.small_gpu_memory * GB_TO_BYTES diff --git a/worker.py b/worker.py index 0136945..f555902 100755 --- a/worker.py +++ b/worker.py @@ -35,7 +35,7 @@ cgroup_add_pid, cgroup_kill, cgroup_delete, cgroup_clear_all_children, cgroup_is_populated, ) from miniray.lib.sig_term_handler import SigTermHandler -from miniray.lib.resource_manager import ResourceManager, ResourceLimitError +from miniray.lib.resource_manager import ResourceManager, ResourceLimitError, GPUHealthError from miniray.lib.worker_helpers import ExponentialBackoff from miniray.lib.triton_helpers import TRITON_SERVER_ADDRESS, check_triton_server_health, wait_for_triton_server from miniray.lib.system_helpers import ( @@ -708,6 +708,9 @@ def main(): task.finish() timings['start_task'] += time.perf_counter() - t0 last_init_timings = task.init_timings + except GPUHealthError: + traceback.print_exc() + sys.exit(1) except Exception as e: fatal_error = e traceback.print_exc()