Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions lib/resource_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ class ResourceLimitError(Exception):
pass


class GPUHealthError(Exception):
pass


def check_gpu_status_worker(gpu_bus_ids, output):
pynvml.nvmlInit()
while True:
Expand Down Expand Up @@ -117,14 +121,10 @@ def get_limits(self, task_uuid: str) -> Limits:

def consume(self, limits: Limits, job: str, task_uuid: str) -> None:
if self.gpus:
try:
# don't start tasks if the gpus are not responding, or if the gpu status reading is stale
if time.time() > self.gpu_status.last_reading + 20:
raise Exception("waiting for gpu status reading...")
elif not self.gpu_status.valid:
raise Exception("unable to read gpu status")
except Exception as e:
raise ResourceLimitError(str(e)) from e
if time.time() > self.gpu_status.last_reading + 20:
raise GPUHealthError("GPU status reading is stale")
if not self.gpu_status.valid:
raise GPUHealthError("Unable to read GPU status")

mem_bytes = limits.memory * GB_TO_BYTES
small_gpu_mem_bytes = limits.small_gpu_memory * GB_TO_BYTES
Expand Down
5 changes: 4 additions & 1 deletion worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
cgroup_add_pid, cgroup_kill, cgroup_delete, cgroup_clear_all_children, cgroup_is_populated,
)
from miniray.lib.sig_term_handler import SigTermHandler
from miniray.lib.resource_manager import ResourceManager, ResourceLimitError
from miniray.lib.resource_manager import ResourceManager, ResourceLimitError, GPUHealthError
from miniray.lib.worker_helpers import ExponentialBackoff
from miniray.lib.triton_helpers import TRITON_SERVER_ADDRESS, check_triton_server_health, wait_for_triton_server
from miniray.lib.system_helpers import (
Expand Down Expand Up @@ -708,6 +708,9 @@ def main():
task.finish()
timings['start_task'] += time.perf_counter() - t0
last_init_timings = task.init_timings
except GPUHealthError:
traceback.print_exc()
sys.exit(1)
except Exception as e:
fatal_error = e
traceback.print_exc()
Expand Down
Loading