From e60a53f63231a351c9463409a4cca13a57565d57 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 11 May 2026 10:26:56 -0400 Subject: [PATCH 1/7] device/gpu: proactive eviction with adaptive percentage threshold Introduce a two-tier proactive GPU memory eviction mechanism to hide eviction latency and avoid task stalls on allocation failure. Tier-1 (pre-flight in parsec_device_data_reserve_space): before walking each task's data flows, free clean LRU entries while zone utilisation exceeds mem_evict_threshold percent of total capacity. Uses the new parsec_device_try_evict_lru_one() helper which handles the readers / ref-count / trylock / CAS-readers race conditions that the reactive path already handles, and detects full-cycle scans via a cycling_sentinel. Tier-2 (parsec_device_kernel_scheduler): when the clean LRU is empty and zone pressure is above the threshold, proactively enqueue a D2H writeback task on exec_stream[1] so dirty-page eviction latency overlaps with the upcoming H2D and kernel stages rather than blocking the critical path. Adaptive threshold: each device carries mem_evict_threshold, initialised to parsec_gpu_mem_evict_upper (default 95%). When parsec_device_progress_stream returns PARSEC_HOOK_RETURN_NEXT for the kernel-push stream -- meaning every queued task failed to acquire memory, a true stall -- the threshold is stepped down by 5 percentage points toward parsec_gpu_mem_evict_lower (default 80%). This is intentionally coarser than per-task adjustment: a single task failing does not signal a stall; only a full pass of the pending queue failing does. New MCA parameters registered by the CUDA and Level Zero components: device_{cuda,level_zero}_mem_evict_upper (default 95) device_{cuda,level_zero}_mem_evict_lower (default 80) All zone-pressure checks are guarded with #if !defined(PARSEC_GPU_ALLOC_PER_TILE) since that mode has no zone allocator; tier-2 falls back to the simple clean-LRU-empty condition in that mode. Signed-off-by: Joseph Schuchart Co-Authored-By: Claude Sonnet 4.6 --- .../mca/device/cuda/device_cuda_component.c | 6 + parsec/mca/device/device_gpu.c | 150 ++++++++++++++++++ parsec/mca/device/device_gpu.h | 7 + .../level_zero/device_level_zero_component.c | 6 + parsec/mca/device/transfer_gpu.c | 10 ++ 5 files changed, 179 insertions(+) diff --git a/parsec/mca/device/cuda/device_cuda_component.c b/parsec/mca/device/cuda/device_cuda_component.c index 3529917b0..bc18a83ea 100644 --- a/parsec/mca/device/cuda/device_cuda_component.c +++ b/parsec/mca/device/cuda/device_cuda_component.c @@ -161,6 +161,12 @@ static int device_cuda_component_register(void) (void)parsec_mca_param_reg_int_name("device_cuda", "max_number_of_ejected_data", "Sets up the maximum number of blocks that can be ejected from GPU memory", false, false, MAX_PARAM_COUNT, &parsec_gpu_d2h_max_flows); + (void)parsec_mca_param_reg_int_name("device_cuda", "mem_evict_upper", + "Upper threshold (percentage of total GPU zone capacity) at which proactive clean-LRU eviction and D2H writeback begin. When a task stalls waiting for memory, the per-device threshold is stepped down by 5 points toward mem_evict_lower.", + false, false, 95, &parsec_gpu_mem_evict_upper); + (void)parsec_mca_param_reg_int_name("device_cuda", "mem_evict_lower", + "Lower bound (percentage of total GPU zone capacity) to which the adaptive eviction threshold may be reduced after repeated stalls.", + false, false, 80, &parsec_gpu_mem_evict_lower); (void)parsec_mca_param_reg_int_name("device_cuda", "max_streams", "Maximum number of Streams to use for the GPU engine; 2 streams are used for communication between host and device, so the minimum is 3", false, false, PARSEC_GPU_MAX_STREAMS, &parsec_cuda_max_streams); diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index d57ac904e..497ff1292 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -715,6 +715,7 @@ parsec_device_memory_reserve( parsec_device_gpu_module_t* gpu_device, #endif gpu_device->mem_block_size = eltsize; gpu_device->mem_nb_blocks = mem_elem_per_gpu; + gpu_device->mem_evict_threshold = parsec_gpu_mem_evict_upper; return PARSEC_SUCCESS; } @@ -826,6 +827,95 @@ parsec_device_memory_release( parsec_device_gpu_module_t* gpu_device ) return PARSEC_SUCCESS; } +/** + * Try to evict one entry from the clean LRU (gpu_mem_lru) by detaching it from + * its original data and freeing its zone allocation back to the zone allocator. + * + * @param[in] gpu_device the GPU device whose clean LRU is targeted + * @param[in,out] cycling_sentinel cycle-detector: caller initialises to NULL; + * updated to the first entry that could not be evicted. When + * we pop that entry again we know we have looped the entire LRU + * without finding a free-able entry. + * + * @return 1 if a zone block was freed, 0 if the LRU is empty or fully cycling. + */ +#if !defined(PARSEC_GPU_ALLOC_PER_TILE) +static int +parsec_device_try_evict_lru_one( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_data_copy_t **cycling_sentinel ) +{ + parsec_gpu_data_copy_t *lru_gpu_elem; + parsec_data_t *oldmaster; + + retry: + lru_gpu_elem = (parsec_gpu_data_copy_t*)parsec_list_pop_front(&gpu_device->gpu_mem_lru); + if( NULL == lru_gpu_elem ) + return 0; + PARSEC_LIST_ITEM_SINGLETON(lru_gpu_elem); + + if( *cycling_sentinel == lru_gpu_elem ) { + parsec_list_push_front(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)lru_gpu_elem); + return 0; + } + + /* Dangling reader: the copy is temporarily untracked in the LRU; skip it */ + if( 0 != lru_gpu_elem->readers ) + goto retry; + + /* Outstanding object references: not safe to free yet; push back and note cycle */ + if( lru_gpu_elem->super.super.obj_reference_count > 1 ) { + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == *cycling_sentinel ) *cycling_sentinel = lru_gpu_elem; + goto retry; + } + + if( NULL != lru_gpu_elem->original ) { + oldmaster = lru_gpu_elem->original; + if( !parsec_atomic_trylock(&oldmaster->lock) ) { + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == *cycling_sentinel ) *cycling_sentinel = lru_gpu_elem; + goto retry; + } + /* Guard against a concurrent d2d reader acquiring the copy */ + if( !parsec_atomic_cas_int32(&lru_gpu_elem->readers, 0, + -PARSEC_DEVICE_DATA_COPY_ATOMIC_SENTINEL) ) { + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == *cycling_sentinel ) *cycling_sentinel = lru_gpu_elem; + parsec_atomic_unlock(&oldmaster->lock); + goto retry; + } + int do_unlock = oldmaster->super.obj_reference_count != 1; + parsec_data_copy_detach(oldmaster, lru_gpu_elem, gpu_device->super.device_index); + parsec_atomic_wmb(); + if( do_unlock ) + parsec_atomic_unlock(&oldmaster->lock); + } + +#if defined(PARSEC_PROF_TRACE) + if( (gpu_device->trackable_events & PARSEC_PROFILE_GPU_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable) ) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_free_memory_key, + (int64_t)lru_gpu_elem->device_private, + gpu_device->super.device_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_use_memory_key_end, + (uint64_t)lru_gpu_elem->device_private, + gpu_device->super.device_index, NULL, 0); + } +#endif + assert( 0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + zone_free(gpu_device->memory, (void*)lru_gpu_elem->device_private); + lru_gpu_elem->device_private = NULL; + gpu_device->super.nb_evictions++; + PARSEC_OBJ_RELEASE(lru_gpu_elem); + assert( NULL == lru_gpu_elem ); + return 1; +} +#endif /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */ + /** * Try to find memory space to move all data on the GPU. We attach a device_elem to * a memory_elem as soon as a device_elem is available. If we fail to find enough @@ -854,6 +944,24 @@ parsec_device_data_reserve_space( parsec_device_gpu_module_t* gpu_device, (void)copy_readers_update; // potentially unused +#if !defined(PARSEC_GPU_ALLOC_PER_TILE) + /* Tier-1 proactive eviction: free clean LRU entries while zone usage exceeds + * gpu_device->mem_evict_threshold percent of total capacity. The threshold + * starts at parsec_gpu_mem_evict_upper (default 95%) and is lowered in 5-point + * steps (floor: parsec_gpu_mem_evict_lower, default 80%) whenever the device stalled + * because the reactive path also failed to find memory. */ + { + size_t total_capacity = (size_t)gpu_device->mem_nb_blocks * gpu_device->mem_block_size; + parsec_gpu_data_copy_t *cycling = NULL; + while( zone_in_use(gpu_device->memory) * 100 > + (size_t)gpu_device->mem_evict_threshold * total_capacity ) { + if( !parsec_device_try_evict_lru_one(gpu_device, &cycling) ) + break; + data_avail_epoch = 1; + } + } +#endif /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */ + /** * Parse all the input and output flows of data and ensure all have * corresponding data on the GPU available. @@ -2576,6 +2684,41 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, gpu_device->super.device_index, gpu_device->super.name, parsec_device_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task)); } + + /* Tier-2 proactive dirty-page writeback: when the clean LRU is empty and zone + * memory pressure exceeds the watermark, queue a D2H transfer on exec_stream[1] + * now so its latency overlaps with the upcoming H2D stage and kernel execution. + * This converts a potential blocking wait (dirty page eviction on the critical + * path) into an overlapped background transfer. + * + * In PARSEC_GPU_ALLOC_PER_TILE mode there is no zone allocator; fall back to the + * simple condition of clean LRU being empty. */ + if( !parsec_list_nolock_is_empty(&gpu_device->gpu_mem_owned_lru) && + parsec_list_nolock_is_empty(&gpu_device->gpu_mem_lru) ) { + int _do_d2h = 0; +#if !defined(PARSEC_GPU_ALLOC_PER_TILE) + { + size_t total_capacity = (size_t)gpu_device->mem_nb_blocks * gpu_device->mem_block_size; + size_t in_use = zone_in_use(gpu_device->memory); + _do_d2h = in_use * 100 > + (size_t)gpu_device->mem_evict_threshold * total_capacity; + } +#else + _do_d2h = 1; +#endif /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */ + if( _do_d2h ) { + parsec_gpu_task_t *_w2r = parsec_gpu_create_w2r_task(gpu_device, es); + if( NULL != _w2r ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%d:%s]: Proactive D2H writeback: clean LRU empty, zone usage above %d%% threshold", + gpu_device->super.device_index, gpu_device->super.name, + gpu_device->mem_evict_threshold); + PARSEC_PUSH_TASK(gpu_device->exec_stream[1]->fifo_pending, + (parsec_list_item_t*)_w2r); + } + } + } + rc = parsec_device_progress_stream( gpu_device, gpu_device->exec_stream[0], parsec_device_kernel_push, @@ -2594,6 +2737,13 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, } assert(NULL == progress_task); + /* Every task in the pending queue failed to acquire GPU memory on this iteration: + * we are truly stalled. Step the eviction threshold down so tier-1 evicts more + * aggressively starting from the next scheduling iteration. */ + if( PARSEC_HOOK_RETURN_NEXT == rc && + gpu_device->mem_evict_threshold - 5 >= parsec_gpu_mem_evict_lower ) + gpu_device->mem_evict_threshold -= 5; + /* TODO: check this */ /* If we can extract data go for it, otherwise try to drain the pending tasks */ gpu_task = parsec_gpu_create_w2r_task(gpu_device, es); diff --git a/parsec/mca/device/device_gpu.h b/parsec/mca/device/device_gpu.h index fa25b87a3..098eb10ca 100644 --- a/parsec/mca/device/device_gpu.h +++ b/parsec/mca/device/device_gpu.h @@ -245,6 +245,11 @@ struct parsec_device_gpu_module_s { parsec_gpu_exec_stream_t **exec_stream; size_t mem_block_size; int64_t mem_nb_blocks; + int32_t mem_evict_threshold; /**< Current eviction threshold (% of total zone + * capacity). Starts at parsec_gpu_mem_evict_upper + * and is stepped down by 5 points (to + * parsec_gpu_mem_evict_lower) each time a task + * stalls waiting for zone memory. */ #if defined(PARSEC_PROF_TRACE) int trackable_events; #endif /* PARSEC_PROF_TRACE */ @@ -279,6 +284,8 @@ typedef struct parsec_gpu_workspace_s { PARSEC_DECLSPEC extern int parsec_gpu_output_stream; PARSEC_DECLSPEC extern int parsec_gpu_verbosity; PARSEC_DECLSPEC extern int32_t parsec_gpu_d2h_max_flows; +PARSEC_DECLSPEC extern int32_t parsec_gpu_mem_evict_upper; +PARSEC_DECLSPEC extern int32_t parsec_gpu_mem_evict_lower; /** * Debugging functions. diff --git a/parsec/mca/device/level_zero/device_level_zero_component.c b/parsec/mca/device/level_zero/device_level_zero_component.c index f50f2a817..287e9f4a6 100644 --- a/parsec/mca/device/level_zero/device_level_zero_component.c +++ b/parsec/mca/device/level_zero/device_level_zero_component.c @@ -271,6 +271,12 @@ static int device_level_zero_component_register(void) (void)parsec_mca_param_reg_int_name("device_level_zero", "max_number_of_ejected_data", "Sets up the maximum number of blocks that can be ejected from GPU memory", false, false, MAX_PARAM_COUNT, &parsec_gpu_d2h_max_flows); + (void)parsec_mca_param_reg_int_name("device_level_zero", "mem_evict_upper", + "Upper threshold (percentage of total GPU zone capacity) at which proactive clean-LRU eviction and D2H writeback begin. When a task stalls waiting for memory, the per-device threshold is stepped down by 5 points toward mem_evict_lower.", + false, false, 95, &parsec_gpu_mem_evict_upper); + (void)parsec_mca_param_reg_int_name("device_level_zero", "mem_evict_lower", + "Lower bound (percentage of total GPU zone capacity) to which the adaptive eviction threshold may be reduced after repeated stalls.", + false, false, 80, &parsec_gpu_mem_evict_lower); (void)parsec_mca_param_reg_int_name("device_level_zero", "max_streams", "Maximum number of Streams to use for the GPU engine; 2 streams are used for communication between host and device, so the minimum is 3", false, false, PARSEC_GPU_MAX_STREAMS, &parsec_level_zero_max_streams); diff --git a/parsec/mca/device/transfer_gpu.c b/parsec/mca/device/transfer_gpu.c index 50b0d886e..7484b9f1a 100644 --- a/parsec/mca/device/transfer_gpu.c +++ b/parsec/mca/device/transfer_gpu.c @@ -180,6 +180,16 @@ static const parsec_symbol_t symb_gpu_d2h_task_param = { int32_t parsec_gpu_d2h_max_flows = 0; +/* Proactive eviction thresholds (percentage of total zone capacity). + * Registered as MCA parameters by each GPU backend component. + * mem_evict_upper: initial percentage at which proactive eviction begins (default 95). + * mem_evict_lower: floor to which the per-device threshold may adapt downwards (default 80). + * When a task stalls because no zone memory could be freed, the per-device + * mem_evict_threshold is lowered by 5 points (clamped to mem_evict_lower) so + * future eviction runs start sooner. */ +int32_t parsec_gpu_mem_evict_upper = 95; +int32_t parsec_gpu_mem_evict_lower = 80; + static const parsec_task_class_t parsec_gpu_d2h_task_class = { .name = "GPU D2H data transfer", .task_class_id = 0, From cee37a1d2cb9c4f458f13b788a95581cb5020407 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 11 May 2026 11:07:50 -0400 Subject: [PATCH 2/7] device/gpu: size-aware proactive D2H writeback with in-flight tracking parsec_gpu_create_w2r_task() now accepts a required_size (bytes) and an out-parameter selected_size so callers know exactly how much dirty data was selected for the D2H transfer. The selection loop stops as soon as the accumulated nb_elts bytes reach required_size (or the max-flows cap is hit, whichever comes first). A new per-device field mem_evict_in_flight tracks the total bytes of dirty GPU data currently queued or executing on exec_stream[1]: - incremented in parsec_gpu_create_w2r_task() by the bytes selected - decremented in parsec_gpu_complete_w2r_task() as each copy finishes Tier-2 in parsec_device_kernel_scheduler() now: 1. Computes needed = zone_in_use - threshold_bytes 2. Derives still_needed = needed - mem_evict_in_flight (avoiding redundant D2H tasks when enough data is already being evicted) 3. Loops calling parsec_gpu_create_w2r_task(still_needed) and pushing each resulting task to exec_stream[1] until still_needed is satisfied or the owned LRU is exhausted The reactive fallback (all push tasks stalled) passes SIZE_MAX so it drains as many dirty pages as the max-flows cap allows, unchanged from previous behavior. PARSEC_GPU_ALLOC_PER_TILE mode (no zone allocator) also uses SIZE_MAX and issues one batch, keeping the prior behavior. Signed-off-by: Joseph Schuchart Co-Authored-By: Claude Sonnet 4.6 --- parsec/mca/device/device_gpu.c | 43 ++++++++++++++++++++++++-------- parsec/mca/device/device_gpu.h | 8 +++++- parsec/mca/device/transfer_gpu.c | 20 +++++++++++---- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index 497ff1292..b9692b4d9 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -2695,28 +2695,49 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, * simple condition of clean LRU being empty. */ if( !parsec_list_nolock_is_empty(&gpu_device->gpu_mem_owned_lru) && parsec_list_nolock_is_empty(&gpu_device->gpu_mem_lru) ) { - int _do_d2h = 0; #if !defined(PARSEC_GPU_ALLOC_PER_TILE) { size_t total_capacity = (size_t)gpu_device->mem_nb_blocks * gpu_device->mem_block_size; size_t in_use = zone_in_use(gpu_device->memory); - _do_d2h = in_use * 100 > - (size_t)gpu_device->mem_evict_threshold * total_capacity; + size_t threshold_bytes = (size_t)gpu_device->mem_evict_threshold * total_capacity / 100; + if( in_use > threshold_bytes ) { + /* Compute how many more bytes of dirty-page eviction are needed beyond + * what is already in-flight on exec_stream[1]. */ + size_t needed = in_use - threshold_bytes; + size_t still_needed = (needed > gpu_device->mem_evict_in_flight) ? + (needed - gpu_device->mem_evict_in_flight) : 0; + while( still_needed > 0 ) { + size_t selected = 0; + parsec_gpu_task_t *_w2r = parsec_gpu_create_w2r_task(gpu_device, es, + still_needed, &selected); + if( NULL == _w2r ) break; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%d:%s]: Proactive D2H writeback: clean LRU empty, " + "zone above %d%% threshold; needed %zu, selected %zu bytes", + gpu_device->super.device_index, gpu_device->super.name, + gpu_device->mem_evict_threshold, still_needed, selected); + PARSEC_PUSH_TASK(gpu_device->exec_stream[1]->fifo_pending, + (parsec_list_item_t*)_w2r); + still_needed = (still_needed > selected) ? (still_needed - selected) : 0; + } + } } #else - _do_d2h = 1; -#endif /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */ - if( _do_d2h ) { - parsec_gpu_task_t *_w2r = parsec_gpu_create_w2r_task(gpu_device, es); + { + /* No zone allocator in ALLOC_PER_TILE mode: issue one D2H batch whenever + * the clean LRU is empty and the dirty LRU is non-empty. */ + size_t selected = 0; + parsec_gpu_task_t *_w2r = parsec_gpu_create_w2r_task(gpu_device, es, + SIZE_MAX, &selected); if( NULL != _w2r ) { PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, - "GPU[%d:%s]: Proactive D2H writeback: clean LRU empty, zone usage above %d%% threshold", - gpu_device->super.device_index, gpu_device->super.name, - gpu_device->mem_evict_threshold); + "GPU[%d:%s]: Proactive D2H writeback: clean LRU empty, selected %zu bytes", + gpu_device->super.device_index, gpu_device->super.name, selected); PARSEC_PUSH_TASK(gpu_device->exec_stream[1]->fifo_pending, (parsec_list_item_t*)_w2r); } } +#endif /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */ } rc = parsec_device_progress_stream( gpu_device, @@ -2746,7 +2767,7 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, /* TODO: check this */ /* If we can extract data go for it, otherwise try to drain the pending tasks */ - gpu_task = parsec_gpu_create_w2r_task(gpu_device, es); + { size_t _sel = 0; gpu_task = parsec_gpu_create_w2r_task(gpu_device, es, SIZE_MAX, &_sel); } if( NULL != gpu_task ) goto get_data_out_of_device; } diff --git a/parsec/mca/device/device_gpu.h b/parsec/mca/device/device_gpu.h index 098eb10ca..420073c4b 100644 --- a/parsec/mca/device/device_gpu.h +++ b/parsec/mca/device/device_gpu.h @@ -250,6 +250,11 @@ struct parsec_device_gpu_module_s { * and is stepped down by 5 points (to * parsec_gpu_mem_evict_lower) each time a task * stalls waiting for zone memory. */ + size_t mem_evict_in_flight; /**< Bytes of dirty GPU data currently selected for + * D2H eviction (queued or executing on + * exec_stream[1]). Incremented by + * parsec_gpu_create_w2r_task, decremented as each + * copy completes in parsec_gpu_complete_w2r_task. */ #if defined(PARSEC_PROF_TRACE) int trackable_events; #endif /* PARSEC_PROF_TRACE */ @@ -311,7 +316,8 @@ int parsec_device_free_workspace(parsec_device_gpu_module_t * gpu_device); /* sort pending task list by number of spaces needed */ int parsec_device_sort_pending_list(parsec_device_module_t *gpu_device); -parsec_gpu_task_t* parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, parsec_execution_stream_t *es); +parsec_gpu_task_t* parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, parsec_execution_stream_t *es, + size_t required_size, size_t *selected_size); int parsec_gpu_complete_w2r_task(parsec_device_gpu_module_t *gpu_device, parsec_gpu_task_t *w2r_task, parsec_execution_stream_t *es); void parsec_device_enable_debug(void); diff --git a/parsec/mca/device/transfer_gpu.c b/parsec/mca/device/transfer_gpu.c index 7484b9f1a..2de7ab6bb 100644 --- a/parsec/mca/device/transfer_gpu.c +++ b/parsec/mca/device/transfer_gpu.c @@ -233,16 +233,20 @@ static const parsec_task_class_t parsec_gpu_d2h_task_class = { */ parsec_gpu_task_t* parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, - parsec_execution_stream_t *es) + parsec_execution_stream_t *es, + size_t required_size, + size_t *selected_size) { parsec_gpu_task_t *w2r_task = NULL; parsec_gpu_d2h_task_t *d2h_task = NULL; parsec_gpu_data_copy_t *gpu_copy; parsec_list_item_t* item = (parsec_list_item_t*)gpu_device->gpu_mem_owned_lru.ghost_element.list_next; int nb_cleaned = 0; + size_t _selected = 0; - /* Find a data copy that has no pending users on the GPU, and can be - * safely moved back on the main memory */ + /* Find data copies with no pending GPU readers that can be safely moved back to + * main memory. Stop once nb_cleaned reaches the max-flows cap or we have + * accumulated at least required_size bytes. */ while(nb_cleaned < parsec_gpu_d2h_max_flows) { /* Break at the end of the list */ if( item == &(gpu_device->gpu_mem_owned_lru.ghost_element) ) { @@ -257,7 +261,7 @@ parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, d2h_task = (parsec_gpu_d2h_task_t*)parsec_thread_mempool_allocate(es->context_mempool); if( PARSEC_UNLIKELY(NULL == d2h_task) ) { /* we're running out of memory. Bail out. */ parsec_atomic_unlock( &gpu_copy->original->lock ); - return NULL; + break; } PARSEC_OBJ_CONSTRUCT(d2h_task, parsec_task_t); } @@ -270,17 +274,22 @@ parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "D2H[%d:%s] task %p:\tdata %d -> %p [%p] readers %d", gpu_device->super.device_index, gpu_device->super.name, (void*)d2h_task, nb_cleaned, gpu_copy, gpu_copy->original, gpu_copy->readers); + _selected += gpu_copy->original->nb_elts; nb_cleaned++; - if (MAX_PARAM_COUNT == nb_cleaned) + if( MAX_PARAM_COUNT == nb_cleaned || _selected >= required_size ) break; } else { parsec_atomic_unlock( &gpu_copy->original->lock ); } } + *selected_size = _selected; + if( 0 == nb_cleaned ) return NULL; + gpu_device->mem_evict_in_flight += _selected; + d2h_task->priority = INT32_MAX; d2h_task->task_class = &parsec_gpu_d2h_task_class; d2h_task->taskpool = NULL; @@ -320,6 +329,7 @@ int parsec_gpu_complete_w2r_task(parsec_device_gpu_module_t *gpu_device, gpu_copy->readers--; gpu_copy->data_transfer_status = PARSEC_DATA_STATUS_COMPLETE_TRANSFER; gpu_device->super.data_out_to_host += gpu_copy->original->nb_elts; /* TODO: not hardcoded, use datatype size */ + gpu_device->mem_evict_in_flight -= gpu_copy->original->nb_elts; assert(gpu_copy->readers >= 0); original = gpu_copy->original; From c9f7ba73fd26a2ac96c0a9b882c495a0c752ea3b Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 11 May 2026 11:18:05 -0400 Subject: [PATCH 3/7] device/gpu: guard reactive D2H eviction when evictions already in flight The reactive fallback that issues a D2H writeback when all kernel-push tasks stall on memory now checks mem_evict_in_flight first. If evictions are already active, queuing another batch would create a storm of D2H tasks that pile up faster than they complete. Instead, let the in-flight transfers finish and free zone memory before issuing more. Signed-off-by: Joseph Schuchart Co-Authored-By: Claude Sonnet 4.6 --- parsec/mca/device/device_gpu.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index b9692b4d9..0a098a5c1 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -2766,8 +2766,12 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, gpu_device->mem_evict_threshold -= 5; /* TODO: check this */ - /* If we can extract data go for it, otherwise try to drain the pending tasks */ - { size_t _sel = 0; gpu_task = parsec_gpu_create_w2r_task(gpu_device, es, SIZE_MAX, &_sel); } + /* If we can extract data go for it, otherwise try to drain the pending tasks. + * Skip if evictions are already in flight to avoid a storm of D2H tasks. */ + if( 0 == gpu_device->mem_evict_in_flight ) { + size_t _sel = 0; + gpu_task = parsec_gpu_create_w2r_task(gpu_device, es, SIZE_MAX, &_sel); + } if( NULL != gpu_task ) goto get_data_out_of_device; } From dcf8468c0971fcdcb4819a4c07cfb119848ea4c5 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 11 May 2026 13:50:28 -0400 Subject: [PATCH 4/7] device/gpu: address review feedback on proactive eviction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three issues raised in code review: 1. MCA parameter scope mem_evict_upper and mem_evict_lower were registered once per GPU backend component, which means the second registration overwrites the first when both CUDA and Level Zero are enabled, and the params end up under backend-specific namespaces. Move the registrations to parsec_mca_device_init() in device.c under the "device" namespace (device_mem_evict_upper / device_mem_evict_lower), guarded by PARSEC_HAVE_CUDA || PARSEC_HAVE_HIP || PARSEC_HAVE_LEVEL_ZERO so the extern references are only present when transfer_gpu.c is compiled. 2. data_avail_epoch style Tier-1 used `data_avail_epoch = 1` while the rest of the function uses `data_avail_epoch++`. Changed to `++` for consistency. 3. Threshold step-down condition The previous placement fired on every PARSEC_HOOK_RETURN_NEXT from parsec_device_progress_stream, which tries one task per call — not all tasks. This could drive mem_evict_threshold to its minimum rapidly. The step-down is now integrated into the mem_evict_in_flight == 0 guard: we only lower the threshold when there are no active evictions AND parsec_gpu_create_w2r_task also returns NULL (no dirty pages available to queue). That is the true "stuck" condition. Signed-off-by: Joseph Schuchart Co-Authored-By: Claude Sonnet 4.6 --- parsec/mca/device/cuda/device_cuda_component.c | 6 ------ parsec/mca/device/device.c | 18 ++++++++++++++++++ parsec/mca/device/device_gpu.c | 18 +++++++++--------- .../level_zero/device_level_zero_component.c | 6 ------ 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/parsec/mca/device/cuda/device_cuda_component.c b/parsec/mca/device/cuda/device_cuda_component.c index bc18a83ea..3529917b0 100644 --- a/parsec/mca/device/cuda/device_cuda_component.c +++ b/parsec/mca/device/cuda/device_cuda_component.c @@ -161,12 +161,6 @@ static int device_cuda_component_register(void) (void)parsec_mca_param_reg_int_name("device_cuda", "max_number_of_ejected_data", "Sets up the maximum number of blocks that can be ejected from GPU memory", false, false, MAX_PARAM_COUNT, &parsec_gpu_d2h_max_flows); - (void)parsec_mca_param_reg_int_name("device_cuda", "mem_evict_upper", - "Upper threshold (percentage of total GPU zone capacity) at which proactive clean-LRU eviction and D2H writeback begin. When a task stalls waiting for memory, the per-device threshold is stepped down by 5 points toward mem_evict_lower.", - false, false, 95, &parsec_gpu_mem_evict_upper); - (void)parsec_mca_param_reg_int_name("device_cuda", "mem_evict_lower", - "Lower bound (percentage of total GPU zone capacity) to which the adaptive eviction threshold may be reduced after repeated stalls.", - false, false, 80, &parsec_gpu_mem_evict_lower); (void)parsec_mca_param_reg_int_name("device_cuda", "max_streams", "Maximum number of Streams to use for the GPU engine; 2 streams are used for communication between host and device, so the minimum is 3", false, false, PARSEC_GPU_MAX_STREAMS, &parsec_cuda_max_streams); diff --git a/parsec/mca/device/device.c b/parsec/mca/device/device.c index f593eab47..4cb31f87e 100644 --- a/parsec/mca/device/device.c +++ b/parsec/mca/device/device.c @@ -301,6 +301,12 @@ no_valid_device: { PARSEC_OBJ_CLASS_INSTANCE(parsec_device_module_t, parsec_object_t, NULL, NULL); +#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_HIP) || defined(PARSEC_HAVE_LEVEL_ZERO) +/* Defined in transfer_gpu.c; registered here so they apply to all GPU backends. */ +extern int32_t parsec_gpu_mem_evict_upper; +extern int32_t parsec_gpu_mem_evict_lower; +#endif + int parsec_mca_device_init(void) { char** parsec_device_list = NULL; @@ -313,6 +319,18 @@ int parsec_mca_device_init(void) PARSEC_OBJ_CONSTRUCT(&parsec_per_device_infos, parsec_info_t); PARSEC_OBJ_CONSTRUCT(&parsec_per_stream_infos, parsec_info_t); +#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_HIP) || defined(PARSEC_HAVE_LEVEL_ZERO) + (void)parsec_mca_param_reg_int_name("device", "mem_evict_upper", + "Upper threshold (percentage of total GPU zone capacity) at which proactive " + "clean-LRU eviction and D2H writeback begin. When the device is truly stalled " + "(no in-flight evictions and no dirty pages left to queue), the per-device " + "threshold is stepped down by 5 points toward device_mem_evict_lower.", + false, false, 95, &parsec_gpu_mem_evict_upper); + (void)parsec_mca_param_reg_int_name("device", "mem_evict_lower", + "Lower bound (percentage of total GPU zone capacity) to which the adaptive " + "eviction threshold may be reduced after repeated stalls.", + false, false, 80, &parsec_gpu_mem_evict_lower); +#endif /* PARSEC_HAVE_CUDA || PARSEC_HAVE_HIP || PARSEC_HAVE_LEVEL_ZERO */ (void)parsec_mca_param_reg_int_name("device", "show_capabilities", "Show the detailed devices capabilities", false, false, parsec_debug_verbose >= 4 || (parsec_debug_verbose >= 3 && parsec_debug_rank == 0), NULL); diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index 0a098a5c1..1f73b4101 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -957,7 +957,7 @@ parsec_device_data_reserve_space( parsec_device_gpu_module_t* gpu_device, (size_t)gpu_device->mem_evict_threshold * total_capacity ) { if( !parsec_device_try_evict_lru_one(gpu_device, &cycling) ) break; - data_avail_epoch = 1; + data_avail_epoch++; } } #endif /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */ @@ -2758,19 +2758,19 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, } assert(NULL == progress_task); - /* Every task in the pending queue failed to acquire GPU memory on this iteration: - * we are truly stalled. Step the eviction threshold down so tier-1 evicts more - * aggressively starting from the next scheduling iteration. */ - if( PARSEC_HOOK_RETURN_NEXT == rc && - gpu_device->mem_evict_threshold - 5 >= parsec_gpu_mem_evict_lower ) - gpu_device->mem_evict_threshold -= 5; - /* TODO: check this */ /* If we can extract data go for it, otherwise try to drain the pending tasks. - * Skip if evictions are already in flight to avoid a storm of D2H tasks. */ + * Skip if evictions are already in flight to avoid a storm of D2H tasks. + * If there are no in-flight evictions and nothing left to queue from the dirty + * LRU, we are truly stuck: step the eviction threshold down so tier-1 starts + * freeing pages earlier on the next attempt. */ if( 0 == gpu_device->mem_evict_in_flight ) { size_t _sel = 0; gpu_task = parsec_gpu_create_w2r_task(gpu_device, es, SIZE_MAX, &_sel); + if(gpu_device->mem_evict_threshold - 5 >= parsec_gpu_mem_evict_lower) { + /* We had to trigger a proactive D2H writeback so reduce the threshold */ + gpu_device->mem_evict_threshold -= 5; + } } if( NULL != gpu_task ) goto get_data_out_of_device; diff --git a/parsec/mca/device/level_zero/device_level_zero_component.c b/parsec/mca/device/level_zero/device_level_zero_component.c index 287e9f4a6..f50f2a817 100644 --- a/parsec/mca/device/level_zero/device_level_zero_component.c +++ b/parsec/mca/device/level_zero/device_level_zero_component.c @@ -271,12 +271,6 @@ static int device_level_zero_component_register(void) (void)parsec_mca_param_reg_int_name("device_level_zero", "max_number_of_ejected_data", "Sets up the maximum number of blocks that can be ejected from GPU memory", false, false, MAX_PARAM_COUNT, &parsec_gpu_d2h_max_flows); - (void)parsec_mca_param_reg_int_name("device_level_zero", "mem_evict_upper", - "Upper threshold (percentage of total GPU zone capacity) at which proactive clean-LRU eviction and D2H writeback begin. When a task stalls waiting for memory, the per-device threshold is stepped down by 5 points toward mem_evict_lower.", - false, false, 95, &parsec_gpu_mem_evict_upper); - (void)parsec_mca_param_reg_int_name("device_level_zero", "mem_evict_lower", - "Lower bound (percentage of total GPU zone capacity) to which the adaptive eviction threshold may be reduced after repeated stalls.", - false, false, 80, &parsec_gpu_mem_evict_lower); (void)parsec_mca_param_reg_int_name("device_level_zero", "max_streams", "Maximum number of Streams to use for the GPU engine; 2 streams are used for communication between host and device, so the minimum is 3", false, false, PARSEC_GPU_MAX_STREAMS, &parsec_level_zero_max_streams); From a9904e915d0ce7f47dfc028f13d5706cc694b7f6 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 11 May 2026 17:44:49 -0400 Subject: [PATCH 5/7] device/gpu: address second round of review feedback Three issues from code review: 1. Proactive D2H tasks not counted in device->mutex Tier-2 proactive w2r tasks were pushed to exec_stream[1]->fifo_pending without incrementing device->mutex, so the GPU manager thread could exit (when the last regular task decremented mutex to zero) while proactive D2H transfers were still queued or in flight. Fix: introduce PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER. When tier-2 creates and pushes such a task it increments device->mutex. At complete_task: the proactive path calls parsec_gpu_complete_w2r_task (which already frees the task) and then decrements mutex via the same exit logic used by regular tasks, including the "last one out" exit path that returns PARSEC_HOOK_RETURN_ASYNC. 2. Parameter validation for mem_evict_upper / mem_evict_lower After registering the MCA parameters, look up the actual values (which may have been overridden via env/config), clamp each to [0,100] with a warning, and swap them if lower > upper. 3. Variables moved from transfer_gpu.c to device.c parsec_gpu_mem_evict_upper and parsec_gpu_mem_evict_lower are now defined in device.c so they always exist regardless of which GPU backends are compiled in. The MCA registrations are now unconditional (no PARSEC_HAVE_CUDA/HIP/LEVEL_ZERO guard needed). Extern declarations remain in device_gpu.h for use by GPU-side code. Signed-off-by: Joseph Schuchart Co-Authored-By: Claude Sonnet 4.6 --- parsec/mca/device/device.c | 40 +++++++++++++++++++++++++------- parsec/mca/device/device_gpu.c | 23 +++++++++++++++++- parsec/mca/device/device_gpu.h | 11 +++++---- parsec/mca/device/transfer_gpu.c | 15 ++++-------- 4 files changed, 64 insertions(+), 25 deletions(-) diff --git a/parsec/mca/device/device.c b/parsec/mca/device/device.c index 4cb31f87e..38623659b 100644 --- a/parsec/mca/device/device.c +++ b/parsec/mca/device/device.c @@ -301,11 +301,12 @@ no_valid_device: { PARSEC_OBJ_CLASS_INSTANCE(parsec_device_module_t, parsec_object_t, NULL, NULL); -#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_HIP) || defined(PARSEC_HAVE_LEVEL_ZERO) -/* Defined in transfer_gpu.c; registered here so they apply to all GPU backends. */ -extern int32_t parsec_gpu_mem_evict_upper; -extern int32_t parsec_gpu_mem_evict_lower; -#endif +/* Proactive GPU eviction thresholds, registered as MCA parameters in + * parsec_mca_device_init() below. Defined here (not transfer_gpu.c) so the + * symbols always exist and the parameters can be registered unconditionally, + * regardless of which GPU backends were compiled in. */ +int32_t parsec_gpu_mem_evict_upper = 95; +int32_t parsec_gpu_mem_evict_lower = 80; int parsec_mca_device_init(void) { @@ -319,18 +320,39 @@ int parsec_mca_device_init(void) PARSEC_OBJ_CONSTRUCT(&parsec_per_device_infos, parsec_info_t); PARSEC_OBJ_CONSTRUCT(&parsec_per_stream_infos, parsec_info_t); -#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_HIP) || defined(PARSEC_HAVE_LEVEL_ZERO) (void)parsec_mca_param_reg_int_name("device", "mem_evict_upper", "Upper threshold (percentage of total GPU zone capacity) at which proactive " "clean-LRU eviction and D2H writeback begin. When the device is truly stalled " "(no in-flight evictions and no dirty pages left to queue), the per-device " - "threshold is stepped down by 5 points toward device_mem_evict_lower.", + "threshold is stepped down by 5 points toward device_mem_evict_lower. " + "Valid range [0,100]; must be >= device_mem_evict_lower.", false, false, 95, &parsec_gpu_mem_evict_upper); (void)parsec_mca_param_reg_int_name("device", "mem_evict_lower", "Lower bound (percentage of total GPU zone capacity) to which the adaptive " - "eviction threshold may be reduced after repeated stalls.", + "eviction threshold may be reduced after repeated stalls. " + "Valid range [0,100]; must be <= device_mem_evict_upper.", false, false, 80, &parsec_gpu_mem_evict_lower); -#endif /* PARSEC_HAVE_CUDA || PARSEC_HAVE_HIP || PARSEC_HAVE_LEVEL_ZERO */ + if( 0 < (rc = parsec_mca_param_find("device", NULL, "mem_evict_upper")) ) + parsec_mca_param_lookup_int(rc, &parsec_gpu_mem_evict_upper); + if( 0 < (rc = parsec_mca_param_find("device", NULL, "mem_evict_lower")) ) + parsec_mca_param_lookup_int(rc, &parsec_gpu_mem_evict_lower); + if( parsec_gpu_mem_evict_upper < 0 || parsec_gpu_mem_evict_upper > 100 ) { + parsec_warning("device_mem_evict_upper=%d is out of range [0,100], clamped", + parsec_gpu_mem_evict_upper); + parsec_gpu_mem_evict_upper = parsec_gpu_mem_evict_upper < 0 ? 0 : 100; + } + if( parsec_gpu_mem_evict_lower < 0 || parsec_gpu_mem_evict_lower > 100 ) { + parsec_warning("device_mem_evict_lower=%d is out of range [0,100], clamped", + parsec_gpu_mem_evict_lower); + parsec_gpu_mem_evict_lower = parsec_gpu_mem_evict_lower < 0 ? 0 : 100; + } + if( parsec_gpu_mem_evict_lower > parsec_gpu_mem_evict_upper ) { + parsec_warning("device_mem_evict_lower=%d > device_mem_evict_upper=%d, swapping", + parsec_gpu_mem_evict_lower, parsec_gpu_mem_evict_upper); + int32_t _tmp = parsec_gpu_mem_evict_lower; + parsec_gpu_mem_evict_lower = parsec_gpu_mem_evict_upper; + parsec_gpu_mem_evict_upper = _tmp; + } (void)parsec_mca_param_reg_int_name("device", "show_capabilities", "Show the detailed devices capabilities", false, false, parsec_debug_verbose >= 4 || (parsec_debug_verbose >= 3 && parsec_debug_rank == 0), NULL); diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index 1f73b4101..c837f444d 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -2711,6 +2711,8 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, parsec_gpu_task_t *_w2r = parsec_gpu_create_w2r_task(gpu_device, es, still_needed, &selected); if( NULL == _w2r ) break; + _w2r->task_type = PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER; + parsec_atomic_fetch_add_int32(&gpu_device->mutex, 1); PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%d:%s]: Proactive D2H writeback: clean LRU empty, " "zone above %d%% threshold; needed %zu, selected %zu bytes", @@ -2730,6 +2732,8 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, parsec_gpu_task_t *_w2r = parsec_gpu_create_w2r_task(gpu_device, es, SIZE_MAX, &selected); if( NULL != _w2r ) { + _w2r->task_type = PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER; + parsec_atomic_fetch_add_int32(&gpu_device->mutex, 1); PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%d:%s]: Proactive D2H writeback: clean LRU empty, selected %zu bytes", gpu_device->super.device_index, gpu_device->super.name, selected); @@ -2867,9 +2871,26 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec)); /* Everything went fine so far, the result is correct and back in the main memory */ PARSEC_LIST_ITEM_SINGLETON(gpu_task); - if (gpu_task->task_type == PARSEC_GPU_TASK_TYPE_D2HTRANSFER) { + if (gpu_task->task_type == PARSEC_GPU_TASK_TYPE_D2HTRANSFER || + gpu_task->task_type == PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER) { + int _proactive = (gpu_task->task_type == PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER); parsec_gpu_complete_w2r_task(gpu_device, gpu_task, es); + /* gpu_task freed inside parsec_gpu_complete_w2r_task */ gpu_task = progress_task; + if( _proactive ) { + /* Proactive task owned a mutex count; release it now. */ + rc = parsec_atomic_fetch_dec_int32( &(gpu_device->mutex) ); + if( 1 == rc ) { /* I was the last one */ +#if defined(PARSEC_PROF_TRACE) + if( gpu_device->trackable_events & PARSEC_PROFILE_GPU_TRACK_OWN ) + PARSEC_PROFILING_TRACE( es->es_profile, parsec_gpu_own_GPU_key_end, + (unsigned long)es, PROFILE_OBJECT_ID_NULL, NULL ); +#endif + PARSEC_DEBUG_VERBOSE(5, parsec_gpu_output_stream, "GPU[%d:%s]: Leaving GPU management", + gpu_device->super.device_index, gpu_device->super.name); + return PARSEC_HOOK_RETURN_ASYNC; + } + } goto fetch_task_from_shared_queue; } if (gpu_task->task_type == PARSEC_GPU_TASK_TYPE_D2D_COMPLETE) { diff --git a/parsec/mca/device/device_gpu.h b/parsec/mca/device/device_gpu.h index 420073c4b..e8786fa82 100644 --- a/parsec/mca/device/device_gpu.h +++ b/parsec/mca/device/device_gpu.h @@ -326,11 +326,12 @@ void parsec_device_enable_debug(void); char *parsec_device_describe_gpu_task( char *tmp, size_t len, parsec_gpu_task_t *gpu_task ); #endif -#define PARSEC_GPU_TASK_TYPE_KERNEL 0x0000 -#define PARSEC_GPU_TASK_TYPE_D2HTRANSFER 0x1000 -#define PARSEC_GPU_TASK_TYPE_PREFETCH 0x2000 -#define PARSEC_GPU_TASK_TYPE_WARMUP 0x4000 -#define PARSEC_GPU_TASK_TYPE_D2D_COMPLETE 0x8000 +#define PARSEC_GPU_TASK_TYPE_KERNEL 0x0000 +#define PARSEC_GPU_TASK_TYPE_D2HTRANSFER 0x1000 +#define PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER 0x1001 /**< Tier-2 proactive D2H: counts in device->mutex */ +#define PARSEC_GPU_TASK_TYPE_PREFETCH 0x2000 +#define PARSEC_GPU_TASK_TYPE_WARMUP 0x4000 +#define PARSEC_GPU_TASK_TYPE_D2D_COMPLETE 0x8000 #if defined(PARSEC_PROF_TRACE) #define PARSEC_PROFILE_GPU_TRACK_DATA_IN 0x0001 diff --git a/parsec/mca/device/transfer_gpu.c b/parsec/mca/device/transfer_gpu.c index 2de7ab6bb..4374adec8 100644 --- a/parsec/mca/device/transfer_gpu.c +++ b/parsec/mca/device/transfer_gpu.c @@ -180,15 +180,9 @@ static const parsec_symbol_t symb_gpu_d2h_task_param = { int32_t parsec_gpu_d2h_max_flows = 0; -/* Proactive eviction thresholds (percentage of total zone capacity). - * Registered as MCA parameters by each GPU backend component. - * mem_evict_upper: initial percentage at which proactive eviction begins (default 95). - * mem_evict_lower: floor to which the per-device threshold may adapt downwards (default 80). - * When a task stalls because no zone memory could be freed, the per-device - * mem_evict_threshold is lowered by 5 points (clamped to mem_evict_lower) so - * future eviction runs start sooner. */ -int32_t parsec_gpu_mem_evict_upper = 95; -int32_t parsec_gpu_mem_evict_lower = 80; +/* parsec_gpu_mem_evict_upper and parsec_gpu_mem_evict_lower are defined in + * device.c so they exist even in non-GPU builds and can be registered as MCA + * parameters unconditionally. */ static const parsec_task_class_t parsec_gpu_d2h_task_class = { .name = "GPU D2H data transfer", @@ -322,7 +316,8 @@ int parsec_gpu_complete_w2r_task(parsec_device_gpu_module_t *gpu_device, PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "D2H[%d:%s] task %p: %d data transferred to host", gpu_device->super.device_index, gpu_device->super.name, (void*)task, task->locals[0].value); - assert(gpu_task->task_type == PARSEC_GPU_TASK_TYPE_D2HTRANSFER); + assert(gpu_task->task_type == PARSEC_GPU_TASK_TYPE_D2HTRANSFER || + gpu_task->task_type == PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER); for( int i = 0; i < task->locals[0].value; i++ ) { gpu_copy = task->data[i].data_out; parsec_atomic_lock(&gpu_copy->original->lock); From 1b834d9dfaa0dd072c454d06848b78af97dd4588 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Thu, 14 May 2026 13:43:42 -0400 Subject: [PATCH 6/7] Only evict from dirty LRU if the device is about to run dry Pushing the inputs of a new task does not mean that we have to immediately evict data. If there are still pending tasks in the fifo we can wait for memory to become available as a result of them completing. This catches cases where we are pushing taks faster than we can execute and building up long fifos. If the fifos run empty we need to react and thus evict data. Once we evict, we adjust the proactive threshold to avoid that in the future. Signed-off-by: Joseph Schuchart --- parsec/mca/device/device_gpu.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index c837f444d..89afb33f6 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -2606,6 +2606,20 @@ parsec_device_kernel_cleanout( parsec_device_gpu_module_t *gpu_device, return 0; } +/** + * Returns false if at least one of the execution stream fifos has pending tasks. + * Otherwise, returns true, meaning that the GPU has no new work to schedule into the stream. + */ +static bool gpu_device_exec_streams_fifo_empty( parsec_device_gpu_module_t *gpu_device ) +{ + for (int i = 2; i < gpu_device->nb_exec_streams; i++) { + if( !parsec_list_nolock_is_empty(gpu_device->exec_streams[i].fifo_pending) ) { + return false; + } + } + return true; +} + /** * This version is based on 4 streams: one for transfers from the memory to * the GPU, 2 for kernel executions and one for transfers from the GPU into @@ -2767,12 +2781,16 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, * Skip if evictions are already in flight to avoid a storm of D2H tasks. * If there are no in-flight evictions and nothing left to queue from the dirty * LRU, we are truly stuck: step the eviction threshold down so tier-1 starts - * freeing pages earlier on the next attempt. */ - if( 0 == gpu_device->mem_evict_in_flight ) { + * freeing pages earlier on the next attempt. + * We don't evict from the dirty LRU if there are pending tasks in the execution streams. + * There is a good chance that memory will become available once the active tasks complete and we still + * have more tasks to execute. */ + if( 0 == gpu_device->mem_evict_in_flight && gpu_device_exec_streams_fifo_empty(gpu_device) ) { size_t _sel = 0; gpu_task = parsec_gpu_create_w2r_task(gpu_device, es, SIZE_MAX, &_sel); if(gpu_device->mem_evict_threshold - 5 >= parsec_gpu_mem_evict_lower) { - /* We had to trigger a proactive D2H writeback so reduce the threshold */ + /* We had to trigger a reactive D2H writeback so reduce the threshold to + * be more aggressive in the proactive part. */ gpu_device->mem_evict_threshold -= 5; } } From a22c93ccb2bbd19e670ad6f98678e88ce078479d Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 8 Jun 2026 19:46:58 -0400 Subject: [PATCH 7/7] Fix typos Signed-off-by: Joseph Schuchart --- parsec/mca/device/device_gpu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index 4997f32b0..a0e5e01b7 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -3478,8 +3478,8 @@ parsec_device_kernel_cleanout( parsec_device_gpu_module_t *gpu_device, */ static bool gpu_device_exec_streams_fifo_empty( parsec_device_gpu_module_t *gpu_device ) { - for (int i = 2; i < gpu_device->nb_exec_streams; i++) { - if( !parsec_list_nolock_is_empty(gpu_device->exec_streams[i].fifo_pending) ) { + for (int i = 2; i < gpu_device->num_exec_streams; i++) { + if( !parsec_list_nolock_is_empty(gpu_device->exec_stream[i].fifo_pending) ) { return false; } }