From 057afe2f70bfb46bee08f923e9294fd357e03403 Mon Sep 17 00:00:00 2001 From: wolfchimneyrock <12381503+wolfchimneyrock@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:01:00 -0500 Subject: [PATCH] vulkan: release command buffer memory at each begin on Arm's driver libmali keeps the memory vkCmdBeginRendering allocates across the implicit reset of vkBeginCommandBuffer, so the process grew by ~490 MB an hour. Reset with RELEASE_RESOURCES before each begin on VK_DRIVER_ID_ARM_PROPRIETARY; LARECOMP_VK_CB_RELEASE=0/1 overrides. Co-Authored-By: Claude Opus 5.5 --- docs/vulkan-gpu-defaults.md | 1 + runtime/nvn/vulkan/device.c | 6 ++++++ runtime/nvn/vulkan/record.c | 5 ++++- runtime/nvn/vulkan/vk.h | 1 + 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/vulkan-gpu-defaults.md b/docs/vulkan-gpu-defaults.md index 4f07462e0..ccf491ab1 100644 --- a/docs/vulkan-gpu-defaults.md +++ b/docs/vulkan-gpu-defaults.md @@ -15,6 +15,7 @@ Vendor IDs: 0x13B5 Arm (Mali), 0x5143 Qualcomm (Adreno), 0x8086 Intel, 0x1002 AM | Structured shaders (static branch stacks, no jump loop) | Mali, Adreno | PanVK (RK3588), Turnip | #562 | `LARECOMP_VK_STRUCTURE` | | c1 inlined, hot constant buffer words as push constants | Mali, Adreno | PanVK; Turnip (Adreno 643) | #563, #579 | `LARECOMP_VK_PUSH_CB`, `LARECOMP_VK_INLINE_C1` | | Split pass barriers | Mali | PanVK | | `LARECOMP_VK_SPLIT_BARRIER` | +| Command buffers release their memory at each begin (a memory leak without it) | Arm's driver (not PanVK) | libmali g29p1 (Mali-G610, RK3588) | | `LARECOMP_VK_CB_RELEASE` | | Branch stacks as variables instead of arrays | Intel | | #462 | `LARECOMP_VK_STACK_ARRAYS=1` gives arrays | | Placeholder colour attachment for depth-only passes | Intel | UHD 630 on Windows (a driver crash without it, #8, #46) | | `LARECOMP_VK_PLACEHOLDER_COLOR` | | Guest heap as driver memory | RADV, PanVK | PanVK (Mali-G610), RADV | #564, #483 | `LARECOMP_VK_HEAP_DMABUF` | diff --git a/runtime/nvn/vulkan/device.c b/runtime/nvn/vulkan/device.c index 2c13da864..d4ba7ce81 100644 --- a/runtime/nvn/vulkan/device.c +++ b/runtime/nvn/vulkan/device.c @@ -693,6 +693,12 @@ static void create_device(void) { // issue #483: an 81-minute run without faults, the Field 15 -> 21 fps, ~85% fewer uploads). PanVK's device name // is the GPU's ("Mali-G610 MC4"), hence the driver ID. g_vk.heap_dmabuf_default = strstr(g_vk.props.deviceName, "RADV") != NULL || drv.driverID == VK_DRIVER_ID_MESA_PANVK; + // command buffers released before each begin (record.c begin_cb_run) by default on Arm's own driver: the implicit + // reset of vkBeginCommandBuffer keeps the memory vkCmdBeginRendering allocates (64 KB blocks of host memory), so + // the process grew by ~490 MB an hour, until it ran out of memory (libmali g29p1 on RK3588; PanVK and Turnip don't). + const char *cbr = getenv("LARECOMP_VK_CB_RELEASE"); + g_vk.cb_release = cbr ? atoi(cbr) != 0 : drv.driverID == VK_DRIVER_ID_ARM_PROPRIETARY; + if (g_vk.cb_release) LOG("[vk] command buffers release their memory at each begin%s\n", cbr ? " (LARECOMP_VK_CB_RELEASE)" : ": Arm's driver"); #ifdef VK_EXT_graphics_pipeline_library if (gpl_ext) snprintf(gpl, sizeof gpl, "%s (fast linking %s, independent interpolation decoration %s)", diff --git a/runtime/nvn/vulkan/record.c b/runtime/nvn/vulkan/record.c index ede9710e7..522d34a6f 100644 --- a/runtime/nvn/vulkan/record.c +++ b/runtime/nvn/vulkan/record.c @@ -534,8 +534,11 @@ void rec_UpdateDescriptorSets(uint32_t n, const VkWriteDescriptorSet *w) { // ================================================================== command buffers, submissions, presents static void begin_cb_run(const void *p) { + VkCommandBuffer cb = *(const VkCommandBuffer *)p; + // (g_vk.cb_release, device.c: Arm's driver keeps vkCmdBeginRendering's memory across the implicit reset below) + if (g_vk.cb_release) VK_CHECK(vkResetCommandBuffer(cb, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT)); VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; - VK_CHECK(vkBeginCommandBuffer(*(const VkCommandBuffer *)p, &bi)); + VK_CHECK(vkBeginCommandBuffer(cb, &bi)); } void rec_BeginCommandBuffer(VkCommandBuffer cb) { if (on <= 0) { begin_cb_run(&cb); return; } diff --git a/runtime/nvn/vulkan/vk.h b/runtime/nvn/vulkan/vk.h index 6ec30b247..d24b92a3f 100644 --- a/runtime/nvn/vulkan/vk.h +++ b/runtime/nvn/vulkan/vk.h @@ -44,6 +44,7 @@ typedef struct { bool gpl_probe; bool map_placed, unmap_reserve; // VK_EXT_map_memory_placed: driver memory mapped at a given address (exec.c guest_heap_dmabuf) bool heap_dmabuf_default; // the heap as driver memory by default: RADV and Mesa's PanVK (device.c; LARECOMP_VK_HEAP_DMABUF overrides) + bool cb_release; // command buffers reset with RELEASE_RESOURCES before each begin: Arm's driver (device.c; LARECOMP_VK_CB_RELEASE) bool ssbo_chunks; // stores and atomics from fragment and vertex shaders (exec.c: the heap in chunks counts misses) bool lazy_heap, bda; // the heap imported chunk by chunk (device.c lazy_heap_predicted), buffer device addresses enabled for it bool heap_probe; // one import tried at the start, lazy chunks if it runs out of memory (exec.c whole_import_probe_oom)