diff --git a/src/asahi/vulkan/hk_device_memory.c b/src/asahi/vulkan/hk_device_memory.c index 35fdcae4c706..4120a267e564 100644 --- a/src/asahi/vulkan/hk_device_memory.c +++ b/src/asahi/vulkan/hk_device_memory.c @@ -15,10 +15,57 @@ #include "util/u_atomic.h" #include "util/u_debug.h" #include "vk_common_entrypoints.h" +#include "util/os_misc.h" #include #include +/* GPU memory is system memory that the kernel cannot evict or swap, and + * heapBudget is only advisory, so the heap limit alone lets a process (or + * several) take every free page and invoke the OOM killer on the desktop. + * Refuse an allocation of 1 MiB or more that would leave less than + * max(1 GiB, 2% of RAM) available. Applications that allocate per tensor + * (tens of MiB per call) are covered. Smaller allocations always pass: + * below the reserve every vkAllocateMemory would fail, including ones the + * compositor needs. + */ +#define HK_MEMORY_RESERVE_MIN (1ull << 30) +#define HK_MEMORY_CHECK_MIN (1ull << 20) + +static bool +hk_system_memory_fits_now(uint64_t size) +{ + uint64_t available, total; + + /* If the kernel does not tell us, only the heap limit applies. */ + if (!os_get_available_system_memory(&available)) + return true; + + uint64_t reserve = HK_MEMORY_RESERVE_MIN; + if (os_get_total_physical_memory(&total)) + reserve = MAX2(reserve, total / 50); + + return available >= size + reserve; +} + +static bool +hk_system_memory_fits(struct hk_device *dev, uint64_t size) +{ + if (size < HK_MEMORY_CHECK_MIN) + return true; + + if (hk_system_memory_fits_now(size)) + return true; + + /* Freed device memory is parked in the driver's BO cache and still counts + * as used for MemAvailable. The cache is only trimmed when another BO is + * freed or an allocation fails, so refusing here would keep an application + * that just freed gigabytes from reusing them. Give the cache back first. + */ + agx_bo_cache_evict_all(&dev->dev); + return hk_system_memory_fits_now(size); +} + /* Supports opaque fd only */ const VkExternalMemoryProperties hk_opaque_fd_mem_props = { .externalMemoryFeatures = VK_EXTERNAL_MEMORY_FEATURE_EXPORTABLE_BIT | @@ -226,6 +273,13 @@ hk_AllocateMemory(VkDevice device, const VkMemoryAllocateInfo *pAllocateInfo, if (type->propertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) flags |= AGX_BO_WRITEBACK; + if (!hk_system_memory_fits(dev, aligned_size)) { + result = vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, + "Not enough free system memory for %" PRIu64 + " MiB", aligned_size >> 20); + goto fail_alloc; + } + mem->bo = agx_bo_create(&dev->dev, aligned_size, 0, flags, "App memory"); if (!mem->bo) { result = vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY); diff --git a/src/asahi/vulkan/meson.build b/src/asahi/vulkan/meson.build index 70d395afaf8a..7e5dadd7c250 100644 --- a/src/asahi/vulkan/meson.build +++ b/src/asahi/vulkan/meson.build @@ -417,7 +417,9 @@ if with_tests link_args : [ld_args_gc_sections, hk_wrap_test_link_args, '-Wl,--wrap=agx_bo_bind', '-Wl,--wrap=agx_bo_cache_evict_all', '-Wl,--wrap=agx_bo_export', - '-Wl,--wrap=hk_get_cgroup_memory_available', '-Wl,--wrap=__vk_errorf']), + '-Wl,--wrap=hk_get_cgroup_memory_available', '-Wl,--wrap=__vk_errorf', + '-Wl,--wrap=os_get_available_system_memory', + '-Wl,--wrap=os_get_total_physical_memory']), suite : ['asahi']) test('hk_tes_query', executable('hk_tes_query_test', files('tests/test-hk-tes-query.c'), include_directories : [inc_gallium, inc_gallium_aux, inc_include, inc_src, diff --git a/src/asahi/vulkan/tests/test-hk-memory-oom.c b/src/asahi/vulkan/tests/test-hk-memory-oom.c index c23a0a2043c2..64045b91b449 100644 --- a/src/asahi/vulkan/tests/test-hk-memory-oom.c +++ b/src/asahi/vulkan/tests/test-hk-memory-oom.c @@ -12,6 +12,7 @@ #include "wsi_common_private.h" #include "wsi_common_entrypoints.h" #include "drm-uapi/drm_fourcc.h" +#include "util/os_misc.h" #include #include @@ -38,6 +39,28 @@ __typeof__(agx_bo_cache_evict_all) __real_agx_bo_cache_evict_all; __typeof__(agx_bo_export) __wrap_agx_bo_export; __typeof__(hk_get_cgroup_memory_available) __wrap_hk_get_cgroup_memory_available; __typeof__(__vk_errorf) __wrap___vk_errorf; +__typeof__(os_get_available_system_memory) __wrap_os_get_available_system_memory; +__typeof__(os_get_total_physical_memory) __wrap_os_get_total_physical_memory; + +/* The reserve check reads real RAM in production. Fixed values keep this test + * independent of the machine it runs on. */ +static uint64_t fake_available = UINT64_C(64) * 1024 * MIB; +static uint64_t fake_total = UINT64_C(32) * 1024 * MIB; +static bool evict_frees_memory; + +bool +__wrap_os_get_available_system_memory(uint64_t *size) +{ + *size = fake_available; + return true; +} + +bool +__wrap_os_get_total_physical_memory(uint64_t *size) +{ + *size = fake_total; + return true; +} /* Mock only backend allocation/close and VM_BIND. BO cache insertion, fetch, * reference counting, VA release and WSI object cleanup use production code. */ @@ -95,6 +118,8 @@ void __wrap_agx_bo_cache_evict_all(struct agx_device *dev) { ++trims; + if (evict_frees_memory) + fake_available = UINT64_C(64) * 1024 * MIB; cached = 0; __real_agx_bo_cache_evict_all(dev); } @@ -450,6 +475,29 @@ main(void) passing = 4; assert(alloc(dev, 1024 * MIB, &memory) == VK_SUCCESS); hk_FreeMemory(hk_device_to_handle(dev), memory, &allocator); + assert(!pinned && !host_live && !kernel_oom_attempts); + + /* The free-memory reserve is max(1 GiB, 2% of RAM), 1 GiB for 32 GiB of + * RAM. One byte short of size plus reserve is refused before any BO is + * allocated, and the BO cache is given back first. Exactly enough passes. */ + fake_available = 2048 * MIB - 1; + before = bo_calls; + old_trims = trims; + assert(alloc(dev, 1024 * MIB, &memory) == VK_ERROR_OUT_OF_DEVICE_MEMORY); + assert(bo_calls == before && trims == old_trims + 1); + fake_available = 2048 * MIB; + assert(alloc(dev, 1024 * MIB, &memory) == VK_SUCCESS); + hk_FreeMemory(hk_device_to_handle(dev), memory, &allocator); + + /* Giving the BO cache back raises the available memory: the retry passes. */ + fake_available = 1024 * MIB; + evict_frees_memory = true; + old_trims = trims; + assert(alloc(dev, 1024 * MIB, &memory) == VK_SUCCESS); + assert(trims == old_trims + 1); + hk_FreeMemory(hk_device_to_handle(dev), memory, &allocator); + evict_frees_memory = false; + assert(!pinned && !host_live && !kernel_oom_attempts); wsi_headless_finish_wsi(&pdev->wsi_device, vk_default_allocator()); vk_object_base_finish(&dev->vk.base); diff --git a/src/compiler/spirv/spirv_to_nir.c b/src/compiler/spirv/spirv_to_nir.c index 09b26dac15a5..4536774d6bf0 100644 --- a/src/compiler/spirv/spirv_to_nir.c +++ b/src/compiler/spirv/spirv_to_nir.c @@ -4856,6 +4856,21 @@ vtn_handle_image(struct vtn_builder *b, SpvOp opcode, SpvMemorySemanticsMask after_semantics; vtn_split_barrier_semantics(b, semantics, &before_semantics, &after_semantics); + /* MakeTexelAvailable and MakeTexelVisible apply to the texel of this very + * access, unlike MakeAvailable on a release atomic. The split above puts + * MakeAvailable before the operation and MakeVisible after it, which is + * right for atomics but leaves a MakeTexelAvailable write outside its own + * availability operation and a MakeTexelVisible read unordered after it. + * Image reads and writes carry no Acquire/Release bits, so swap the sides: + * a write is made available after it, a read is made visible before it. + */ + if (opcode == SpvOpImageRead || opcode == SpvOpImageSparseRead || + opcode == SpvOpImageWrite) { + SpvMemorySemanticsMask tmp = before_semantics; + before_semantics = after_semantics; + after_semantics = tmp; + } + if (before_semantics) vtn_emit_memory_barrier(b, scope, before_semantics); diff --git a/src/compiler/spirv/tests/avail_vis.cpp b/src/compiler/spirv/tests/avail_vis.cpp index 922694f0678f..151729abc196 100644 --- a/src/compiler/spirv/tests/avail_vis.cpp +++ b/src/compiler/spirv/tests/avail_vis.cpp @@ -134,6 +134,167 @@ TEST_F(AvailabilityVisibility, opstore_avail) EXPECT_EQ(nir_intrinsic_execution_scope(intrinsic), SCOPE_NONE); } +/* Position of the first intrinsic of the given op in the entry point, or -1. */ +static int +intrinsic_position(nir_shader *shader, nir_intrinsic_op op) +{ + int pos = 0; + nir_function_impl *impl = nir_shader_get_entrypoint(shader); + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type == nir_instr_type_intrinsic && + nir_instr_as_intrinsic(instr)->intrinsic == op) + return pos; + pos++; + } + } + return -1; +} + +TEST_F(AvailabilityVisibility, opimageread_vis) +{ + /* + OpCapability Shader + OpCapability VulkanMemoryModel + OpCapability VulkanMemoryModelDeviceScope + OpMemoryModel Logical Vulkan + OpEntryPoint GLCompute %main "main" %img + OpExecutionMode %main LocalSize 1 1 1 + OpDecorate %img DescriptorSet 0 + OpDecorate %img Binding 0 + %void = OpTypeVoid + %fn = OpTypeFunction %void + %uint = OpTypeInt 32 0 + %int = OpTypeInt 32 1 + %v2int = OpTypeVector %int 2 + %v4uint = OpTypeVector %uint 4 + %imgtype = OpTypeImage %uint 2D 0 0 0 2 R32ui + %ptr = OpTypePointer UniformConstant %imgtype + %img = OpVariable %ptr UniformConstant + %int_0 = OpConstant %int 0 + %coord = OpConstantComposite %v2int %int_0 %int_0 + %device = OpConstant %int 1 + %zero = OpConstantNull %v4uint + %main = OpFunction %void None %fn + %l = OpLabel + %im = OpLoad %imgtype %img + %v = OpImageRead %v4uint %im %coord MakeTexelVisible|NonPrivateTexel %device + OpImageWrite %im %coord %v + OpReturn + OpFunctionEnd + */ + static const uint32_t words[] = { + 0x07230203, 0x00010500, 0x00070000, 0x00000012, 0x00000000, 0x00020011, + 0x00000001, 0x00020011, 0x000014e1, 0x00020011, 0x000014e2, 0x0003000e, + 0x00000000, 0x00000003, 0x0006000f, 0x00000005, 0x00000001, 0x6e69616d, + 0x00000000, 0x00000002, 0x00060010, 0x00000001, 0x00000011, 0x00000001, + 0x00000001, 0x00000001, 0x00040047, 0x00000002, 0x00000022, 0x00000000, + 0x00040047, 0x00000002, 0x00000021, 0x00000000, 0x00020013, 0x00000003, + 0x00030021, 0x00000004, 0x00000003, 0x00040015, 0x00000005, 0x00000020, + 0x00000000, 0x00040015, 0x00000006, 0x00000020, 0x00000001, 0x00040017, + 0x00000007, 0x00000006, 0x00000002, 0x00040017, 0x00000008, 0x00000005, + 0x00000004, 0x00090019, 0x00000009, 0x00000005, 0x00000001, 0x00000000, + 0x00000000, 0x00000000, 0x00000002, 0x00000021, 0x00040020, 0x0000000a, + 0x00000000, 0x00000009, 0x0004003b, 0x0000000a, 0x00000002, 0x00000000, + 0x0004002b, 0x00000006, 0x0000000b, 0x00000000, 0x0005002c, 0x00000007, + 0x0000000c, 0x0000000b, 0x0000000b, 0x0004002b, 0x00000006, 0x0000000d, + 0x00000001, 0x0003002e, 0x00000008, 0x0000000e, 0x00050036, 0x00000003, + 0x00000001, 0x00000000, 0x00000004, 0x000200f8, 0x0000000f, 0x0004003d, + 0x00000009, 0x00000010, 0x00000002, 0x00070062, 0x00000008, 0x00000011, + 0x00000010, 0x0000000c, 0x00000600, 0x0000000d, 0x00040063, 0x00000010, + 0x0000000c, 0x00000011, 0x000100fd, 0x00010038, + }; + + get_nir(sizeof(words) / sizeof(words[0]), words); + + nir_intrinsic_instr *intrinsic = find_intrinsic(nir_intrinsic_barrier, 0); + ASSERT_NE(intrinsic, nullptr); + + EXPECT_EQ(nir_intrinsic_memory_semantics(intrinsic), NIR_MEMORY_MAKE_VISIBLE); + EXPECT_NE(nir_intrinsic_memory_modes(intrinsic) & nir_var_image, 0); + EXPECT_EQ(nir_intrinsic_memory_scope(intrinsic), SCOPE_DEVICE); + EXPECT_EQ(nir_intrinsic_execution_scope(intrinsic), SCOPE_NONE); + + /* MakeTexelVisible makes the texel visible before it is read. */ + int barrier = intrinsic_position(shader, nir_intrinsic_barrier); + int load = intrinsic_position(shader, nir_intrinsic_image_deref_load); + ASSERT_GE(barrier, 0); + ASSERT_GE(load, 0); + EXPECT_LT(barrier, load); +} + +TEST_F(AvailabilityVisibility, opimagewrite_avail) +{ + /* + OpCapability Shader + OpCapability VulkanMemoryModel + OpCapability VulkanMemoryModelDeviceScope + OpMemoryModel Logical Vulkan + OpEntryPoint GLCompute %main "main" %img + OpExecutionMode %main LocalSize 1 1 1 + OpDecorate %img DescriptorSet 0 + OpDecorate %img Binding 0 + %void = OpTypeVoid + %fn = OpTypeFunction %void + %uint = OpTypeInt 32 0 + %int = OpTypeInt 32 1 + %v2int = OpTypeVector %int 2 + %v4uint = OpTypeVector %uint 4 + %imgtype = OpTypeImage %uint 2D 0 0 0 2 R32ui + %ptr = OpTypePointer UniformConstant %imgtype + %img = OpVariable %ptr UniformConstant + %int_0 = OpConstant %int 0 + %coord = OpConstantComposite %v2int %int_0 %int_0 + %device = OpConstant %int 1 + %zero = OpConstantNull %v4uint + %main = OpFunction %void None %fn + %l = OpLabel + %im = OpLoad %imgtype %img + %v = OpImageRead %v4uint %im %coord + OpImageWrite %im %coord %v MakeTexelAvailable|NonPrivateTexel %device + OpReturn + OpFunctionEnd + */ + static const uint32_t words[] = { + 0x07230203, 0x00010500, 0x00070000, 0x00000012, 0x00000000, 0x00020011, + 0x00000001, 0x00020011, 0x000014e1, 0x00020011, 0x000014e2, 0x0003000e, + 0x00000000, 0x00000003, 0x0006000f, 0x00000005, 0x00000001, 0x6e69616d, + 0x00000000, 0x00000002, 0x00060010, 0x00000001, 0x00000011, 0x00000001, + 0x00000001, 0x00000001, 0x00040047, 0x00000002, 0x00000022, 0x00000000, + 0x00040047, 0x00000002, 0x00000021, 0x00000000, 0x00020013, 0x00000003, + 0x00030021, 0x00000004, 0x00000003, 0x00040015, 0x00000005, 0x00000020, + 0x00000000, 0x00040015, 0x00000006, 0x00000020, 0x00000001, 0x00040017, + 0x00000007, 0x00000006, 0x00000002, 0x00040017, 0x00000008, 0x00000005, + 0x00000004, 0x00090019, 0x00000009, 0x00000005, 0x00000001, 0x00000000, + 0x00000000, 0x00000000, 0x00000002, 0x00000021, 0x00040020, 0x0000000a, + 0x00000000, 0x00000009, 0x0004003b, 0x0000000a, 0x00000002, 0x00000000, + 0x0004002b, 0x00000006, 0x0000000b, 0x00000000, 0x0005002c, 0x00000007, + 0x0000000c, 0x0000000b, 0x0000000b, 0x0004002b, 0x00000006, 0x0000000d, + 0x00000001, 0x0003002e, 0x00000008, 0x0000000e, 0x00050036, 0x00000003, + 0x00000001, 0x00000000, 0x00000004, 0x000200f8, 0x0000000f, 0x0004003d, + 0x00000009, 0x00000010, 0x00000002, 0x00050062, 0x00000008, 0x00000011, + 0x00000010, 0x0000000c, 0x00060063, 0x00000010, 0x0000000c, 0x00000011, + 0x00000500, 0x0000000d, 0x000100fd, 0x00010038, + }; + + get_nir(sizeof(words) / sizeof(words[0]), words); + + nir_intrinsic_instr *intrinsic = find_intrinsic(nir_intrinsic_barrier, 0); + ASSERT_NE(intrinsic, nullptr); + + EXPECT_EQ(nir_intrinsic_memory_semantics(intrinsic), NIR_MEMORY_MAKE_AVAILABLE); + EXPECT_NE(nir_intrinsic_memory_modes(intrinsic) & nir_var_image, 0); + EXPECT_EQ(nir_intrinsic_memory_scope(intrinsic), SCOPE_DEVICE); + EXPECT_EQ(nir_intrinsic_execution_scope(intrinsic), SCOPE_NONE); + + /* MakeTexelAvailable makes the texel available after it is written. */ + int barrier = intrinsic_position(shader, nir_intrinsic_barrier); + int store = intrinsic_position(shader, nir_intrinsic_image_deref_store); + ASSERT_GE(barrier, 0); + ASSERT_GE(store, 0); + EXPECT_GT(barrier, store); +} + TEST_F(AvailabilityVisibility, opcopymemory_visavail_both_combined) { /*