Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions src/asahi/vulkan/hk_device_memory.c
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,57 @@
#include "util/u_atomic.h"
#include "util/u_debug.h"
#include "vk_common_entrypoints.h"
#include "util/os_misc.h"

#include <inttypes.h>
#include <sys/mman.h>

/* GPU memory is system memory that the kernel cannot evict or swap, and
* heapBudget is only advisory, so the heap limit alone lets a process (or
* several) take every free page and invoke the OOM killer on the desktop.
* Refuse an allocation of 1 MiB or more that would leave less than
* max(1 GiB, 2% of RAM) available. Applications that allocate per tensor
* (tens of MiB per call) are covered. Smaller allocations always pass:
* below the reserve every vkAllocateMemory would fail, including ones the
* compositor needs.
*/
#define HK_MEMORY_RESERVE_MIN (1ull << 30)
#define HK_MEMORY_CHECK_MIN (1ull << 20)

static bool
hk_system_memory_fits_now(uint64_t size)
{
uint64_t available, total;

/* If the kernel does not tell us, only the heap limit applies. */
if (!os_get_available_system_memory(&available))
return true;

uint64_t reserve = HK_MEMORY_RESERVE_MIN;
if (os_get_total_physical_memory(&total))
reserve = MAX2(reserve, total / 50);

return available >= size + reserve;
}

static bool
hk_system_memory_fits(struct hk_device *dev, uint64_t size)
{
if (size < HK_MEMORY_CHECK_MIN)
return true;

if (hk_system_memory_fits_now(size))
return true;

/* Freed device memory is parked in the driver's BO cache and still counts
* as used for MemAvailable. The cache is only trimmed when another BO is
* freed or an allocation fails, so refusing here would keep an application
* that just freed gigabytes from reusing them. Give the cache back first.
*/
agx_bo_cache_evict_all(&dev->dev);
return hk_system_memory_fits_now(size);
}

/* Supports opaque fd only */
const VkExternalMemoryProperties hk_opaque_fd_mem_props = {
.externalMemoryFeatures = VK_EXTERNAL_MEMORY_FEATURE_EXPORTABLE_BIT |
Expand Down Expand Up @@ -226,6 +273,13 @@ hk_AllocateMemory(VkDevice device, const VkMemoryAllocateInfo *pAllocateInfo,
if (type->propertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT)
flags |= AGX_BO_WRITEBACK;

if (!hk_system_memory_fits(dev, aligned_size)) {
result = vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY,
"Not enough free system memory for %" PRIu64
" MiB", aligned_size >> 20);
goto fail_alloc;
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
}

mem->bo = agx_bo_create(&dev->dev, aligned_size, 0, flags, "App memory");
if (!mem->bo) {
result = vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY);
Expand Down
4 changes: 3 additions & 1 deletion src/asahi/vulkan/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,9 @@ if with_tests
link_args : [ld_args_gc_sections, hk_wrap_test_link_args,
'-Wl,--wrap=agx_bo_bind', '-Wl,--wrap=agx_bo_cache_evict_all',
'-Wl,--wrap=agx_bo_export',
'-Wl,--wrap=hk_get_cgroup_memory_available', '-Wl,--wrap=__vk_errorf']),
'-Wl,--wrap=hk_get_cgroup_memory_available', '-Wl,--wrap=__vk_errorf',
'-Wl,--wrap=os_get_available_system_memory',
'-Wl,--wrap=os_get_total_physical_memory']),
suite : ['asahi'])
test('hk_tes_query', executable('hk_tes_query_test', files('tests/test-hk-tes-query.c'),
include_directories : [inc_gallium, inc_gallium_aux, inc_include, inc_src,
Expand Down
48 changes: 48 additions & 0 deletions src/asahi/vulkan/tests/test-hk-memory-oom.c
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include "wsi_common_private.h"
#include "wsi_common_entrypoints.h"
#include "drm-uapi/drm_fourcc.h"
#include "util/os_misc.h"

#include <assert.h>
#include <stdarg.h>
Expand All @@ -38,6 +39,28 @@ __typeof__(agx_bo_cache_evict_all) __real_agx_bo_cache_evict_all;
__typeof__(agx_bo_export) __wrap_agx_bo_export;
__typeof__(hk_get_cgroup_memory_available) __wrap_hk_get_cgroup_memory_available;
__typeof__(__vk_errorf) __wrap___vk_errorf;
__typeof__(os_get_available_system_memory) __wrap_os_get_available_system_memory;
__typeof__(os_get_total_physical_memory) __wrap_os_get_total_physical_memory;

/* The reserve check reads real RAM in production. Fixed values keep this test
* independent of the machine it runs on. */
static uint64_t fake_available = UINT64_C(64) * 1024 * MIB;
static uint64_t fake_total = UINT64_C(32) * 1024 * MIB;
static bool evict_frees_memory;

bool
__wrap_os_get_available_system_memory(uint64_t *size)
{
*size = fake_available;
return true;
}

bool
__wrap_os_get_total_physical_memory(uint64_t *size)
{
*size = fake_total;
return true;
}

/* Mock only backend allocation/close and VM_BIND. BO cache insertion, fetch,
* reference counting, VA release and WSI object cleanup use production code. */
Expand Down Expand Up @@ -95,6 +118,8 @@ void
__wrap_agx_bo_cache_evict_all(struct agx_device *dev)
{
++trims;
if (evict_frees_memory)
fake_available = UINT64_C(64) * 1024 * MIB;
cached = 0;
__real_agx_bo_cache_evict_all(dev);
}
Expand Down Expand Up @@ -450,6 +475,29 @@ main(void)
passing = 4;
assert(alloc(dev, 1024 * MIB, &memory) == VK_SUCCESS);
hk_FreeMemory(hk_device_to_handle(dev), memory, &allocator);
assert(!pinned && !host_live && !kernel_oom_attempts);

/* The free-memory reserve is max(1 GiB, 2% of RAM), 1 GiB for 32 GiB of
* RAM. One byte short of size plus reserve is refused before any BO is
* allocated, and the BO cache is given back first. Exactly enough passes. */
fake_available = 2048 * MIB - 1;
before = bo_calls;
old_trims = trims;
assert(alloc(dev, 1024 * MIB, &memory) == VK_ERROR_OUT_OF_DEVICE_MEMORY);
assert(bo_calls == before && trims == old_trims + 1);
fake_available = 2048 * MIB;
assert(alloc(dev, 1024 * MIB, &memory) == VK_SUCCESS);
hk_FreeMemory(hk_device_to_handle(dev), memory, &allocator);

/* Giving the BO cache back raises the available memory: the retry passes. */
fake_available = 1024 * MIB;
evict_frees_memory = true;
old_trims = trims;
assert(alloc(dev, 1024 * MIB, &memory) == VK_SUCCESS);
assert(trims == old_trims + 1);
hk_FreeMemory(hk_device_to_handle(dev), memory, &allocator);
evict_frees_memory = false;

assert(!pinned && !host_live && !kernel_oom_attempts);
wsi_headless_finish_wsi(&pdev->wsi_device, vk_default_allocator());
vk_object_base_finish(&dev->vk.base);
Expand Down
15 changes: 15 additions & 0 deletions src/compiler/spirv/spirv_to_nir.c
Original file line number Diff line number Diff line change
Expand Up @@ -4856,6 +4856,21 @@ vtn_handle_image(struct vtn_builder *b, SpvOp opcode,
SpvMemorySemanticsMask after_semantics;
vtn_split_barrier_semantics(b, semantics, &before_semantics, &after_semantics);

/* MakeTexelAvailable and MakeTexelVisible apply to the texel of this very
* access, unlike MakeAvailable on a release atomic. The split above puts
* MakeAvailable before the operation and MakeVisible after it, which is
* right for atomics but leaves a MakeTexelAvailable write outside its own
* availability operation and a MakeTexelVisible read unordered after it.
* Image reads and writes carry no Acquire/Release bits, so swap the sides:
* a write is made available after it, a read is made visible before it.
*/
if (opcode == SpvOpImageRead || opcode == SpvOpImageSparseRead ||
opcode == SpvOpImageWrite) {
SpvMemorySemanticsMask tmp = before_semantics;
before_semantics = after_semantics;
after_semantics = tmp;
}

if (before_semantics)
vtn_emit_memory_barrier(b, scope, before_semantics);

Expand Down
161 changes: 161 additions & 0 deletions src/compiler/spirv/tests/avail_vis.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,167 @@ TEST_F(AvailabilityVisibility, opstore_avail)
EXPECT_EQ(nir_intrinsic_execution_scope(intrinsic), SCOPE_NONE);
}

/* Position of the first intrinsic of the given op in the entry point, or -1. */
static int
intrinsic_position(nir_shader *shader, nir_intrinsic_op op)
{
int pos = 0;
nir_function_impl *impl = nir_shader_get_entrypoint(shader);
nir_foreach_block(block, impl) {
nir_foreach_instr(instr, block) {
if (instr->type == nir_instr_type_intrinsic &&
nir_instr_as_intrinsic(instr)->intrinsic == op)
return pos;
pos++;
}
}
return -1;
}

TEST_F(AvailabilityVisibility, opimageread_vis)
{
/*
OpCapability Shader
OpCapability VulkanMemoryModel
OpCapability VulkanMemoryModelDeviceScope
OpMemoryModel Logical Vulkan
OpEntryPoint GLCompute %main "main" %img
OpExecutionMode %main LocalSize 1 1 1
OpDecorate %img DescriptorSet 0
OpDecorate %img Binding 0
%void = OpTypeVoid
%fn = OpTypeFunction %void
%uint = OpTypeInt 32 0
%int = OpTypeInt 32 1
%v2int = OpTypeVector %int 2
%v4uint = OpTypeVector %uint 4
%imgtype = OpTypeImage %uint 2D 0 0 0 2 R32ui
%ptr = OpTypePointer UniformConstant %imgtype
%img = OpVariable %ptr UniformConstant
%int_0 = OpConstant %int 0
%coord = OpConstantComposite %v2int %int_0 %int_0
%device = OpConstant %int 1
%zero = OpConstantNull %v4uint
%main = OpFunction %void None %fn
%l = OpLabel
%im = OpLoad %imgtype %img
%v = OpImageRead %v4uint %im %coord MakeTexelVisible|NonPrivateTexel %device
OpImageWrite %im %coord %v
OpReturn
OpFunctionEnd
*/
static const uint32_t words[] = {
0x07230203, 0x00010500, 0x00070000, 0x00000012, 0x00000000, 0x00020011,
0x00000001, 0x00020011, 0x000014e1, 0x00020011, 0x000014e2, 0x0003000e,
0x00000000, 0x00000003, 0x0006000f, 0x00000005, 0x00000001, 0x6e69616d,
0x00000000, 0x00000002, 0x00060010, 0x00000001, 0x00000011, 0x00000001,
0x00000001, 0x00000001, 0x00040047, 0x00000002, 0x00000022, 0x00000000,
0x00040047, 0x00000002, 0x00000021, 0x00000000, 0x00020013, 0x00000003,
0x00030021, 0x00000004, 0x00000003, 0x00040015, 0x00000005, 0x00000020,
0x00000000, 0x00040015, 0x00000006, 0x00000020, 0x00000001, 0x00040017,
0x00000007, 0x00000006, 0x00000002, 0x00040017, 0x00000008, 0x00000005,
0x00000004, 0x00090019, 0x00000009, 0x00000005, 0x00000001, 0x00000000,
0x00000000, 0x00000000, 0x00000002, 0x00000021, 0x00040020, 0x0000000a,
0x00000000, 0x00000009, 0x0004003b, 0x0000000a, 0x00000002, 0x00000000,
0x0004002b, 0x00000006, 0x0000000b, 0x00000000, 0x0005002c, 0x00000007,
0x0000000c, 0x0000000b, 0x0000000b, 0x0004002b, 0x00000006, 0x0000000d,
0x00000001, 0x0003002e, 0x00000008, 0x0000000e, 0x00050036, 0x00000003,
0x00000001, 0x00000000, 0x00000004, 0x000200f8, 0x0000000f, 0x0004003d,
0x00000009, 0x00000010, 0x00000002, 0x00070062, 0x00000008, 0x00000011,
0x00000010, 0x0000000c, 0x00000600, 0x0000000d, 0x00040063, 0x00000010,
0x0000000c, 0x00000011, 0x000100fd, 0x00010038,
};

get_nir(sizeof(words) / sizeof(words[0]), words);

nir_intrinsic_instr *intrinsic = find_intrinsic(nir_intrinsic_barrier, 0);
ASSERT_NE(intrinsic, nullptr);

EXPECT_EQ(nir_intrinsic_memory_semantics(intrinsic), NIR_MEMORY_MAKE_VISIBLE);
EXPECT_NE(nir_intrinsic_memory_modes(intrinsic) & nir_var_image, 0);
EXPECT_EQ(nir_intrinsic_memory_scope(intrinsic), SCOPE_DEVICE);
EXPECT_EQ(nir_intrinsic_execution_scope(intrinsic), SCOPE_NONE);

/* MakeTexelVisible makes the texel visible before it is read. */
int barrier = intrinsic_position(shader, nir_intrinsic_barrier);
int load = intrinsic_position(shader, nir_intrinsic_image_deref_load);
ASSERT_GE(barrier, 0);
ASSERT_GE(load, 0);
EXPECT_LT(barrier, load);
}

TEST_F(AvailabilityVisibility, opimagewrite_avail)
{
/*
OpCapability Shader
OpCapability VulkanMemoryModel
OpCapability VulkanMemoryModelDeviceScope
OpMemoryModel Logical Vulkan
OpEntryPoint GLCompute %main "main" %img
OpExecutionMode %main LocalSize 1 1 1
OpDecorate %img DescriptorSet 0
OpDecorate %img Binding 0
%void = OpTypeVoid
%fn = OpTypeFunction %void
%uint = OpTypeInt 32 0
%int = OpTypeInt 32 1
%v2int = OpTypeVector %int 2
%v4uint = OpTypeVector %uint 4
%imgtype = OpTypeImage %uint 2D 0 0 0 2 R32ui
%ptr = OpTypePointer UniformConstant %imgtype
%img = OpVariable %ptr UniformConstant
%int_0 = OpConstant %int 0
%coord = OpConstantComposite %v2int %int_0 %int_0
%device = OpConstant %int 1
%zero = OpConstantNull %v4uint
%main = OpFunction %void None %fn
%l = OpLabel
%im = OpLoad %imgtype %img
%v = OpImageRead %v4uint %im %coord
OpImageWrite %im %coord %v MakeTexelAvailable|NonPrivateTexel %device
OpReturn
OpFunctionEnd
*/
static const uint32_t words[] = {
0x07230203, 0x00010500, 0x00070000, 0x00000012, 0x00000000, 0x00020011,
0x00000001, 0x00020011, 0x000014e1, 0x00020011, 0x000014e2, 0x0003000e,
0x00000000, 0x00000003, 0x0006000f, 0x00000005, 0x00000001, 0x6e69616d,
0x00000000, 0x00000002, 0x00060010, 0x00000001, 0x00000011, 0x00000001,
0x00000001, 0x00000001, 0x00040047, 0x00000002, 0x00000022, 0x00000000,
0x00040047, 0x00000002, 0x00000021, 0x00000000, 0x00020013, 0x00000003,
0x00030021, 0x00000004, 0x00000003, 0x00040015, 0x00000005, 0x00000020,
0x00000000, 0x00040015, 0x00000006, 0x00000020, 0x00000001, 0x00040017,
0x00000007, 0x00000006, 0x00000002, 0x00040017, 0x00000008, 0x00000005,
0x00000004, 0x00090019, 0x00000009, 0x00000005, 0x00000001, 0x00000000,
0x00000000, 0x00000000, 0x00000002, 0x00000021, 0x00040020, 0x0000000a,
0x00000000, 0x00000009, 0x0004003b, 0x0000000a, 0x00000002, 0x00000000,
0x0004002b, 0x00000006, 0x0000000b, 0x00000000, 0x0005002c, 0x00000007,
0x0000000c, 0x0000000b, 0x0000000b, 0x0004002b, 0x00000006, 0x0000000d,
0x00000001, 0x0003002e, 0x00000008, 0x0000000e, 0x00050036, 0x00000003,
0x00000001, 0x00000000, 0x00000004, 0x000200f8, 0x0000000f, 0x0004003d,
0x00000009, 0x00000010, 0x00000002, 0x00050062, 0x00000008, 0x00000011,
0x00000010, 0x0000000c, 0x00060063, 0x00000010, 0x0000000c, 0x00000011,
0x00000500, 0x0000000d, 0x000100fd, 0x00010038,
};

get_nir(sizeof(words) / sizeof(words[0]), words);

nir_intrinsic_instr *intrinsic = find_intrinsic(nir_intrinsic_barrier, 0);
ASSERT_NE(intrinsic, nullptr);

EXPECT_EQ(nir_intrinsic_memory_semantics(intrinsic), NIR_MEMORY_MAKE_AVAILABLE);
EXPECT_NE(nir_intrinsic_memory_modes(intrinsic) & nir_var_image, 0);
EXPECT_EQ(nir_intrinsic_memory_scope(intrinsic), SCOPE_DEVICE);
EXPECT_EQ(nir_intrinsic_execution_scope(intrinsic), SCOPE_NONE);

/* MakeTexelAvailable makes the texel available after it is written. */
int barrier = intrinsic_position(shader, nir_intrinsic_barrier);
int store = intrinsic_position(shader, nir_intrinsic_image_deref_store);
ASSERT_GE(barrier, 0);
ASSERT_GE(store, 0);
EXPECT_GT(barrier, store);
}

TEST_F(AvailabilityVisibility, opcopymemory_visavail_both_combined)
{
/*
Expand Down