From 56ab33c38c74ea004ccfea6f7843d6479ad8d2cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:02:59 +0100 Subject: [PATCH 001/109] pvr: a powervr context whose job submission fails with EIO is lost The powervr kernel marks a context faulty after a GPU timeout, and every later SUBMIT_JOBS on it returns -EIO (pvr_queue.c). Nothing before the next submission noticed, and every later submission failed again. The powervr winsys now keeps a terminal, atomic "lost" state. The first SUBMIT_JOBS that returns EIO sets it and prints, once, the errno and the job type (render, compute, transfer, null). The line starts with "G5P" on the A9, whose logs already use that prefix, and with "pvr" elsewhere. Every submit path goes through one helper. A new winsys op, is_lost, reports the state; pvrsrvkm's always answers false. Assisted-by: LLM --- .../vulkan/winsys/powervr/pvr_drm.c | 42 +++++++++++++++++++ .../vulkan/winsys/powervr/pvr_drm.h | 11 +++++ .../winsys/powervr/pvr_drm_job_compute.c | 10 ++--- .../winsys/powervr/pvr_drm_job_render.c | 8 +--- .../winsys/powervr/pvr_drm_job_transfer.c | 8 +--- src/imagination/vulkan/winsys/pvr_winsys.h | 8 ++++ .../vulkan/winsys/pvrsrvkm/pvr_srv.c | 1 + 7 files changed, 69 insertions(+), 19 deletions(-) diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm.c b/src/imagination/vulkan/winsys/powervr/pvr_drm.c index c45a5385e62..e601654af9d 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm.c @@ -42,6 +42,42 @@ #include "vk_alloc.h" #include "vk_drm_syncobj.h" #include "vk_log.h" +#include "util/log.h" +#include "util/u_atomic.h" + +/* Terminal loss diagnostics keep the first failing job's errno. */ +void pvr_drm_report_loss(struct pvr_drm_winsys *ws, const char *job_type, + const char *source, int status) +{ + if (p_atomic_cmpxchg(&ws->loss_reported, 0, 1) == 0) { + ws->base.loss_source = source; + p_atomic_set(&ws->lost, 1); + const char *message = strerror(status < 0 ? -status : status); + mesa_loge("%s: device lost: job=%s %s=%d (%s)", + ws->is_g5p ? "G5P" : "pvr", job_type, source, + status, message); + } +} + +VkResult pvr_drm_submit_jobs(struct pvr_drm_winsys *ws, + struct drm_pvr_ioctl_submit_jobs_args *args, + const char *job_type) +{ + if (drmIoctl(ws->base.render_fd, DRM_IOCTL_PVR_SUBMIT_JOBS, args) == 0) + return VK_SUCCESS; + const int err = errno; + if (err == EIO) + pvr_drm_report_loss(ws, job_type, "SUBMIT_JOBS errno", err); + const char *message = strerror(err); + return vk_errorf(NULL, err == EIO ? VK_ERROR_DEVICE_LOST : VK_ERROR_OUT_OF_DEVICE_MEMORY, + "Failed to submit %s job (errno %d: %s)", job_type, err, message); +} + +static bool pvr_drm_winsys_is_lost(struct pvr_winsys *ws) +{ + struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ws); + return p_atomic_read(&drm_ws->lost); +} static void pvr_drm_finish_heaps(struct pvr_drm_winsys *const drm_ws) { @@ -391,6 +427,7 @@ static void pvr_drm_winsys_get_heaps_info(struct pvr_winsys *ws, static const struct pvr_winsys_ops drm_winsys_ops = { .destroy = pvr_drm_winsys_destroy, + .is_lost = pvr_drm_winsys_is_lost, .device_info_init = pvr_drm_winsys_device_info_init, .get_heaps_info = pvr_drm_winsys_get_heaps_info, .buffer_create = pvr_drm_winsys_buffer_create, @@ -723,6 +760,11 @@ VkResult pvr_drm_winsys_create(const int render_fd, drm_ws->bvnc = gpu_info.gpu_id; + struct pvr_device_info info; + if (!pvr_device_info_init(&info, drm_ws->bvnc)) { + drm_ws->is_g5p = PVR_HAS_FEATURE(&info, apple_g5p); + } + result = pvr_ioctl(render_fd, DRM_IOCTL_PVR_CREATE_VM_CONTEXT, &create_vm_context_args, diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm.h b/src/imagination/vulkan/winsys/powervr/pvr_drm.h index c8b1aa0b8d8..6e7a75e2cf4 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm.h +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm.h @@ -43,6 +43,10 @@ struct pvr_drm_winsys { /* Packed bvnc */ uint64_t bvnc; + bool is_g5p; + int loss_reported; + int lost; /* Atomic and terminal. */ + /* Protects imported BOs creation/freeing, which includes the bo_map member * below. */ @@ -85,6 +89,13 @@ struct pvr_drm_winsys { uint32_t vm_context; }; +struct drm_pvr_ioctl_submit_jobs_args; +VkResult pvr_drm_submit_jobs(struct pvr_drm_winsys *ws, + struct drm_pvr_ioctl_submit_jobs_args *args, + const char *job_type); +void pvr_drm_report_loss(struct pvr_drm_winsys *ws, const char *job_type, + const char *source, int status); + /******************************************* helper macros *******************************************/ diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c index b25c66f28dc..572b86f3149 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c @@ -172,8 +172,8 @@ VkResult pvr_drm_winsys_compute_submit( UNUSED const struct pvr_device_info *dev_info, struct vk_sync *signal_sync) { - const struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); - const struct pvr_drm_winsys_compute_ctx *drm_ctx = + struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); + struct pvr_drm_winsys_compute_ctx *drm_ctx = to_pvr_drm_winsys_compute_ctx(ctx); struct drm_pvr_sync_op sync_ops[2]; @@ -215,9 +215,5 @@ VkResult pvr_drm_winsys_compute_submit( } /* Returns VK_ERROR_OUT_OF_DEVICE_MEMORY to match pvrsrv. */ - return pvr_ioctlf(drm_ws->base.render_fd, - DRM_IOCTL_PVR_SUBMIT_JOBS, - &args, - VK_ERROR_OUT_OF_DEVICE_MEMORY, - "Failed to submit compute job"); + return pvr_drm_submit_jobs(drm_ws, &args, "compute"); } diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c index 0c543a3eeb4..7697ca5954c 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c @@ -474,7 +474,7 @@ VkResult pvr_drm_winsys_render_submit( struct vk_sync *signal_sync_frag) { - const struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); + struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); const struct pvr_drm_winsys_render_ctx *drm_ctx = to_pvr_drm_winsys_render_ctx(ctx); const struct pvr_winsys_geometry_state *const geom_state = @@ -643,11 +643,7 @@ VkResult pvr_drm_winsys_render_submit( flock(lock_fd, LOCK_EX); /* Returns VK_ERROR_OUT_OF_DEVICE_MEMORY to match pvrsrv. */ - const VkResult result = pvr_ioctlf(drm_ws->base.render_fd, - DRM_IOCTL_PVR_SUBMIT_JOBS, - &args, - VK_ERROR_OUT_OF_DEVICE_MEMORY, - "Failed to submit render job"); + const VkResult result = pvr_drm_submit_jobs(drm_ws, &args, "render"); if (lock_fd >= 0) flock(lock_fd, LOCK_UN); diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c index 9ae0f64dade..2e33676e32d 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c @@ -128,7 +128,7 @@ VkResult pvr_drm_winsys_transfer_submit( UNUSED const struct pvr_device_info *const dev_info, struct vk_sync *signal_sync) { - const struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); + struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); const struct pvr_drm_winsys_transfer_ctx *drm_ctx = to_pvr_drm_winsys_transfer_ctx(ctx); @@ -172,9 +172,5 @@ VkResult pvr_drm_winsys_transfer_submit( } /* Returns VK_ERROR_OUT_OF_DEVICE_MEMORY to match pvrsrv. */ - return pvr_ioctlf(drm_ws->base.render_fd, - DRM_IOCTL_PVR_SUBMIT_JOBS, - &args, - VK_ERROR_OUT_OF_DEVICE_MEMORY, - "Failed to submit transfer job"); + return pvr_drm_submit_jobs(drm_ws, &args, "transfer"); } diff --git a/src/imagination/vulkan/winsys/pvr_winsys.h b/src/imagination/vulkan/winsys/pvr_winsys.h index 00a9bdb48e7..4d9bf29e29e 100644 --- a/src/imagination/vulkan/winsys/pvr_winsys.h +++ b/src/imagination/vulkan/winsys/pvr_winsys.h @@ -356,8 +356,15 @@ struct pvr_winsys_render_submit_info { } fragment, fragment_pr; }; +static inline bool pvr_winsys_never_lost(struct pvr_winsys *ws) +{ + (void)ws; + return false; +} + struct pvr_winsys_ops { void (*destroy)(struct pvr_winsys *ws); + bool (*is_lost)(struct pvr_winsys *ws); VkResult (*device_info_init)(struct pvr_winsys *ws, struct pvr_device_info *dev_info, struct pvr_device_runtime_info *runtime_info); @@ -459,6 +466,7 @@ struct pvr_winsys { uint32_t log2_page_size; const struct vk_sync_type *sync_types[3]; + const char *loss_source; struct vk_sync_type syncobj_type; struct vk_sync_timeline_type timeline_syncobj_type; diff --git a/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c b/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c index 341f07e05e8..33468d98b0c 100644 --- a/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c +++ b/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c @@ -649,6 +649,7 @@ PER_ARCH_FUNCS(rogue); #define SRV_WINSYS(arch) \ static const struct pvr_winsys_ops pvr_##arch##_srv_winsys_ops = { \ + .is_lost = pvr_winsys_never_lost, \ .destroy = pvr_srv_winsys_destroy, \ .device_info_init = pvr_srv_winsys_device_info_init, \ .get_heaps_info = pvr_srv_winsys_get_heaps_info, \ From 5ddf51542d63b8215e29ebad58e78d222580e120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:02:59 +0100 Subject: [PATCH 002/109] pvr: a lost device reports DEVICE_LOST from waits and fails new memory allocations vkAllocateMemory may not return DEVICE_LOST; report OUT_OF_DEVICE_MEMORY on a lost device instead. Internal BOs keep allocating so that pipeline creation does not fail into Zink's device-memory retry loop. Waits and submits report loss from the logical device's winsys. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_device.c | 9 +++++++++ src/imagination/vulkan/pvr_device.c | 3 +++ 2 files changed, 12 insertions(+) diff --git a/src/imagination/vulkan/pvr_arch_device.c b/src/imagination/vulkan/pvr_arch_device.c index 38e04b8a1b6..a409d1319be 100644 --- a/src/imagination/vulkan/pvr_arch_device.c +++ b/src/imagination/vulkan/pvr_arch_device.c @@ -215,6 +215,14 @@ VkResult pvr_arch_pds_compute_shader_create_and_upload( return result; } +static VkResult pvr_device_check_status(struct vk_device *vk) +{ + struct pvr_device *device = container_of(vk, struct pvr_device, vk); + if (device->ws->ops->is_lost(device->ws)) + return vk_device_set_lost(vk, "device lost (%s)", device->ws->loss_source ? device->ws->loss_source : "kernel job error"); + return VK_SUCCESS; +} + static VkResult pvr_device_init_compute_fence_program(struct pvr_device *device) { struct pvr_pds_compute_shader_program program; @@ -824,6 +832,7 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, device->instance = instance; device->pdevice = pdevice; device->ws = ws; + device->vk.check_status = pvr_device_check_status; /* Before anything that creates a render target dataset. */ simple_mtx_init(&device->rt_cache_mtx, mtx_plain); diff --git a/src/imagination/vulkan/pvr_device.c b/src/imagination/vulkan/pvr_device.c index f1334d55132..1cbef9fbdf4 100644 --- a/src/imagination/vulkan/pvr_device.c +++ b/src/imagination/vulkan/pvr_device.c @@ -197,6 +197,9 @@ VkResult pvr_AllocateMemory(VkDevice _device, { const VkImportMemoryFdInfoKHR *fd_info = NULL; VK_FROM_HANDLE(pvr_device, device, _device); + + if (vk_device_is_lost_no_report(&device->vk)) + return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); enum pvr_winsys_bo_type type = PVR_WINSYS_BO_TYPE_GPU; struct pvr_device_memory *mem; VkResult result; From 40ec197c3647228afd23b5bdf71273d2fd007afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:02:59 +0100 Subject: [PATCH 003/109] pvr: silence repeated unmap errors after device loss The base already releases the mapping, CPU map and GEM object regardless of VM_UNMAP's result. Keep that unconditional release and stop repeating the failure log after a device loss. Assisted-by: LLM --- .../vulkan/winsys/powervr/pvr_drm.h | 1 + .../vulkan/winsys/powervr/pvr_drm_bo.c | 21 ++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm.h b/src/imagination/vulkan/winsys/powervr/pvr_drm.h index 6e7a75e2cf4..496fc737db7 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm.h +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm.h @@ -46,6 +46,7 @@ struct pvr_drm_winsys { bool is_g5p; int loss_reported; int lost; /* Atomic and terminal. */ + int unmap_loss_reported; /* Protects imported BOs creation/freeing, which includes the bo_map member * below. diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c index 04f8309f04a..4d8ad78b6e6 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c @@ -43,6 +43,7 @@ #include "util/macros.h" #include "vk_alloc.h" #include "vk_log.h" +#include "util/log.h" static VkResult pvr_drm_create_gem_bo(struct pvr_drm_winsys *drm_ws, uint32_t drm_flags, @@ -558,11 +559,21 @@ void pvr_drm_winsys_vma_unmap(struct pvr_winsys_vma *vma) /* Address should be mapped. */ assert(vma->bo); - pvr_ioctlf(drm_ws->base.render_fd, - DRM_IOCTL_PVR_VM_UNMAP, - &args, - VK_ERROR_UNKNOWN, - "Unmap failed"); + if (p_atomic_read(&drm_ws->lost)) { + if (drmIoctl(drm_ws->base.render_fd, DRM_IOCTL_PVR_VM_UNMAP, &args) && + p_atomic_cmpxchg(&drm_ws->unmap_loss_reported, 0, 1) == 0) { + const int err = errno; + const char *message = strerror(err); + mesa_logw("%s: VM_UNMAP after loss ignored (errno %d: %s)", + drm_ws->is_g5p ? "G5P" : "pvr", err, message); + } + } else { + pvr_ioctlf(drm_ws->base.render_fd, + DRM_IOCTL_PVR_VM_UNMAP, + &args, + VK_ERROR_UNKNOWN, + "Unmap failed"); + } pvr_drm_buffer_release(drm_bo); From 6c1f29b2ec3ed815ac60c26b2cc931f00496bc3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 03:33:44 +0100 Subject: [PATCH 004/109] pvr: a failed VM_MAP returns an out-of-memory code vkBindBufferMemory and vkBindImageMemory permit the out-of-memory codes, but not MEMORY_MAP_FAILED or DEVICE_LOST. Map EIO to host-memory exhaustion so Zink, which retries only device-memory exhaustion, does not sleep on a lost device. Other mapping failures return OUT_OF_DEVICE_MEMORY. Assisted-by: LLM --- src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c index 4d8ad78b6e6..f1cefc60b67 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c @@ -526,12 +526,14 @@ VkResult pvr_drm_winsys_vma_map(struct pvr_winsys_vma *vma, return vk_error(NULL, VK_ERROR_MEMORY_MAP_FAILED); } - result = pvr_ioctl(drm_ws->base.render_fd, - DRM_IOCTL_PVR_VM_MAP, - &args, - VK_ERROR_MEMORY_MAP_FAILED); - if (result != VK_SUCCESS) - return result; + if (drmIoctl(drm_ws->base.render_fd, DRM_IOCTL_PVR_VM_MAP, &args)) { + const int err = errno; + const char *message = strerror(err); + return vk_errorf(NULL, + err == EIO ? VK_ERROR_OUT_OF_HOST_MEMORY + : VK_ERROR_OUT_OF_DEVICE_MEMORY, + "DRM_IOCTL_PVR_VM_MAP failed (errno %d: %s)", err, message); + } pvr_drm_buffer_acquire(drm_bo); From 7556db6d048eff5ef47cfca5c17b6f173b9e7053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:02:59 +0100 Subject: [PATCH 005/109] pvr: detect a loss from job fence errors on Apple GPUs Use a driver-owned physical-device sync type to inspect tagged binary job fences after blocking completion waits. The wrapper uses the logical device's winsys so waits and submissions observe the same terminal loss. Keep the native sync type in the winsys and rebuild the services timeline wrapper on the driver-owned type. Polls and availability waits do not export sync files. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_queue.c | 20 +++++----- src/imagination/vulkan/pvr_device.c | 17 ++++++++ src/imagination/vulkan/pvr_device.h | 4 ++ src/imagination/vulkan/pvr_physical_device.c | 15 ++++++- src/imagination/vulkan/pvr_physical_device.h | 8 ++++ .../vulkan/winsys/powervr/pvr_drm.c | 40 +++++++++++++++++++ .../vulkan/winsys/powervr/pvr_drm.h | 19 +++++++++ .../winsys/powervr/pvr_drm_job_compute.c | 5 ++- .../vulkan/winsys/powervr/pvr_drm_job_null.c | 2 + .../winsys/powervr/pvr_drm_job_render.c | 4 ++ .../winsys/powervr/pvr_drm_job_transfer.c | 5 ++- src/imagination/vulkan/winsys/pvr_winsys.h | 3 ++ .../vulkan/winsys/pvrsrvkm/pvr_srv.c | 2 +- 13 files changed, 130 insertions(+), 14 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_queue.c b/src/imagination/vulkan/pvr_arch_queue.c index 86f91e2d1da..0329145575e 100644 --- a/src/imagination/vulkan/pvr_arch_queue.c +++ b/src/imagination/vulkan/pvr_arch_queue.c @@ -281,7 +281,7 @@ g5p_merge_syncs(struct pvr_device *device, struct vk_sync *a, struct vk_sync *b, waits[n++] = (struct vk_sync_wait){ .sync = a, .stage_mask = ~(VkPipelineStageFlags2)0 }; if (b) waits[n++] = (struct vk_sync_wait){ .sync = b, .stage_mask = ~(VkPipelineStageFlags2)0 }; - result = vk_sync_create(&device->vk, &device->pdevice->ws->syncobj_type, 0U, 0UL, out); + result = vk_sync_create(&device->vk, &device->pdevice->sync_type, 0U, 0UL, out); if (result != VK_SUCCESS) return result; const struct vk_sync_signal signal = { .sync = *out, .stage_mask = ~(VkPipelineStageFlags2)0 }; @@ -311,7 +311,7 @@ pvr_process_graphics_cmd_for_view(struct pvr_device *device, job->view_state.view_index = view_index; result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &geom_signal_sync); @@ -320,7 +320,7 @@ pvr_process_graphics_cmd_for_view(struct pvr_device *device, if (job->run_frag) { result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &frag_signal_sync); @@ -683,7 +683,7 @@ static VkResult pvr_process_compute_cmd(struct pvr_device *device, VkResult result; result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &sync); @@ -721,7 +721,7 @@ static VkResult pvr_process_transfer_cmds(struct pvr_device *device, VkResult result; result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &sync); @@ -868,7 +868,7 @@ static VkResult pvr_process_query_cmd(struct pvr_device *device, */ result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &sync); @@ -939,7 +939,7 @@ pvr_process_event_cmd_barrier(struct pvr_device *device, struct vk_sync *signal_sync; result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &signal_sync); @@ -1008,7 +1008,7 @@ pvr_process_event_cmd_set_or_reset(struct pvr_device *device, } result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &signal_sync); @@ -1119,7 +1119,7 @@ pvr_process_event_cmd_wait(struct pvr_device *device, assert(wait_count <= (sub_cmd->count + 1)); result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &signal_sync); @@ -1591,7 +1591,7 @@ static VkResult pvr_process_queue_waits(struct pvr_queue *queue, } result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &queue->next_job_wait_sync[i]); diff --git a/src/imagination/vulkan/pvr_device.c b/src/imagination/vulkan/pvr_device.c index 1cbef9fbdf4..7d88fa6f397 100644 --- a/src/imagination/vulkan/pvr_device.c +++ b/src/imagination/vulkan/pvr_device.c @@ -60,6 +60,7 @@ #include "pvr_winsys.h" #include "pvr_wsi.h" #include "util/log.h" +#include "util/os_time.h" #include "util/macros.h" #include "util/mesa-blake3.h" #include "util/os_misc.h" @@ -190,6 +191,22 @@ void pvr_rstate_entry_remove(struct pvr_device *device, simple_mtx_unlock(&device->rs_mtx); } +/* Inspect job errors only for blocking completion waits, on the logical + * device's winsys. Polling and availability waits retain their usual cost. */ +VkResult pvr_sync_wait_many(struct vk_device *vk, uint32_t count, + const struct vk_sync_wait *waits, + enum vk_sync_wait_flags flags, uint64_t timeout) +{ + struct pvr_device *device = container_of(vk, struct pvr_device, vk); + const bool slow = !(flags & VK_SYNC_WAIT_PENDING) && + timeout > (uint64_t)os_time_get_nano(); + VkResult result = device->pdevice->native_wait_many(vk, count, waits, flags, timeout); + if (result == VK_SUCCESS && slow && device->ws->ops->check_job_fences && + device->ws->ops->check_job_fences(device->ws, count, waits) == VK_ERROR_DEVICE_LOST) + return vk_device_check_status(vk); + return result; +} + VkResult pvr_AllocateMemory(VkDevice _device, const VkMemoryAllocateInfo *pAllocateInfo, const VkAllocationCallbacks *pAllocator, diff --git a/src/imagination/vulkan/pvr_device.h b/src/imagination/vulkan/pvr_device.h index 8ebde5092b7..b22b767561d 100644 --- a/src/imagination/vulkan/pvr_device.h +++ b/src/imagination/vulkan/pvr_device.h @@ -282,6 +282,10 @@ static inline struct pvr_device *vk_to_pvr_device(struct vk_device *device) return container_of(device, struct pvr_device, vk); } +VkResult pvr_sync_wait_many(struct vk_device *vk, uint32_t count, + const struct vk_sync_wait *waits, + enum vk_sync_wait_flags flags, uint64_t timeout); + VkResult pvr_device_tile_buffer_ensure_cap(struct pvr_device *device, uint32_t capacity); diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index f5812d1016c..3f1009ddec3 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -1356,7 +1356,20 @@ VkResult pvr_physical_device_init(struct pvr_physical_device *pdevice, goto err_vk_physical_device_finish; } - pdevice->vk.supported_sync_types = ws->sync_types; + pdevice->sync_type = ws->syncobj_type; + if (ws->job_sync_size) { + pdevice->sync_type.size = MAX2(pdevice->sync_type.size, ws->job_sync_size); + pdevice->native_wait_many = pdevice->sync_type.wait_many; + pdevice->sync_type.wait_many = pvr_sync_wait_many; + } + pdevice->sync_types[0] = &pdevice->sync_type; + pdevice->sync_types[1] = NULL; + pdevice->sync_types[2] = NULL; + if (ws->sync_types[1]) { + pdevice->timeline_sync_type = vk_sync_timeline_get_type(&pdevice->sync_type); + pdevice->sync_types[1] = &pdevice->timeline_sync_type.sync; + } + pdevice->vk.supported_sync_types = pdevice->sync_types; pdevice->pco_ctx = pco_ctx_create(&pdevice->dev_info, &pdevice->dev_runtime_info, NULL); diff --git a/src/imagination/vulkan/pvr_physical_device.h b/src/imagination/vulkan/pvr_physical_device.h index 20f410ba890..05e532f81f3 100644 --- a/src/imagination/vulkan/pvr_physical_device.h +++ b/src/imagination/vulkan/pvr_physical_device.h @@ -26,6 +26,8 @@ #include "pvr_device_info.h" #include "pvr_formats.h" +#include "vk_sync.h" +#include "vk_sync_timeline.h" #if defined(VK_USE_PLATFORM_DISPLAY_KHR) || \ defined(VK_USE_PLATFORM_WAYLAND_KHR) || \ @@ -63,6 +65,12 @@ struct pvr_physical_device { dev_t render_devid; struct pvr_winsys *ws; + struct vk_sync_type sync_type; + const struct vk_sync_type *sync_types[3]; + struct vk_sync_timeline_type timeline_sync_type; + VkResult (*native_wait_many)(struct vk_device *, uint32_t, + const struct vk_sync_wait *, + enum vk_sync_wait_flags, uint64_t); struct pvr_device_info dev_info; struct pvr_device_runtime_info dev_runtime_info; diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm.c b/src/imagination/vulkan/winsys/powervr/pvr_drm.c index e601654af9d..9231b94c7de 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm.c @@ -44,6 +44,10 @@ #include "vk_log.h" #include "util/log.h" #include "util/u_atomic.h" +#include "util/os_time.h" +#include +#include +#include /* Terminal loss diagnostics keep the first failing job's errno. */ void pvr_drm_report_loss(struct pvr_drm_winsys *ws, const char *job_type, @@ -79,6 +83,37 @@ static bool pvr_drm_winsys_is_lost(struct pvr_winsys *ws) return p_atomic_read(&drm_ws->lost); } +static VkResult pvr_drm_check_job_fences(struct pvr_winsys *base, + uint32_t count, + const struct vk_sync_wait *waits) +{ + struct pvr_drm_winsys *ws = to_pvr_drm_winsys(base); + for (uint32_t i = 0; i < count; i++) { + /* Job out-syncobjs are binary. A timeline export would not name the + * specific point that was waited, so leave those to the runtime. + */ + if ((waits[i].sync->flags & VK_SYNC_IS_TIMELINE) || + !ws->base.job_sync_size || + waits[i].sync->type->size != ws->base.job_sync_size) + continue; + const char *job_type = ((struct pvr_drm_job_sync *)waits[i].sync)->job_type; + if (!job_type) + continue; + struct vk_drm_syncobj *sync = vk_sync_as_drm_syncobj(waits[i].sync); + int fd = -1; + if (!sync || drmSyncobjExportSyncFile(ws->base.render_fd, sync->syncobj, &fd)) + continue; + struct sync_file_info info = { 0 }; + const int ret = ioctl(fd, SYNC_IOC_FILE_INFO, &info); + close(fd); + if (!ret && info.status < 0) { + pvr_drm_report_loss(ws, job_type, "fence status", info.status); + return VK_ERROR_DEVICE_LOST; + } + } + return VK_SUCCESS; +} + static void pvr_drm_finish_heaps(struct pvr_drm_winsys *const drm_ws) { if (!pvr_winsys_helper_winsys_heap_finish( @@ -428,6 +463,7 @@ static void pvr_drm_winsys_get_heaps_info(struct pvr_winsys *ws, static const struct pvr_winsys_ops drm_winsys_ops = { .destroy = pvr_drm_winsys_destroy, .is_lost = pvr_drm_winsys_is_lost, + .check_job_fences = pvr_drm_check_job_fences, .device_info_init = pvr_drm_winsys_device_info_init, .get_heaps_info = pvr_drm_winsys_get_heaps_info, .buffer_create = pvr_drm_winsys_buffer_create, @@ -763,6 +799,10 @@ VkResult pvr_drm_winsys_create(const int render_fd, struct pvr_device_info info; if (!pvr_device_info_init(&info, drm_ws->bvnc)) { drm_ws->is_g5p = PVR_HAS_FEATURE(&info, apple_g5p); + drm_ws->job_fence_errors = drm_ws->is_g5p || + PVR_HAS_FEATURE(&info, apple_g9p); + drm_ws->base.job_sync_size = drm_ws->job_fence_errors + ? sizeof(struct pvr_drm_job_sync) : 0; } result = pvr_ioctl(render_fd, diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm.h b/src/imagination/vulkan/winsys/powervr/pvr_drm.h index 496fc737db7..2e5379f26fb 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm.h +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm.h @@ -29,10 +29,17 @@ #include #include "pvr_winsys.h" +#include "vk_drm_syncobj.h" #include "util/macros.h" #include "util/rwlock.h" #include "util/sparse_array.h" +/* Identify job out-fences without inspecting ordinary poll waits. */ +struct pvr_drm_job_sync { + struct vk_drm_syncobj base; + const char *job_type; +}; + struct pvr_drm_winsys_heap { struct pvr_winsys_heap base; }; @@ -44,6 +51,10 @@ struct pvr_drm_winsys { uint64_t bvnc; bool is_g5p; + /* The kernel sets a job fence's error when it resets (pvr_apple.c); + * upstream pvr_queue.c does not. + */ + bool job_fence_errors; int loss_reported; int lost; /* Atomic and terminal. */ int unmap_loss_reported; @@ -97,6 +108,14 @@ VkResult pvr_drm_submit_jobs(struct pvr_drm_winsys *ws, void pvr_drm_report_loss(struct pvr_drm_winsys *ws, const char *job_type, const char *source, int status); +static inline void pvr_drm_tag_job_sync(struct pvr_drm_winsys *ws, + struct vk_sync *sync, const char *job_type) +{ + if (ws->job_fence_errors && sync && ws->base.job_sync_size && + sync->type->size == ws->base.job_sync_size) + ((struct pvr_drm_job_sync *)sync)->job_type = job_type; +} + /******************************************* helper macros *******************************************/ diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c index 572b86f3149..77656c5f443 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c @@ -215,5 +215,8 @@ VkResult pvr_drm_winsys_compute_submit( } /* Returns VK_ERROR_OUT_OF_DEVICE_MEMORY to match pvrsrv. */ - return pvr_drm_submit_jobs(drm_ws, &args, "compute"); + const VkResult result = pvr_drm_submit_jobs(drm_ws, &args, "compute"); + if (result == VK_SUCCESS) + pvr_drm_tag_job_sync(drm_ws, signal_sync, "compute"); + return result; } diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_null.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_null.c index 4c93d183de3..d4b70229850 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_null.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_null.c @@ -44,6 +44,8 @@ VkResult pvr_drm_winsys_null_job_submit(struct pvr_winsys *ws, struct vk_sync_signal *signal_sync) { const struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ws); + pvr_drm_tag_job_sync(to_pvr_drm_winsys(ws), signal_sync->sync, + "queue completion (geometry/fragment/compute)"); uint32_t tmp_syncobj; VkResult result; int ret; diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c index 7697ca5954c..ddf43186e42 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c @@ -645,6 +645,10 @@ VkResult pvr_drm_winsys_render_submit( /* Returns VK_ERROR_OUT_OF_DEVICE_MEMORY to match pvrsrv. */ const VkResult result = pvr_drm_submit_jobs(drm_ws, &args, "render"); + if (result == VK_SUCCESS) { + pvr_drm_tag_job_sync(drm_ws, signal_sync_geom, "geometry"); + pvr_drm_tag_job_sync(drm_ws, signal_sync_frag, "fragment"); + } if (lock_fd >= 0) flock(lock_fd, LOCK_UN); return result; diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c index 2e33676e32d..9e31fdaa504 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c @@ -172,5 +172,8 @@ VkResult pvr_drm_winsys_transfer_submit( } /* Returns VK_ERROR_OUT_OF_DEVICE_MEMORY to match pvrsrv. */ - return pvr_drm_submit_jobs(drm_ws, &args, "transfer"); + const VkResult result = pvr_drm_submit_jobs(drm_ws, &args, "transfer"); + if (result == VK_SUCCESS) + pvr_drm_tag_job_sync(drm_ws, signal_sync, "transfer"); + return result; } diff --git a/src/imagination/vulkan/winsys/pvr_winsys.h b/src/imagination/vulkan/winsys/pvr_winsys.h index 4d9bf29e29e..a5e8d866055 100644 --- a/src/imagination/vulkan/winsys/pvr_winsys.h +++ b/src/imagination/vulkan/winsys/pvr_winsys.h @@ -365,6 +365,8 @@ static inline bool pvr_winsys_never_lost(struct pvr_winsys *ws) struct pvr_winsys_ops { void (*destroy)(struct pvr_winsys *ws); bool (*is_lost)(struct pvr_winsys *ws); + VkResult (*check_job_fences)(struct pvr_winsys *ws, uint32_t count, + const struct vk_sync_wait *waits); VkResult (*device_info_init)(struct pvr_winsys *ws, struct pvr_device_info *dev_info, struct pvr_device_runtime_info *runtime_info); @@ -467,6 +469,7 @@ struct pvr_winsys { const struct vk_sync_type *sync_types[3]; const char *loss_source; + uint32_t job_sync_size; struct vk_sync_type syncobj_type; struct vk_sync_timeline_type timeline_syncobj_type; diff --git a/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c b/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c index 33468d98b0c..68ab52d3222 100644 --- a/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c +++ b/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c @@ -824,7 +824,7 @@ static VkResult pvr_srv_create_presignaled_sync(struct pvr_device *device, goto err_close_sw_fence; result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &sync); From 428301fe83598a33198f1cc340b2227e3981c294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:02:59 +0100 Subject: [PATCH 006/109] pvr/g5p: a lost queue keeps signalling its submissions The deferred-submit path still has to signal submissions that cannot run after a loss. Signal unsubmitted payloads so exports and waiters can finish, while retaining the device-lost result for the caller. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_queue.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_queue.c b/src/imagination/vulkan/pvr_arch_queue.c index 0329145575e..2f7765b01a5 100644 --- a/src/imagination/vulkan/pvr_arch_queue.c +++ b/src/imagination/vulkan/pvr_arch_queue.c @@ -59,10 +59,12 @@ #include "pvr_bo.h" #include "pvr_pipeline.h" +#include "util/log.h" #include "util/macros.h" #include "util/u_atomic.h" #include "vk_alloc.h" #include "vk_common_entrypoints.h" +#include "vk_enum_to_str.h" #include "vk_fence.h" #include "vk_log.h" #include "vk_object.h" @@ -1708,6 +1710,18 @@ static void pvr_signal_unsubmitted(struct vk_device *device, static VkResult pvr_driver_queue_submit(struct vk_queue *queue, struct vk_queue_submit *submit) { + struct pvr_queue *driver_queue = container_of(queue, struct pvr_queue, vk); + const bool is_g5p = + PVR_HAS_FEATURE(&driver_queue->device->pdevice->dev_info, apple_g5p); + + if (is_g5p && vk_queue_is_lost(queue)) { + for (uint32_t i = 0; i < submit->signal_count; i++) + pvr_signal_unsubmitted(&driver_queue->device->vk, + submit->signals[i].sync, + submit->signals[i].signal_value); + return VK_SUCCESS; + } + const int64_t g9p_t0 = os_time_get_nano(); VkResult result = pvr_driver_queue_submit_jobs(queue, submit); @@ -1721,8 +1735,6 @@ static VkResult pvr_driver_queue_submit(struct vk_queue *queue, * reset, freezing the desktop; signalled, it sees the device lost and can recover */ if (result != VK_SUCCESS) { - struct pvr_queue *driver_queue = container_of(queue, struct pvr_queue, vk); - for (uint32_t i = 0; i < submit->signal_count; i++) { if (PVR_HAS_FEATURE(&driver_queue->device->pdevice->dev_info, apple_g5p)) pvr_signal_unsubmitted(&driver_queue->device->vk, @@ -1732,6 +1744,7 @@ static VkResult pvr_driver_queue_submit(struct vk_queue *queue, vk_sync_signal(&driver_queue->device->vk, submit->signals[i].sync, submit->signals[i].signal_value); } + } if (result == VK_ERROR_DEVICE_LOST) return vk_queue_set_lost(queue, "GPU context lost"); From 19deaf41f58ccf7ccd9f2e602cecf7e9434c7aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 05:24:14 +0100 Subject: [PATCH 007/109] vulkan: allow pending submits when stopping a lost submit thread A driver_submit failure makes the worker mark its queue lost and exit with runtime-owned submits still listed. Drain returns on loss; thread shutdown must allow the pending list so queue finish can perform its existing cleanup. A forced-thread lavapipe empty-submit repro with an injected driver_submit out-of-device-memory failure hits the pending-list assertion before this change. Afterwards it reports DEVICE_LOST and destroys the device cleanly. Healthy queues still require an empty list. Release builds are unchanged. A worker blocked on a never-signalled wait can still block thread join; that pre-existing problem is separate from this teardown assertion. Assisted-by: LLM --- src/vulkan/runtime/vk_queue.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/vulkan/runtime/vk_queue.c b/src/vulkan/runtime/vk_queue.c index c26ae4e5efd..f6916a14b7d 100644 --- a/src/vulkan/runtime/vk_queue.c +++ b/src/vulkan/runtime/vk_queue.c @@ -828,7 +828,11 @@ vk_queue_stop_submit_thread(struct vk_queue *queue) thrd_join(queue->submit.thread, NULL); - assert(list_is_empty(&queue->submit.submits)); + /* A failed submit may remain after queue loss. vk_queue_finish destroys + * pending submits after the worker has exited. + */ + assert(list_is_empty(&queue->submit.submits) || + vk_device_is_lost_no_report(queue->base.device)); queue->submit.mode = VK_QUEUE_SUBMIT_MODE_IMMEDIATE; } From 3a558db130dbfe7654ea64f69242479c37b79986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 03:40:05 +0100 Subject: [PATCH 008/109] pvr: log error origins with PVR_DEBUG=errors Use the runtime's per-instance debug logging switch so every vk_error names its origin. No runtime hook or process-global callback is needed. Assisted-by: LLM --- src/imagination/common/pvr_debug.c | 1 + src/imagination/common/pvr_debug.h | 2 ++ src/imagination/vulkan/pvr_instance.c | 1 + 3 files changed, 4 insertions(+) diff --git a/src/imagination/common/pvr_debug.c b/src/imagination/common/pvr_debug.c index e910f4b217f..764436641e0 100644 --- a/src/imagination/common/pvr_debug.c +++ b/src/imagination/common/pvr_debug.c @@ -38,6 +38,7 @@ static const struct debug_named_value debug_control[] = { "Dump descriptor set and pipeline layouts." }, { "info", PVR_DEBUG_INFO, "Display information about the driver and device." }, + { "errors", PVR_DEBUG_ERRORS, "Log every Vulkan error the driver returns, with its origin." }, DEBUG_NAMED_VALUE_END }; /* clang-format on */ diff --git a/src/imagination/common/pvr_debug.h b/src/imagination/common/pvr_debug.h index 33d22424ab1..7d205bcf7c1 100644 --- a/src/imagination/common/pvr_debug.h +++ b/src/imagination/common/pvr_debug.h @@ -39,6 +39,8 @@ extern uint32_t PVR_DEBUG; #define PVR_DEBUG_VK_DUMP_DESCRIPTOR_SET_LAYOUT BITFIELD_BIT(2) #define PVR_DEBUG_INFO BITFIELD_BIT(3) +#define PVR_DEBUG_ERRORS BITFIELD_BIT(8) + void pvr_process_debug_variable(void); #endif /* PVR_DEBUG_H */ diff --git a/src/imagination/vulkan/pvr_instance.c b/src/imagination/vulkan/pvr_instance.c index da1f495bfdd..dc54f7023c1 100644 --- a/src/imagination/vulkan/pvr_instance.c +++ b/src/imagination/vulkan/pvr_instance.c @@ -371,6 +371,7 @@ VkResult pvr_CreateInstance(const VkInstanceCreateInfo *pCreateInfo, goto err_free_instance; pvr_process_debug_variable(); + instance->vk.enable_debug_logging = PVR_IS_DEBUG_SET(ERRORS); pvr_init_dri_options(instance); instance->active_device_count = 0; From 938a6d94b3adc867ed18a600b3367d5ff6d7edd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:39 +0100 Subject: [PATCH 009/109] pvr/g5p: a CPU transfer the A9 path cannot do is named and skipped Any refusal in the A9's CPU transfer path (no CPU mapping, an element past the binding, an unsupported conversion, a multisample depth plane) returned VK_ERROR_DEVICE_LOST from the queue submit under one message. Each refusal now records why, and the submit logs one rate-limited line per refused command: the reason, fill or copy, the scissor, and every surface (format, size, stride, layout, samples, z, address; the image's format, extent, levels, layers, samples and the surface's offset in it). The command is skipped, its destination keeps what it held, and the queue continues. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_g5p_layout.c | 94 +++++++++++++++++--- 1 file changed, 82 insertions(+), 12 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_g5p_layout.c b/src/imagination/vulkan/pvr_arch_g5p_layout.c index 6725acaed96..884b3f71281 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_layout.c +++ b/src/imagination/vulkan/pvr_arch_g5p_layout.c @@ -16,6 +16,8 @@ */ #include +#include +#include #include #include #include @@ -24,6 +26,7 @@ #include "vk_format.h" #include "vk_sync.h" +#include "vk_enum_to_str.h" #include "util/format/u_format.h" #include "util/log.h" @@ -53,6 +56,18 @@ struct g5p_surface { uint64_t image_offset; }; +/* Why the CPU transfer refused (the last refusal on this thread, for the error line) */ +static thread_local char g5p_why[192]; +static bool PRINTFLIKE(1, 2) +G5P_WHY(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vsnprintf(g5p_why, sizeof(g5p_why), fmt, ap); + va_end(ap); + return false; +} + static bool g5p_surface_init(struct pvr_device *device, const struct pvr_transfer_cmd_surface *s, struct g5p_surface *out) { @@ -89,8 +104,8 @@ g5p_surface_init(struct pvr_device *device, const struct pvr_transfer_cmd_surfac out->aspects = s->g5p_aspect; out->image_offset = s->g5p_image ? s->dev_addr.addr - s->g5p_image->dev_addr.addr - pvr_single_plane_const(s->g5p_image)->offset : 0; if (!out->base) - mesa_loge("G5P transfer: no CPU mapping for %#" PRIx64, s->dev_addr.addr); - return out->base != NULL; + return G5P_WHY("no CPU mapping of a tracked memory binding covers %#" PRIx64, s->dev_addr.addr); + return true; } /* Spread the low 16 coordinate bits into even bit positions. G5P uses @@ -126,13 +141,18 @@ g5p_offset(const struct g5p_surface *s, unsigned x, unsigned y) static uint8_t * g5p_element(struct g5p_surface *s, unsigned x, unsigned y) { - if (x >= s->width || y >= s->height) + if (x >= s->width || y >= s->height) { + G5P_WHY("element (%u,%u) outside the %ux%u surface", x, y, s->width, s->height); return NULL; + } const size_t offset = g5p_offset(s, x, y); - if (offset > s->available || s->bpp > s->available - offset) + if (offset > s->available || s->bpp > s->available - offset) { + G5P_WHY("element (%u,%u) at byte %zu + %u past the %" PRIu64 " bytes bound from its address", + x, y, offset, s->bpp, (uint64_t)s->available); return NULL; + } return s->base + offset + s->component_offset; } @@ -158,7 +178,8 @@ g5p_ds_plane(const struct g5p_surface *s, VkImageAspectFlagBits aspect, const struct pvr_image *image = s->image; const struct pvr_g5p_ds_format *own = pvr_g5p_ds_format(image->vk.format); if (!own || (stencil ? !own->stencil : !own->depth) || image->vk.samples != 1) - return false; + return G5P_WHY("depth/stencil plane of %s, %u samples (one-sample depth/stencil images only)", + vk_Format_to_str(image->vk.format), image->vk.samples); p->format = stencil ? PIPE_FORMAT_S8_UINT : own->depth_bytes == 2 ? PIPE_FORMAT_Z16_UNORM : PIPE_FORMAT_Z32_FLOAT; p->cpp = stencil ? 1 : own->depth_bytes; @@ -686,8 +707,8 @@ g5p_copy(const struct pvr_rect_mapping *mapping, struct g5p_surface *src, struct util_format_unpack_s_8uint(src->format, &stencil, from, 1); util_format_pack_s_8uint(dst->format, to, &stencil, 1); } else { - mesa_loge("G5P transfer: unsupported depth/stencil conversion %u -> %u", src->format, dst->format); - return false; + return G5P_WHY("depth/stencil conversion %s -> %s", util_format_short_name(src->format), + util_format_short_name(dst->format)); } } else if (integer) { uint32_t v[4]; @@ -714,7 +735,7 @@ g5p_copy_aspects(const struct pvr_rect_mapping *mapping, struct g5p_surface *src VkImageAspectFlags aspects = src->image ? src->aspects : dst->aspects; if (src->image && dst->image) aspects &= dst->aspects; const struct pvr_g5p_ds_format *f = pvr_g5p_ds_format(image->vk.format); - if (!f) return false; + if (!f) return G5P_WHY("no G5P depth/stencil layout for %s", vk_Format_to_str(image->vk.format)); for (unsigned i = 0; i < 2; i++) { VkImageAspectFlagBits aspect = i ? VK_IMAGE_ASPECT_STENCIL_BIT : VK_IMAGE_ASPECT_DEPTH_BIT; if (!(aspects & aspect)) continue; @@ -736,7 +757,7 @@ g5p_run(struct pvr_device *device, const struct pvr_transfer_cmd *cmd) if (cmd->flags & PVR_TRANSFER_CMD_FLAGS_FILL) { if (dst.image && vk_format_is_depth_or_stencil(dst.image->vk.format)) { const struct pvr_g5p_ds_format *f = pvr_g5p_ds_format(dst.image->vk.format); - if (!f) return false; + if (!f) return G5P_WHY("no G5P depth/stencil layout for %s", vk_Format_to_str(dst.image->vk.format)); for (unsigned i = 0; i < 2; i++) { VkImageAspectFlagBits aspect = i ? VK_IMAGE_ASPECT_STENCIL_BIT : VK_IMAGE_ASPECT_DEPTH_BIT; struct g5p_surface p; @@ -767,6 +788,51 @@ g5p_run(struct pvr_device *device, const struct pvr_transfer_cmd *cmd) return true; } +/* One surface of a refused transfer, for the error line */ +static int +g5p_describe_surface(char *out, size_t n, const char *what, const struct pvr_transfer_cmd_surface *s) +{ + const struct pvr_image *im = s->g5p_image; + int k = snprintf(out, n, " %s{%s %ux%ux%u stride %u %s samples %u z %.1f va %#" PRIx64, what, + vk_Format_to_str(s->vk_format), s->width, s->height, s->depth, s->stride, + s->mem_layout == PVR_MEMLAYOUT_TWIDDLED ? "twiddled" : + s->mem_layout == PVR_MEMLAYOUT_LINEAR ? "linear" : "3d-twiddled", + s->sample_count, s->z_position, s->dev_addr.addr); + if (im && k >= 0 && (size_t)k < n) + k += snprintf(out + k, n - k, " image %s %ux%ux%u levels %u layers %u samples %u size %" PRIu64 " at +%" PRIu64, + vk_Format_to_str(im->vk.format), im->vk.extent.width, im->vk.extent.height, im->vk.extent.depth, + im->vk.mip_levels, im->vk.array_layers, im->vk.samples, (uint64_t)im->total_size, + s->dev_addr.addr - im->dev_addr.addr); + if (k >= 0 && (size_t)k < n) + k += snprintf(out + k, n - k, "}"); + return k; +} + +/* A CPU transfer G5P refuses (g5p_run() false): which surface and why */ +static void +g5p_transfer_refused(const struct pvr_transfer_cmd *cmd) +{ + char line[1024]; + int k = snprintf(line, sizeof(line), "G5P transfer refused: %s; %s scissor %ux%u+%d+%d", + g5p_why[0] ? g5p_why : "?", (cmd->flags & PVR_TRANSFER_CMD_FLAGS_FILL) ? "fill" : "copy", + cmd->scissor.extent.width, cmd->scissor.extent.height, cmd->scissor.offset.x, + cmd->scissor.offset.y); + if (k >= 0 && (size_t)k < sizeof(line)) + k += g5p_describe_surface(line + k, sizeof(line) - k, "dst", &cmd->dst); + for (unsigned i = 0; i < cmd->source_count && k >= 0 && (size_t)k < sizeof(line); i++) { + const struct pvr_transfer_cmd_source *src = &cmd->sources[i]; + k += g5p_describe_surface(line + k, sizeof(line) - k, "src", &src->surface); + if (src->mapping_count && k >= 0 && (size_t)k < sizeof(line)) { + const struct pvr_rect_mapping *m = &src->mappings[0]; + k += snprintf(line + k, sizeof(line) - k, " map[%u] %ux%u+%d+%d -> %ux%u+%d+%d", src->mapping_count, + m->src_rect.extent.width, m->src_rect.extent.height, m->src_rect.offset.x, + m->src_rect.offset.y, m->dst_rect.extent.width, m->dst_rect.extent.height, + m->dst_rect.offset.x, m->dst_rect.offset.y); + } + } + mesa_logw("%s", line); +} + VkResult PVR_PER_ARCH(g5p_transfer_submit)(struct pvr_device *device, struct pvr_sub_cmd_transfer *sub_cmd, const struct vk_sync_wait *waits, uint32_t wait_count, @@ -780,9 +846,13 @@ PVR_PER_ARCH(g5p_transfer_submit)(struct pvr_device *device, struct pvr_sub_cmd_ return result; } list_for_each_entry (struct pvr_transfer_cmd, cmd, sub_cmd->transfer_cmds, link) { - if (!g5p_run(device, cmd)) - return vk_errorf(device, VK_ERROR_DEVICE_LOST, - "G5P transfer surface exceeds its memory's mapping"); + g5p_why[0] = 0; + if (!g5p_run(device, cmd)) { + /* A transfer the CPU path cannot do is named (surface, size, format, layout and why) and skipped: + * its destination keeps what it held and the queue goes on (the game workload previously returned + * DEVICE_LOST). Decide unsupported copies at recording time in a future change. */ + g5p_transfer_refused(cmd); + } } return vk_sync_signal(&device->vk, signal_sync, 0); } From 72f325ec19d9ef7e6421d438cdf8e351de7ac6da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:39 +0100 Subject: [PATCH 010/109] pvr/g5p: add a 3D image's slice offset once in its transfer surface The A9's transfer-slice hook turned a slice of a 3D image into a 2D twiddled surface and added depthPitch * z. pvr_setup_transfer_surface() then took the non-3D branch for that surface and added depthPitch * z again, so slice z of every copy went to slice 2z: slices past the middle fell off the image and were refused, and the rest landed on the wrong slices. The hook now only retypes the surface. A 64x64x16 image with 7 levels now copies exactly. Assisted-by: LLM --- src/imagination/vulkan/pvr_g5p_hooks.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/imagination/vulkan/pvr_g5p_hooks.h b/src/imagination/vulkan/pvr_g5p_hooks.h index 78725bbb67e..b151eab9b1a 100644 --- a/src/imagination/vulkan/pvr_g5p_hooks.h +++ b/src/imagination/vulkan/pvr_g5p_hooks.h @@ -142,7 +142,9 @@ (surface)->g5p_aspect = (aspect); \ } \ if (PVR_G5P_IMAGE_IS_SLICED_3D(image)) { \ - (surface)->dev_addr.addr += (info)->depthPitch * (uint32_t)(fdepth); \ + /* the slice's offset, info->depthPitch * z, is added once after this \ + * hook by pvr_setup_transfer_surface() itself, as for any non-3D \ + * surface: adding it here too put slice z at 2z (observed in 3D transfers) */ \ (surface)->mem_layout = PVR_MEMLAYOUT_TWIDDLED; \ (surface)->depth = 1; \ } From e4e5ab69a9b179ae04fa2fc8f5e9e3e33ec0f412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:25 +0100 Subject: [PATCH 011/109] pvr/g5p: 256-lane compute workgroups WebGPU's core limits want 256 invocations per workgroup and 256 in X and Y; the A9 reported 128. Apple's A9 driver allows 512 threads per threadgroup in each dimension, lowered per pipeline by its resources, and Rogue's model agrees (32 lanes x 16 tasks per USC). The dispatch takes up to 512 lanes, and the A9 compiler emits Apple's software barrier above 32 lanes. That barrier waits for every task of the workgroup, so all of them must be resident: a pipeline whose temporaries per lane exceed rogue_max_wg_temps() for its workgroup is refused by name. maxComputeWorkGroupInvocations and maxComputeWorkGroupSize X/Y go from 128 to 256 on the A9. Tested on the A9 before the powervr interface: 256-lane exchange through shared memory across a barrier, and a 256-lane tree sum with nine barriers, 64 workgroups each. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_compute.h | 3 +- src/imagination/g9p/g9p_a9_intrinsics.h | 2 +- src/imagination/vulkan/pvr_arch_g5p_compute.c | 34 +++++++++++++++---- src/imagination/vulkan/pvr_g5p.h | 7 ++++ src/imagination/vulkan/pvr_physical_device.c | 10 ++++-- 5 files changed, 44 insertions(+), 12 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index eac2eb87e45..ecb8068289f 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -10,7 +10,8 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c NIR_PASS(_, nir, nir_lower_vars_to_explicit_types, nir_var_mem_shared, glsl_get_natural_size_align_bytes); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_shared, nir_address_format_32bit_offset); if (nir->info.shared_size > 16384) { - fprintf(stderr, "g5p: shared allocation exceeds16384 bytes\n"); + if (!out->a9_refusal[0]) + snprintf(out->a9_refusal, sizeof(out->a9_refusal), "shared allocation exceeds 16384 bytes"); return false; } out->a9_local_dwords = 4 + DIV_ROUND_UP(nir->info.shared_size, 4); diff --git a/src/imagination/g9p/g9p_a9_intrinsics.h b/src/imagination/g9p/g9p_a9_intrinsics.h index e73a3f4ac10..1b2703cc9ea 100644 --- a/src/imagination/g9p/g9p_a9_intrinsics.h +++ b/src/imagination/g9p/g9p_a9_intrinsics.h @@ -80,7 +80,7 @@ static bool a9_barrier(struct g9p_ctx *ctx, nir_intrinsic_instr *in) bool device=modes & (nir_var_mem_ssbo | nir_var_mem_global | nir_var_image); if ((modes & ~(nir_var_mem_shared | nir_var_mem_ssbo | nir_var_mem_global | nir_var_image)) || (scope != SCOPE_NONE && scope != SCOPE_INVOCATION && scope != SCOPE_WORKGROUP)) { - fprintf(stderr, "g5p: unsupported barrier memory mode or execution scope\n"); + mesa_logw("g5p: unsupported barrier memory mode or execution scope"); return false; } int fence_pair=-1; diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 4f3ef9d9374..2fba41082f5 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -22,6 +22,7 @@ #include "pvr_physical_device.h" #include "pvr_pipeline.h" #include "pvr_g5p.h" +#include "hwdef/rogue_hw_utils.h" #include "pvr_g5p_compute.h" #include "pvr_job_context.h" #include "pvr_winsys.h" @@ -53,6 +54,25 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( "G5P compute: shader not supported by the backend yet"); goto out; } + /* A barrier above 32 lanes waits for every task of the workgroup + * (Apple's software barrier, g5p_barrier.h), so all of them must be + * resident in one USC at once: Rogue's bound on the temporaries per lane + * for that workgroup size (rogue_max_wg_temps(), as upstream's compute + * pipelines check). A pipeline above it would spin forever: refused. */ + { + const unsigned threads = shader.local_size[0] * shader.local_size[1] * shader.local_size[2]; + const struct pvr_device_info *dev_info = &device->pdevice->dev_info; + if (shader.a9_has_barriers && threads > ROGUE_MAX_INSTANCES_PER_TASK && + rogue_max_wg_temps(dev_info, shader.num_regs, threads, true) < shader.num_regs) { + mesa_logw("G5P refuses: a %u-lane workgroup with barriers needs %u temporaries per lane, above " + "the %u that keep every task resident", threads, shader.num_regs, + rogue_max_wg_temps(dev_info, shader.num_regs, threads, true)); + result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, + "G5P compute: %u-lane barriered workgroup needs %u temporaries per lane", threads, + shader.num_regs); + goto out; + } + } result = pvr_gpu_upload_usc(device, shader.code, shader.code_size, 64, &state->shader_bo); @@ -80,7 +100,7 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( out: ralloc_free(mem_ctx); - return result; + return pvr_refusal_to_api(result); } static void @@ -123,13 +143,13 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, VkResult result; if (base_group[0] || base_group[1] || base_group[2]) - return vk_errorf(cmd_buffer, VK_ERROR_FEATURE_NOT_PRESENT, "G5P: base dispatch not implemented yet"); + return vk_errorf(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY, "G5P: base dispatch not implemented yet"); if (indirect_va && ((indirect_va & 3) || indirect_va >> 40)) - return vk_error(cmd_buffer, VK_ERROR_FEATURE_NOT_PRESENT); + return vk_error(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY); if (!indirect_va && (!group_count[0] || !group_count[1] || !group_count[2])) return VK_SUCCESS; if (!threads || threads > 512 || group_count[0] > 65536 || group_count[1] > 65536 || group_count[2] > 65536) - return vk_error(cmd_buffer, VK_ERROR_FEATURE_NOT_PRESENT); + return vk_error(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY); /* A bodyless shader has no observable invocation. Apple's standalone * empty program does not retire under our CDM launch contract, so do not @@ -138,7 +158,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, /* The compiler's unit-Z specialization is valid only for these actual * dispatch dimensions. Indirect dimensions cannot be checked here. */ if (state->a9_unit_z && (indirect_va || group_count[2] != 1)) { - return vk_errorf(cmd_buffer, VK_ERROR_FEATURE_NOT_PRESENT, + return vk_errorf(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY, "compute workgroup Z specialization requires a direct dispatch with one Z group"); } @@ -166,7 +186,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, } } unsigned images=state->a9_num_images; - if(images && n>15)return vk_errorf(cmd_buffer,VK_ERROR_FEATURE_NOT_PRESENT,"G5P image LOD common31 overlaps the16th buffer pointer"); + if(images && n>15)return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P image LOD common31 overlaps the16th buffer pointer"); unsigned transfers=n+1+images+(state->a9_device_fence ? 1 : 0)+(images ? 1 : 0); if (state->a9_device_fence) { uint32_t zero=0; @@ -516,7 +536,7 @@ PVR_PER_ARCH(g5p_compute_job_submit)(struct pvr_compute_ctx *ctx, * first block here. A chained CDM stream is refused until that form has * run on the device. */ if (cs->pvr_bo != list_first_entry(&cs->pvr_bo_list, struct pvr_bo, link)) - return vk_errorf(d, VK_ERROR_FEATURE_NOT_PRESENT, "G5P: chained CDM stream not implemented yet"); + return vk_errorf(d, VK_ERROR_OUT_OF_HOST_MEMORY, "G5P: chained CDM stream not implemented yet"); struct pvr_winsys_compute_submit_info info = { .wait = wait, .frame_num = d->global_queue_present_count, .job_num = d->global_cmd_buffer_submit_count }; info.fw_stream_len = diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index a0560f31ac5..67403ad76e5 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -20,6 +20,13 @@ #include #include +/* Zink retries device-memory errors; wgpu survives only OOM failures. + * Keep refusal internal and map it at creation/recording API boundaries. */ +static inline VkResult pvr_refusal_to_api(VkResult result) +{ + return result == VK_ERROR_FEATURE_NOT_PRESENT ? VK_ERROR_OUT_OF_HOST_MEMORY : result; +} + #include "pvr_common.h" #include "pvr_limits.h" #include "pvr_macros.h" diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index 3f1009ddec3..3ba24affc6a 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -741,10 +741,14 @@ static bool pvr_physical_device_get_properties( [1] = (64U * 1024U) - 1, [2] = (64U * 1024U) - 1, }, - .maxComputeWorkGroupInvocations = 128U, + /* G5P: 256 (AGXMetalA9: 512 per threadgroup, maxThreadsPerThreadgroup; Rogue's CDM: 32 lanes x 16 + * tasks per USC). Barriers above 32 lanes are Apple's software barrier (g5p_barrier.h); a pipeline whose + * temporaries cannot keep every task of a barriered workgroup resident is refused at creation + * (pvr_arch_g5p_compute.c). */ + .maxComputeWorkGroupInvocations = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 256U : 128U, .maxComputeWorkGroupSize = { - [0] = 128U, - [1] = 128U, + [0] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 256U : 128U, + [1] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 256U : 128U, [2] = 64U, }, From c127ec7b332f895ea7781fcd1625023510a7b484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:25 +0100 Subject: [PATCH 012/109] g9p: texture samples, loads and sizes in A9 compute shaders textureSampleLevel, textureLoad, textureDimensions and textureNumLevels in a compute shader were refused, and WebGPU turns a failed pipeline into a lost device. Apple's A9 kernels (Metal sample(s, uv, level(l)), gradient2d() and read(g, lod), compiled for the A9) use the coupled operand tuples of the vertex sample, byte for byte. g9p_a9_tex.h emits them for 2D float textures: coordinates copied to a fresh block, the level converted to f16 (a u32 level for a read, which uses the fetch sampler), and a wait after each sample. A9 compute no longer lowers texelFetch to a sample, so textureLoad takes the exact read form. Sizes come from a block the CPU computes, as on the fragment path. Arrays, cubes, 3D, compares, offsets, integer textures and implicit-LOD samples are refused by name. The descriptor words go to fixed common-store dwords inside the 64 the dispatch already loads, copied from the bound sets as the fragment path does. A refused compute shader now records and prints its reason. Assisted-by: LLM --- src/imagination/g9p/g9p_a9.h | 12 +- src/imagination/g9p/g9p_a9_compute.h | 21 +- src/imagination/g9p/g9p_a9_tex.h | 276 ++++++++++++++++++ src/imagination/g9p/g9p_compile.c | 24 +- src/imagination/vulkan/pvr_arch_g5p_compute.c | 84 +++++- .../vulkan/pvr_g5p_compute_state.h | 3 +- 6 files changed, 406 insertions(+), 14 deletions(-) create mode 100644 src/imagination/g9p/g9p_a9_tex.h diff --git a/src/imagination/g9p/g9p_a9.h b/src/imagination/g9p/g9p_a9.h index 1bb9a7235b6..f3f3e0e8010 100644 --- a/src/imagination/g9p/g9p_a9.h +++ b/src/imagination/g9p/g9p_a9.h @@ -1,7 +1,17 @@ /* SPDX-License-Identifier: MIT */ #ifndef G9P_A9_H #define G9P_A9_H -#define G9P_A9_FIELDS bool a9; bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; +/* Compute texture words the dispatch puts in the common store (g9p_a9_tex.h) */ +#define G9P_A9_MAX_UPLOADS 7 +enum g9p_a9_upload_kind { G9P_A9_UP_IMAGE = 1, G9P_A9_UP_SAMPLER, G9P_A9_UP_FETCH_SAMPLER, G9P_A9_UP_SIZE }; +struct g9p_a9_upload { + unsigned char kind, set, binding, element; + unsigned char combined; /* a sampler in a combined image sampler: its words at +32 */ + unsigned char level; /* a size block's level */ + unsigned char common, dwords; +}; +#define G9P_A9_FIELDS bool a9; bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; \ + unsigned a9_num_uploads; struct g9p_a9_upload a9_uploads[G9P_A9_MAX_UPLOADS]; char a9_refusal[192]; #define G9P_A9_END_PAD(ctx) do { if ((ctx)->out->a9 && (2*(ctx)->c->n+6)%64==0) G9P_EMIT((ctx)->c,0xc000); } while(0) /* Keep main's G9P numbering; only the A9 backend uses NIR-order breaks. */ #define G9P_A9_EMPTY_BREAK_BEFORE(ctx, side) do { if (!(ctx)->out->a9 || !(side)) (ctx)->ip++; } while (0) diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index ecb8068289f..a08e6e2746d 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -17,5 +17,24 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c out->a9_local_dwords = 4 + DIV_ROUND_UP(nir->info.shared_size, 4); NIR_PASS(_, nir, nir_lower_compute_system_values, &csv); NIR_PASS(_, nir, nir_lower_undef_to_zero, NULL); - return compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx); + if (!compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx)) + return false; + /* the fixed common-store places of texture words (g9p_a9_tex.h) */ + bool sizes = false; + for (unsigned i = 0; i < out->a9_num_uploads; i++) + sizes |= out->a9_uploads[i].kind == G9P_A9_UP_SIZE; + if (out->a9_num_uploads && out->num_buffers > 15) { + if (!out->a9_refusal[0]) + snprintf(out->a9_refusal, sizeof(out->a9_refusal), + "%u buffers with sampled textures (buffer words reach the texture words at common 32)", + out->num_buffers); + return false; + } + if (sizes && out->num_textures) { + if (!out->a9_refusal[0]) + snprintf(out->a9_refusal, sizeof(out->a9_refusal), + "a texture size query together with a storage image (both at common 40-47)"); + return false; + } + return true; } diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h new file mode 100644 index 00000000000..b831c30f02d --- /dev/null +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: MIT */ +#include "../g5p/g5p_usc.h" +/* + * A9 compute samples (WGSL textureSampleLevel/Grad/Load/Dimensions in a + * compute shader). Apple's A9 kernels (sample(s, uv, level(l)), + * gradient2d(), read(g, lod)) use the + * coupled operand tuples of the qualified vertex sample (g5p_usc_sample_ext() + * G5P_TEX_TUPLE_COUPLED_*), byte for byte: + * level(l) G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL, 2D, four channels, f16 level in r2 + * gradient G5P_TEX_TUPLE_COUPLED_GRAD, 2D, four channels, [dx.x dy.x dx.y dy.y] f32 + * The f16 level is Apple's f32-to-f16 conversion (g9p_f16_of(), the vertex + * level form without a last-use flag). Every sample here is followed by a + * wait, so each one issues in the state of Apple's single-sample kernels. + * + * Descriptor words go to fixed common-store dwords, inside the 64 the + * dispatch's DMA task already loads (pvr_arch_g5p_compute.c): texture + * states at 32 and 48 (four-qword aligned), sampler words at 58, 60, 62, + * size blocks (textureDimensions, CPU-computed as the fragment path's) at 40 + * and 44 when no storage image uses 40-47. Buffers keep dwords 0-2n, the + * device fence 56-57, a storage image 40-47 and its level 31. + */ +static const uint8_t a9_tex_image_dw[] = { 32, 48 }; +static const uint8_t a9_tex_sampler_dw[] = { 58, 60, 62 }; +static const uint8_t a9_tex_size_dw[] = { 40, 44 }; + +static void +a9_tex_refuse(struct g9p_ctx *ctx, const char *why) +{ + if (!ctx->out->a9_refusal[0]) + snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); +} + +/* The common dword of an upload, added if new; -1 when its slots are full */ +static int +a9_tex_upload(struct g9p_ctx *ctx, unsigned kind, unsigned set, unsigned binding, unsigned element, + unsigned combined, unsigned level) +{ + struct g9p_shader *out = ctx->out; + const uint8_t *slots = kind == G9P_A9_UP_IMAGE ? a9_tex_image_dw : + kind == G9P_A9_UP_SIZE ? a9_tex_size_dw : a9_tex_sampler_dw; + const unsigned nslots = kind == G9P_A9_UP_IMAGE ? ARRAY_SIZE(a9_tex_image_dw) : + kind == G9P_A9_UP_SIZE ? ARRAY_SIZE(a9_tex_size_dw) : ARRAY_SIZE(a9_tex_sampler_dw); + unsigned used = 0; + + for (unsigned i = 0; i < out->a9_num_uploads; i++) { + const struct g9p_a9_upload *u = &out->a9_uploads[i]; + const bool same_class = u->kind == kind || + ((u->kind == G9P_A9_UP_SAMPLER || u->kind == G9P_A9_UP_FETCH_SAMPLER) && + (kind == G9P_A9_UP_SAMPLER || kind == G9P_A9_UP_FETCH_SAMPLER)); + if (u->kind == kind && u->set == set && u->binding == binding && u->element == element && + u->combined == combined && u->level == level) + return u->common; + used += same_class; + } + if (used == nslots || out->a9_num_uploads == G9P_A9_MAX_UPLOADS || set > 255 || binding > 255 || + element > 255 || level > 255) { + a9_tex_refuse(ctx, kind == G9P_A9_UP_IMAGE ? "more than 2 sampled textures in a compute shader" : + kind == G9P_A9_UP_SIZE ? "more than 2 texture size queries in a compute shader" : + "more than 3 samplers in a compute shader"); + return -1; + } + out->a9_uploads[out->a9_num_uploads++] = (struct g9p_a9_upload){ + .kind = kind, .set = set, .binding = binding, .element = element, .combined = combined, + .level = level, .common = slots[used], + .dwords = kind == G9P_A9_UP_IMAGE ? 8 : kind == G9P_A9_UP_SIZE ? 4 : 2, + }; + return slots[used]; +} + +static bool +a9_tex_var(nir_tex_instr *tex, nir_tex_src_type type, unsigned *set, unsigned *binding, unsigned *element) +{ + const int i = nir_tex_instr_src_index(tex, type); + if (i < 0) + return false; + nir_deref_instr *deref = nir_src_as_deref(tex->src[i].src); + nir_variable *var = nir_deref_instr_get_variable(deref); + if (!var) + return false; + *set = var->data.descriptor_set; + *binding = var->data.binding; + *element = 0; + if (deref->deref_type == nir_deref_type_array) { + if (!nir_src_is_const(deref->arr.index)) + return false; + *element = nir_src_as_uint(deref->arr.index); + } else if (deref->deref_type != nir_deref_type_var) { + return false; + } + return true; +} + +static void +a9_tex_emit_raw(struct g9p_ctx *ctx, const uint8_t *raw, unsigned n) +{ + for (unsigned i = 0; i + 1 < n; i += 2) + G9P_EMIT(ctx->c, raw[i] | raw[i + 1] << 8); +} + +/* A value copied into register @dst (OR copies any bit pattern) */ +static bool +a9_tex_copy(struct g9p_ctx *ctx, unsigned dst, nir_def *def, unsigned comp) +{ + const int r = src_reg(ctx, nir_get_scalar(def, comp)); + if (r < 0) + return false; + g9p_alu2(ctx->c, G9P_OR, dst, r, r, false); + return true; +} + +/* textureDimensions / textureNumLevels at a constant level: words of the + * CPU-computed size block (pvr_g5p_tex_size(): w, h, depth/layers, levels) */ +static bool +a9_txs(struct g9p_ctx *ctx, nir_tex_instr *tex) +{ + unsigned set, binding, element, level = 0; + const int li = nir_tex_instr_src_index(tex, nir_tex_src_lod); + + if (!a9_tex_var(tex, nir_tex_src_texture_deref, &set, &binding, &element)) { + a9_tex_refuse(ctx, "texture size query of a texture without a constant binding"); + return false; + } + if (tex->op == nir_texop_txs && li >= 0) { + if (!nir_src_is_const(tex->src[li].src)) { + a9_tex_refuse(ctx, "textureDimensions at a non-constant level in a compute shader"); + return false; + } + level = nir_src_as_uint(tex->src[li].src); + } + const int common = a9_tex_upload(ctx, G9P_A9_UP_SIZE, set, binding, element, 0, level); + if (common < 0) + return false; + int zero = const_reg(ctx, 0); + if (zero < 0 || (zero = low_reg(ctx, zero)) < 0) + return false; + const unsigned n = tex->op == nir_texop_query_levels ? 1 : tex->def.num_components; + int dst[4]; + for (unsigned c = 0; c < n; c++) { + if (!ctx->last_use[key(&tex->def, c)]) { + dst[c] = -1; + continue; + } + dst[c] = alloc_reg(ctx); + if (dst[c] < 0) + return false; + g9p_mov_uniform(ctx->c, dst[c], common + (tex->op == nir_texop_query_levels ? 3 : c), zero); + } + release(ctx, &tex->instr); + for (unsigned c = 0; c < n; c++) + ctx->reg[key(&tex->def, c)] = dst[c]; + return true; +} + +static bool +a9_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) +{ + if (tex->op == nir_texop_txs || tex->op == nir_texop_query_levels) + return a9_txs(ctx, tex); + + const bool read = tex->op == nir_texop_txf, grad = tex->op == nir_texop_txd; + const int ci = nir_tex_instr_src_index(tex, nir_tex_src_coord); + unsigned set, binding, element, sset, sbinding, selement, combined = 0; + + if (tex->op == nir_texop_tex || tex->op == nir_texop_txb) { + a9_tex_refuse(ctx, "an implicit-LOD or bias sample in a compute shader (no derivatives)"); + return false; + } + if (tex->op != nir_texop_txl && !grad && !read) { + a9_tex_refuse(ctx, "texture operation not supported in compute shaders yet"); + return false; + } + if (tex->sampler_dim != GLSL_SAMPLER_DIM_2D || tex->is_array || tex->is_shadow) { + a9_tex_refuse(ctx, tex->is_shadow ? "a depth-compare sample in a compute shader" : + "only 2D (not array, cube or 3D) samples in compute shaders so far"); + return false; + } + if (nir_alu_type_get_base_type(tex->dest_type) != nir_type_float) { + a9_tex_refuse(ctx, "an integer-texture sample or load in a compute shader"); + return false; + } + for (unsigned i = 0; i < tex->num_srcs; i++) { + switch (tex->src[i].src_type) { + case nir_tex_src_coord: case nir_tex_src_lod: case nir_tex_src_ddx: case nir_tex_src_ddy: + case nir_tex_src_texture_deref: case nir_tex_src_sampler_deref: + break; + default: + a9_tex_refuse(ctx, "a texel offset or other extra texture source in a compute shader"); + return false; + } + } + if (ci < 0 || !a9_tex_var(tex, nir_tex_src_texture_deref, &set, &binding, &element)) { + a9_tex_refuse(ctx, "a compute sample without a constant texture binding"); + return false; + } + if (read) { + sset = sbinding = selement = 0; + } else if (!a9_tex_var(tex, nir_tex_src_sampler_deref, &sset, &sbinding, &selement)) { + /* a combined image sampler: its sampler words are at +32 of the same descriptor */ + sset = set, sbinding = binding, selement = element, combined = 1; + } + const int image = a9_tex_upload(ctx, G9P_A9_UP_IMAGE, set, binding, element, 0, 0); + const int sampler = image < 0 ? -1 : + read ? a9_tex_upload(ctx, G9P_A9_UP_FETCH_SAMPLER, 0, 0, 0, 0, 0) : + a9_tex_upload(ctx, G9P_A9_UP_SAMPLER, sset, sbinding, selement, combined, 0); + if (sampler < 0) + return false; + + /* coordinates: a fresh block, consumed by the sample */ + const int block = alloc_range(ctx, 2); + if (block < 0 || block > 30) + return false; + ctx->temps |= BITFIELD64_RANGE(block, 2); + for (unsigned c = 0; c < 2; c++) + if (!a9_tex_copy(ctx, block + c, tex->src[ci].src.ssa, c)) + return false; + + /* level (f16 for samples, u32 for reads) or gradients */ + int lod; + if (grad) { + const int dxi = nir_tex_instr_src_index(tex, nir_tex_src_ddx); + const int dyi = nir_tex_instr_src_index(tex, nir_tex_src_ddy); + lod = alloc_range(ctx, 4); + if (dxi < 0 || dyi < 0 || lod < 0 || lod > 28) + return false; + ctx->temps |= BITFIELD64_RANGE(lod, 4); + for (unsigned c = 0; c < 2; c++) { + if (!a9_tex_copy(ctx, lod + 2 * c, tex->src[dxi].src.ssa, c) || + !a9_tex_copy(ctx, lod + 2 * c + 1, tex->src[dyi].src.ssa, c)) + return false; + } + } else { + const int li = nir_tex_instr_src_index(tex, nir_tex_src_lod); + int r = li >= 0 ? src_reg(ctx, nir_get_scalar(tex->src[li].src.ssa, 0)) : const_reg(ctx, 0); + if (r < 0) + return false; + if (read) { + lod = alloc_temp(ctx); + if (lod < 0 || lod > 31) + return false; + g9p_alu2(ctx->c, G9P_OR, lod, r, r, false); + } else { + r = one_src(ctx, r); + lod = r < 0 ? r : alloc_low_temp(ctx); + if (lod < 0) + return false; + g9p_f16_of(ctx->c, lod, r); + } + } + release(ctx, &tex->instr); + const int dst = alloc_range(ctx, 4); + if (dst < 0 || dst > 28) + return false; + + struct g5p_tex_ext x = { + .dst_half = 2 * dst, .image_qword = image / 2, .sampler_qword = sampler / 2, + .coord_dword = block, .dim = G5P_TEX_2D, .mask = 0xf, .read = read, + .lod = grad ? G5P_TEX_LOD_GRAD : G5P_TEX_LOD_LEVEL, .lod_dword = lod, + .tuple = grad ? G5P_TEX_TUPLE_COUPLED_GRAD : G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL, + }; + uint8_t raw[G5P_USC_MAX]; + const unsigned n = g5p_usc_sample_ext(raw, &x); + if (!n) { + a9_tex_refuse(ctx, "the compute sample recipe was refused by the encoder"); + return false; + } + a9_tex_emit_raw(ctx, raw, n); + g9p_wait(ctx->c); + for (unsigned c = 0; c < 4; c++) { + ctx->reg[key(&tex->def, c)] = dst + c; + if (c >= tex->def.num_components || !ctx->last_use[key(&tex->def, c)]) + ctx->busy &= ~BITFIELD64_BIT(dst + c); + } + return true; +} +#define G9P_A9_TEX(ctx, tex) \ + do { if ((ctx)->out->a9 && (ctx)->stage == MESA_SHADER_COMPUTE) return a9_tex(ctx, tex); } while (0) diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index 322ea8c9ff1..b5d525ff053 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -18,6 +18,7 @@ #include #include #include +#include "c11/threads.h" #include #include @@ -2827,9 +2828,11 @@ emit_tex_vertex(struct g9p_ctx *ctx, nir_tex_instr *tex) * where the sample reads them (as Apple's code for t.sample(s, uv * 2.0 + * 0.25) does; r0-r1 are free until the colour output). */ +#include "g9p_a9_tex.h" static bool emit_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) { + G9P_A9_TEX(ctx, tex); int ci = nir_tex_instr_src_index(tex, nir_tex_src_coord); int slot = texture_slot(ctx, tex); int first, second, dst; @@ -3622,6 +3625,17 @@ emit_cf_list(struct g9p_ctx *ctx, struct exec_list *list) nir_print_instr(instr, stderr); fprintf(stderr, "\n"); } + /* the refusal for the shader dump (pvr_g5p_shader_dump.h): the backend's own reason + * if it gave one, then the instruction */ + if (ctx->out->a9) { + const size_t used = strnlen(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal)); + FILE *f = fmemopen(ctx->out->a9_refusal + used, sizeof(ctx->out->a9_refusal) - used, "w"); + if (f) { + fprintf(f, "%s%s", used ? ": " : "", ctx->oom ? "out of registers at " : "cannot compile "); + nir_print_instr(instr, f); + fclose(f); + } + } return false; } nir_foreach_def(instr, park_def, ctx); @@ -4247,15 +4261,15 @@ lower_coverage(nir_shader *nir, const struct g9p_coverage *coverage) } /* The last attempt at a vertex shader: in program order, every load at its use */ -static _Thread_local bool g9p_no_schedule; +static thread_local bool g9p_no_schedule; /* * Registers above r31 only once every attempt without them failed: a shader that fits * reloading its values runs far more instances at once (the aquarium's fish vertex shader: * 100 registers with homes on the first attempt, 4 instances, geometry 10x slower) */ -static _Thread_local bool g9p_no_homes; +static thread_local bool g9p_no_homes; /* vertex shaders: attribute loads moved next to their uses (g9p_compile_vertex() tries both) */ -static _Thread_local bool g9p_move_inputs; +static thread_local bool g9p_move_inputs; /* * Vertex outputs right after their values, with emit_output() writing each @@ -4318,7 +4332,7 @@ hoist_vertex_outputs(nir_shader *nir) return progress; } /* the homes an attempt may use: few first (r32-r63), all of them last */ -static _Thread_local unsigned g9p_max_homes = 92; +static thread_local unsigned g9p_max_homes = 92; /* * Loads that are cheap to repeat (uniform buffers, push constants, @@ -4717,6 +4731,8 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, NIR_PASS(_, nir, nir_lower_tex, &offsets); NIR_PASS(_, nir, nir_lower_tex, &levels); } + /* A9 compute reads texels with the sampler's read form and the fetch sampler (g9p_a9_tex.h) */ + if (!(out->a9 && stage == MESA_SHADER_COMPUTE)) NIR_PASS(_, nir, nir_shader_tex_pass, lower_txf, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_shader_instructions_pass, lower_half_packing, nir_metadata_control_flow, NULL); } diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 2fba41082f5..5a7203746af 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -22,6 +22,7 @@ #include "pvr_physical_device.h" #include "pvr_pipeline.h" #include "pvr_g5p.h" +#include "pvr_g5p_tex_encode.h" #include "hwdef/rogue_hw_utils.h" #include "pvr_g5p_compute.h" #include "pvr_job_context.h" @@ -50,6 +51,8 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( goto out; if (!g9p_a9_compile_compute(nir, &shader, mem_ctx)) { + mesa_logw("pvr/g5p: compute pipeline refused: %s", + shader.a9_refusal[0] ? shader.a9_refusal : "no reason recorded"); result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, "G5P compute: shader not supported by the backend yet"); goto out; @@ -95,6 +98,20 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( state->a9_images[i].set=shader.textures[i].set; state->a9_images[i].binding=shader.textures[i].binding; } + /* sampled textures' descriptor words (g9p_a9_tex.h) */ + STATIC_ASSERT(ARRAY_SIZE(state->a9_uploads) == G9P_A9_MAX_UPLOADS); + state->a9_num_uploads = shader.a9_num_uploads; + for (unsigned i = 0; i < shader.a9_num_uploads; i++) { + const struct g9p_a9_upload *u = &shader.a9_uploads[i]; + state->a9_uploads[i].kind = u->kind; + state->a9_uploads[i].set = u->set; + state->a9_uploads[i].binding = u->binding; + state->a9_uploads[i].element = u->element; + state->a9_uploads[i].combined = u->combined; + state->a9_uploads[i].level = u->level; + state->a9_uploads[i].common = u->common; + state->a9_uploads[i].dwords = u->dwords; + } state->uses_workgroup_id = shader.uses_workgroup_id; memcpy(state->local_size, shader.local_size, sizeof(state->local_size)); @@ -134,9 +151,9 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, const uint32_t hw_threads = g5p_cdm_workgroup_threads(threads); const uint32_t count = threads | threads << 16; struct pvr_suballoc_bo *uniforms_bo; - uint32_t uniforms[2 * G9P_MAX_BUFFERS + 4] = { 0 }; - uint32_t bind_data[4 * (G9P_MAX_BUFFERS + 1) + 4] = { 0 }; - uint32_t bind_code[G9P_MAX_BUFFERS + 5]; + uint32_t uniforms[2 * G9P_MAX_BUFFERS + 4 + 8 * G9P_A9_MAX_UPLOADS] = { 0 }; + uint32_t bind_data[4 * (G9P_MAX_BUFFERS + 1) + 4 + 3 * G9P_A9_MAX_UPLOADS] = { 0 }; + uint32_t bind_code[G9P_MAX_BUFFERS + 5 + G9P_A9_MAX_UPLOADS]; uint32_t kernel_data[12] = { 0 }, kernel_code[8]; struct pvr_pds_upload bind, kernel; unsigned bind_dwords, ctrl0, nc = 0; @@ -187,7 +204,55 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, } unsigned images=state->a9_num_images; if(images && n>15)return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P image LOD common31 overlaps the16th buffer pointer"); - unsigned transfers=n+1+images+(state->a9_device_fence ? 1 : 0)+(images ? 1 : 0); + const unsigned base_transfers=n+1+images+(state->a9_device_fence ? 1 : 0)+(images ? 1 : 0); + const unsigned tex0=2*n+1+(state->a9_device_fence ? 2 : 0)+(images ? 1 : 0); + const unsigned nup=state->a9_num_uploads; + unsigned transfers=base_transfers+nup; + /* sampled textures' words, validated as the fragment path's (pvr_arch_g5p_gfx.c), from the + * bound sets' CPU mappings into the uniforms upload, then DMAed to their common dwords */ + for(unsigned i=0;ia9_uploads[0]) *u=&state->a9_uploads[i]; + uint32_t *out=&uniforms[tex0+8*i]; + if(u->kind==G9P_A9_UP_FETCH_SAMPLER) { + const uint64_t policy=pvr_g5p_fetch_sampler_word(); + memcpy(out,&policy,sizeof(policy)); + continue; + } + const struct pvr_descriptor_set *set=u->setsets[u->set] : NULL; + if(!set || !set->layout || !set->mapping || u->binding>=set->layout->binding_count) + return vk_errorf(cmd_buffer,VK_ERROR_UNKNOWN,"G5P compute: texture set %u binding %u is not bound",u->set,u->binding); + const struct pvr_descriptor_set_layout_binding *lb=&set->layout->bindings[u->binding]; + if(u->element>=MAX2(lb->descriptor_count,1)) + return vk_errorf(cmd_buffer,VK_ERROR_UNKNOWN,"G5P compute: set %u binding %u element %u past its descriptors",u->set,u->binding,u->element); + const uint32_t *d=(const uint32_t *)((const uint8_t *)set->mapping+lb->offset+(size_t)u->element*lb->stride); + switch(u->kind) { + case G9P_A9_UP_IMAGE: + if(lb->type==VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER || !pvr_g5p_tex_state_valid(d)) { + mesa_logw("G5P refuses: compute set %u binding %u: a view without A9 texture state",u->set,u->binding); + return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P compute: a view without A9 texture state"); + } + memcpy(out,d,4*8); + break; + case G9P_A9_UP_SAMPLER: + /* a combined descriptor's sampler is at +32, a standalone sampler's at +0 */ + if(lb->type!=VK_DESCRIPTOR_TYPE_SAMPLER) + d+=8; + if(!pvr_g5p_sampler_valid(d)) { + mesa_logw("G5P refuses: compute set %u binding %u: a sampler without A9 words",u->set,u->binding); + return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P compute: a sampler without A9 words"); + } + memcpy(out,d,4*2); + break; + case G9P_A9_UP_SIZE: + if(!pvr_g5p_tex_size(d,u->level,out)) { + mesa_logw("G5P refuses: compute set %u binding %u: an image size without A9 texture state",u->set,u->binding); + return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P compute: an image size without A9 texture state"); + } + break; + default: + return vk_error(cmd_buffer,VK_ERROR_UNKNOWN); + } + } if (state->a9_device_fence) { uint32_t zero=0; struct pvr_suballoc_bo *fence; @@ -198,7 +263,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, } uniforms[2 * n] = count; result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, uniforms, - sizeof(uint32_t) * (2 * n + 1 + (state->a9_device_fence ? 2 : 0) + (images ? 1 : 0)), + sizeof(uint32_t) * (tex0 + 8 * nup), &uniforms_bo); if (result != VK_SUCCESS) return result; @@ -213,7 +278,12 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uint64_t src; uint32_t dest = 2 * k, dwords = 2; - if (k < n && state->buffers[k].set != G9P_PUSH_CONSTANTS) { + if (k >= base_transfers) { + const unsigned i = k - base_transfers; + src = uniforms_bo->dev_addr.addr + 4 * (tex0 + 8 * i); + dest = state->a9_uploads[i].common; + dwords = state->a9_uploads[i].dwords; + } else if (k < n && state->buffers[k].set != G9P_PUSH_CONSTANTS) { const struct pvr_descriptor_set *set = desc->sets[state->buffers[k].set]; const struct pvr_descriptor_set_layout_binding *binding; @@ -304,7 +374,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_csb *csb = &sub_cmd->control_stream; const struct g5p_cdm_record bind_rec = g5p_cdm_dma_task(G9P_CDM_DATA_BLOCKS(bind_dwords / 4), bind.data_offset, - bind.code_offset, (state->a9_device_fence || images) ? 4 : DIV_ROUND_UP(2 * n + 1, 16)); + bind.code_offset, (state->a9_device_fence || images || nup) ? 4 : DIV_ROUND_UP(2 * n + 1, 16)); const struct g5p_cdm_record kernel_rec = g5p_cdm_kernel( G9P_CDM_KERNEL_REGS(state->num_regs) | G9P_CDM_ANY_CLUSTER | G9P_CDM_FENCE | G9P_CDM_DATA_BLOCKS(kernel_dwords / 4), diff --git a/src/imagination/vulkan/pvr_g5p_compute_state.h b/src/imagination/vulkan/pvr_g5p_compute_state.h index 8e2bb1afceb..5b66c47edc5 100644 --- a/src/imagination/vulkan/pvr_g5p_compute_state.h +++ b/src/imagination/vulkan/pvr_g5p_compute_state.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: MIT */ #ifndef PVR_G5P_COMPUTE_STATE_H #define PVR_G5P_COMPUTE_STATE_H -#define PVR_G5P_COMPUTE_STATE_FIELDS bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; +#define PVR_G5P_COMPUTE_STATE_FIELDS bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ + unsigned a9_num_uploads; struct { unsigned char kind, set, binding, element, combined, level, common, dwords; } a9_uploads[7]; #define PVR_G5P_NATIVE_STORAGE(d) PVR_HAS_FEATURE((d), apple_g5p) #endif From 672d214f4e97c69e1a128871853d9ea672b3aec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:26 +0100 Subject: [PATCH 013/109] g9p: A9 compute values above r31 in bank 0 Assisted-by: LLM --- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/g9p/g9p_compile.c | 8 ++++-- src/imagination/g9p/g9p_isa.h | 43 ++++++++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index b831c30f02d..7aa0879d277 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -27,6 +27,7 @@ static void a9_tex_refuse(struct g9p_ctx *ctx, const char *why) { if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index b5d525ff053..daf76496b9f 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -4811,6 +4811,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.frag_coord_slot = frag_coord_slot; ctx.c = rzalloc(mem_ctx, struct g9p_code); ctx.c->fragment = stage == MESA_SHADER_FRAGMENT; + ctx.c->high_bank0 = stage == MESA_SHADER_COMPUTE && out->a9; ctx.reg = ralloc_array(mem_ctx, int8_t, impl->ssa_alloc * 4); ctx.uni = ralloc_array(mem_ctx, int16_t, impl->ssa_alloc * 4); memset(ctx.uni, -1, impl->ssa_alloc * 4 * sizeof(*ctx.uni)); @@ -4907,7 +4908,9 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.reserve_st_low = true; continue; } - if (!ctx.oom || stage == MESA_SHADER_COMPUTE || num_homed >= 1024 || g9p_no_homes) + /* A9 compute parks values above its bank-1 limit in bank 0. */ + if (!ctx.oom || (stage == MESA_SHADER_COMPUTE && !(out->a9)) || num_homed >= 1024 || + g9p_no_homes) return false; stuck = ctx.oom_ip == last_oom_ip ? stuck + 1 : 0; last_oom_ip = ctx.oom_ip; @@ -4957,7 +4960,8 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, const unsigned start = victim_decl ? ctx.reg_first[victim / 4] : ctx.def_ip[victim / 4]; /* a NIR register's range takes in the loops it is used in (use_scalar()) */ const unsigned end = ctx.last_use[victim]; - const unsigned max_homes = g9p_max_homes; + /* compute: at most 96 registers per lane (G9P_CDM_KERNEL_REGS' field; residency) */ + const unsigned max_homes = stage == MESA_SHADER_COMPUTE ? MIN2(g9p_max_homes, 64) : g9p_max_homes; int home = -1; for (unsigned h = 0; home < 0 && h < max_homes; h++) { diff --git a/src/imagination/g9p/g9p_isa.h b/src/imagination/g9p/g9p_isa.h index af114d3ec4a..d9c4c579a03 100644 --- a/src/imagination/g9p/g9p_isa.h +++ b/src/imagination/g9p/g9p_isa.h @@ -24,6 +24,9 @@ struct g9p_code { unsigned n; /* a fragment shader: registers in the other bank, see g9p_w0() */ bool fragment; + /* registers above r31 ("homes") in the fragment bank (bank 0) while the code's own stay in bank 1: A9 compute + * (g9p_mov_to_high()) */ + bool high_bank0; /* the code pushes onto the mask stack (needs g9p_push_init()) */ bool uses_stack; }; @@ -158,6 +161,25 @@ g9p_mov_imm(struct g9p_code *c, unsigned dst, uint32_t imm) } } +/* + * Word fields, named for the encoders below: W0 holds the class byte, the continuation bit 8 and, outside fragment + * code (g9p_w0()), the register-bank bits of sources B (bit 12) and A (bit 13); a destination word (g9p_dst()) + * the half index, the continuation and the bank bit 9; a source word (g9p_srcs()) source A's register at bit 2 + * and B's at bit 10, each with its last-read bit (7 and 15). A register source above r31 sets the source word's + * bit 0 and puts r >> 5 in an extension word (A at bit 1, B at bit 9). + */ +#define G9P_CONT 0x0100 +#define G9P_W0_SRC_B_BANK1 0x1000 +#define G9P_W0_SRC_A_BANK1 0x2000 +#define G9P_W1_BANK1 0x0200 +#define G9P_CLASS_ALU2 0x99 /* g9p_alu2()'s class: two register sources */ +#define G9P_W0_ALU2 (G9P_CLASS_ALU2 | G9P_CONT | G9P_W0_SRC_A_BANK1 | G9P_W0_SRC_B_BANK1) +#define G9P_SRC_A_SHIFT 2 +#define G9P_SRC_B_SHIFT 10 +#define G9P_SRCS_EXT 0x0001 +#define G9P_EXT_A_SHIFT 1 +#define G9P_EXT_B_SHIFT 9 + /* Two register sources; @kill marks both as last use */ static inline uint16_t g9p_srcs(unsigned a, unsigned b, bool kill) @@ -182,19 +204,34 @@ g9p_alu2(struct g9p_code *c, unsigned op, unsigned dst, unsigned a, unsigned b, * r31"). The compiler parks NIR registers there ("homes") and moves them * with OR, which copies any bit pattern. */ +/* + * A9 compute (c->high_bank0): its registers r0-r31 are in bank 1 (W0 bits 11-13, W1 bit 9), but the + * registers above r31 are bank 0's, as in Apple's A9 kernels: + * examined kernels write bank 0 up to r60 and bank 1 up to r2 or r7, and declare 64 + * registers (__GPU_METADATA r.0.0), so the count covers bank 0. The kernel's register allocation comes + * from it (CDM K0 bits 12+ = ceil(regs / 4), AGXMetalA9 ComputeProgramVariant 0x207d2c4b0 -> +0x9b8; + * DOUTU temps (regs & ~3) << 1, EncodeComputePDSProgram 0x207d1ea4c). Homes in bank 1 above r31 + * read wrong values on every lane of a 69-register kernel and lost the GPU; that bank 1 + * holds only 32, as in vertex code, is inferred. A home move keeps its low side in the code's bank and + * its high side in bank 0, the fragment form above (validated at r40/r77/r100 on the A9): an operand's + * header bit and the destination's W1 bit 9 name each side's bank. + */ static inline void g9p_mov_to_high(struct g9p_code *c, unsigned hi, unsigned src) { assert(src < 32 && hi < 128); - G9P_EMIT(c, g9p_w0(c, 0x3199), g9p_dst(c, hi), 0x0100 | src << 2 | src << 10, 0x00ce); + G9P_EMIT(c, g9p_w0(c, G9P_W0_ALU2), c->high_bank0 ? G9P_CONT | 2 * hi : g9p_dst(c, hi), + g9p_srcs(src, src, false), G9P_OR); } static inline void g9p_mov_from_high(struct g9p_code *c, unsigned dst, unsigned hi) { assert(dst < 32 && hi < 128); - G9P_EMIT(c, g9p_w0(c, 0x3199), g9p_dst(c, dst), 0x0101 | (hi & 31) << 2 | (hi & 31) << 10, - 0x0100 | (hi >> 5) << 1 | (hi >> 5) << 9, 0x00ce); + /* on A9 compute (high_bank0) both sources are bank 0's */ + G9P_EMIT(c, g9p_w0(c, G9P_W0_ALU2) & ~(c->high_bank0 ? G9P_W0_SRC_A_BANK1 | G9P_W0_SRC_B_BANK1 : 0), + g9p_dst(c, dst), g9p_srcs(hi & 31, hi & 31, false) | G9P_SRCS_EXT, + G9P_CONT | (hi >> 5) << G9P_EXT_A_SHIFT | (hi >> 5) << G9P_EXT_B_SHIFT, G9P_OR); } /* dst = src + imm, 0 <= imm < 64 */ From 8cca7587f8a1b267257f2f1c475d10e13e12d6d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:26 +0100 Subject: [PATCH 014/109] g9p: the A9 kernel record's register field counts bank 1 only On the A9 the CDM kernel record's register field (word 0, bits 12 and up) and the DOUTU temps size different banks. Apple's compute variant builds the field from its kernels' bank-1 count and the DOUTU temps from the bank-0 count. In all 164 A9 kernels compiled with Apple's offline compiler, the field is 8 registers (iOS's captured 0x2100), while the bank-0 count runs from 8 to 64. An A9 kernel now records its bank-1 count (at most 32) in g9p_shader::a9_bank1_regs, and the kernel record's field uses it. The DOUTU temps keep 32 + homes. The instruction stream is unchanged, and so is every kernel without homes. Assisted-by: LLM --- src/imagination/g9p/g9p_a9.h | 2 +- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/g9p/g9p_cdm.h | 7 +++++++ src/imagination/g9p/g9p_compile.c | 8 +++++++- src/imagination/vulkan/pvr_arch_g5p_compute.c | 6 +++++- src/imagination/vulkan/pvr_g5p_compute_state.h | 2 +- 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/imagination/g9p/g9p_a9.h b/src/imagination/g9p/g9p_a9.h index f3f3e0e8010..218b79c67dc 100644 --- a/src/imagination/g9p/g9p_a9.h +++ b/src/imagination/g9p/g9p_a9.h @@ -10,7 +10,7 @@ struct g9p_a9_upload { unsigned char level; /* a size block's level */ unsigned char common, dwords; }; -#define G9P_A9_FIELDS bool a9; bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; \ +#define G9P_A9_FIELDS bool a9; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; \ unsigned a9_num_uploads; struct g9p_a9_upload a9_uploads[G9P_A9_MAX_UPLOADS]; char a9_refusal[192]; #define G9P_A9_END_PAD(ctx) do { if ((ctx)->out->a9 && (2*(ctx)->c->n+6)%64==0) G9P_EMIT((ctx)->c,0xc000); } while(0) /* Keep main's G9P numbering; only the A9 backend uses NIR-order breaks. */ diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index 7aa0879d277..d3143a7da38 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -27,6 +27,7 @@ static void a9_tex_refuse(struct g9p_ctx *ctx, const char *why) { if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/g9p/g9p_cdm.h b/src/imagination/g9p/g9p_cdm.h index 09776d96a33..97d2d6b7c88 100644 --- a/src/imagination/g9p/g9p_cdm.h +++ b/src/imagination/g9p/g9p_cdm.h @@ -78,6 +78,13 @@ static inline uint32_t g9p_pds_bra_not_if1(uint32_t target) * COMPUTE.md "Register allocation"): field 2n holds 8n registers (field 1-2 * covers r0-r8, 3-4 r0-r16, 5-6 r0-r24, 8 r0-r32). Bit 8 observed set on * every iPadOS kernel record, meaning open. iPadOS's 0x2100 is field 2. + * + * On A9 this field allocates register bank 1 only (destination W1 bit 9 set: the bank G9P's compute code + * uses), and the DOUTU temps allocate bank 0. AGXMetalA9's ComputeProgramVariant (0x207d2c4b0) builds the + * record's word 0 (+0x9b8) from __GPU_METADATA r.3.0, which is 8 in all 164 examined A9 kernels + * (field 2, iPadOS's 0x2100), while EncodeComputePDSProgram (0x207d1ea4c) takes + * the DOUTU temps from r.0.0 (8..64, bank-0 values to r60). Fields 12 and 18 (48 and + * 72 registers, tested with bank-0 register parking) lost the GPU before any store (no CDM completion), so an A9 kernel passes its bank-1 count (at most 32: field 8, the largest measured). */ #define G9P_CDM_KERNEL_REGS(n) (0x100u | (2u * (((n) + 7) / 8)) << 12) #define G9P_CDM_KERNEL_SIZES G9P_CDM_KERNEL_REGS(8) diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index daf76496b9f..d7a9fb614d0 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -4960,7 +4960,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, const unsigned start = victim_decl ? ctx.reg_first[victim / 4] : ctx.def_ip[victim / 4]; /* a NIR register's range takes in the loops it is used in (use_scalar()) */ const unsigned end = ctx.last_use[victim]; - /* compute: at most 96 registers per lane (G9P_CDM_KERNEL_REGS' field; residency) */ + /* compute: at most 64 homes, bank 0 up to r95 (the DOUTU temps; residency) */ const unsigned max_homes = stage == MESA_SHADER_COMPUTE ? MIN2(g9p_max_homes, 64) : g9p_max_homes; int home = -1; @@ -5100,6 +5100,12 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, out->num_regs = MAX2(st + 1, ctx.num_homes || ctx.num_vs_high ? 32 + ctx.num_homes + ctx.num_vs_high : 0); + /* A9 compute: the code's own registers (bank 1, r0..r@st) are what the CDM record's register field + * allocates; its homes are bank 0's, which the DOUTU temps (num_regs) allocate (g9p_mov_to_high()) */ + if (ctx.c->high_bank0) { + assert(st + 1 <= 32); + out->a9_bank1_regs = st + 1; + } out->code = ctx.c->w; out->code_size = 2 * ctx.c->n; if (stage != MESA_SHADER_COMPUTE && st + 1 > 32) { diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 5a7203746af..17d5bd90fdc 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -88,6 +88,7 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( state->buffers[i].binding = shader.buffers[i].binding; } state->num_regs = shader.num_regs; + state->a9_bank1_regs = shader.a9_bank1_regs; state->a9_bodyless = shader.a9_bodyless; state->a9_unit_z = shader.a9_unit_z; state->a9_local_dwords = shader.a9_local_dwords; @@ -335,6 +336,8 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, * threadgroup words 0-1) in the coefficient task. */ g9p_pds_kick_data(kernel_data, usc_offset(device, state->shader_bo)); + /* the DOUTU temps allocate bank 0 (homes up to r(num_regs - 1)); the record's register field below + * allocates bank 1, the code's own registers (g9p_cdm.h G9P_CDM_KERNEL_REGS) */ kernel_data[1] = 0x10 * DIV_ROUND_UP(MAX2(state->num_regs, 1), 8); kernel_data[2] = 0x80000000; kernel_data[4] = 0x90000002; @@ -376,7 +379,8 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, g5p_cdm_dma_task(G9P_CDM_DATA_BLOCKS(bind_dwords / 4), bind.data_offset, bind.code_offset, (state->a9_device_fence || images || nup) ? 4 : DIV_ROUND_UP(2 * n + 1, 16)); const struct g5p_cdm_record kernel_rec = g5p_cdm_kernel( - G9P_CDM_KERNEL_REGS(state->num_regs) | G9P_CDM_ANY_CLUSTER | G9P_CDM_FENCE | + G9P_CDM_KERNEL_REGS(state->a9_bank1_regs ? state->a9_bank1_regs : state->num_regs) | G9P_CDM_ANY_CLUSTER | + G9P_CDM_FENCE | G9P_CDM_DATA_BLOCKS(kernel_dwords / 4), kernel.data_offset, kernel.code_offset, group_count[0], group_count[1], group_count[2], hw_threads, DIV_ROUND_UP(state->a9_local_dwords, 16)); diff --git a/src/imagination/vulkan/pvr_g5p_compute_state.h b/src/imagination/vulkan/pvr_g5p_compute_state.h index 5b66c47edc5..54a0b27b82d 100644 --- a/src/imagination/vulkan/pvr_g5p_compute_state.h +++ b/src/imagination/vulkan/pvr_g5p_compute_state.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: MIT */ #ifndef PVR_G5P_COMPUTE_STATE_H #define PVR_G5P_COMPUTE_STATE_H -#define PVR_G5P_COMPUTE_STATE_FIELDS bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ +#define PVR_G5P_COMPUTE_STATE_FIELDS unsigned a9_bank1_regs; bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ unsigned a9_num_uploads; struct { unsigned char kind, set, binding, element, combined, level, common, dwords; } a9_uploads[7]; #define PVR_G5P_NATIVE_STORAGE(d) PVR_HAS_FEATURE((d), apple_g5p) #endif From 8f5795581c5217a864564650c62880a07457df04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:26 +0100 Subject: [PATCH 015/109] g9p: name the A9 kernel record's bank-1 register field Assisted-by: LLM --- src/imagination/g9p/g9p_a9.h | 2 ++ src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/vulkan/pvr_arch_g5p_compute.c | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/imagination/g9p/g9p_a9.h b/src/imagination/g9p/g9p_a9.h index 218b79c67dc..ecb4408bad2 100644 --- a/src/imagination/g9p/g9p_a9.h +++ b/src/imagination/g9p/g9p_a9.h @@ -10,6 +10,8 @@ struct g9p_a9_upload { unsigned char level; /* a size block's level */ unsigned char common, dwords; }; +/* g9p_shader::a9_bank1_regs: the kernel record's register field allocates bank 1 (g9p_cdm.h) */ +#define G9P_A9_HAS_BANK1_REGS 1 #define G9P_A9_FIELDS bool a9; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; \ unsigned a9_num_uploads; struct g9p_a9_upload a9_uploads[G9P_A9_MAX_UPLOADS]; char a9_refusal[192]; #define G9P_A9_END_PAD(ctx) do { if ((ctx)->out->a9 && (2*(ctx)->c->n+6)%64==0) G9P_EMIT((ctx)->c,0xc000); } while(0) diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index d3143a7da38..af6edc778f5 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -27,6 +27,7 @@ static void a9_tex_refuse(struct g9p_ctx *ctx, const char *why) { if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 17d5bd90fdc..d2afcc8b2bc 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -338,6 +338,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, g9p_pds_kick_data(kernel_data, usc_offset(device, state->shader_bo)); /* the DOUTU temps allocate bank 0 (homes up to r(num_regs - 1)); the record's register field below * allocates bank 1, the code's own registers (g9p_cdm.h G9P_CDM_KERNEL_REGS) */ + const unsigned bank1 = state->a9_bank1_regs ? state->a9_bank1_regs : state->num_regs; kernel_data[1] = 0x10 * DIV_ROUND_UP(MAX2(state->num_regs, 1), 8); kernel_data[2] = 0x80000000; kernel_data[4] = 0x90000002; @@ -379,7 +380,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, g5p_cdm_dma_task(G9P_CDM_DATA_BLOCKS(bind_dwords / 4), bind.data_offset, bind.code_offset, (state->a9_device_fence || images || nup) ? 4 : DIV_ROUND_UP(2 * n + 1, 16)); const struct g5p_cdm_record kernel_rec = g5p_cdm_kernel( - G9P_CDM_KERNEL_REGS(state->a9_bank1_regs ? state->a9_bank1_regs : state->num_regs) | G9P_CDM_ANY_CLUSTER | + G9P_CDM_KERNEL_REGS(bank1) | G9P_CDM_ANY_CLUSTER | G9P_CDM_FENCE | G9P_CDM_DATA_BLOCKS(kernel_dwords / 4), kernel.data_offset, kernel.code_offset, group_count[0], group_count[1], From 3169b0973c306f3d87ca5fc014e5e3e14a3fd9ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:26 +0100 Subject: [PATCH 016/109] g9p: A9 compute pops keep the last-read bit for the last stack operation only G9P's mask-stack pops set the stack register's last-read bit (source word bit 7) on every pop. Apple's A9 compiler sets it only on the program's last stack operation; the A9 fragment backend already follows Apple. In Apple's divergent loop the loop's pop leaves the bit clear and the last pop sets it; ours marked an intermediate pop's counter register dead while lanes still waited on it. On the A9, kernels whose lanes leave a loop at different iterations lost the GPU before almost any store; uniform loops ran. A9 compute now pops without the bit, and the program's final pop keeps it. The A10 sets the bit on every pop and works, so its code is unchanged. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/g9p/g9p_compile.c | 1 + src/imagination/g9p/g9p_isa.h | 22 +++++++++++++++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index af6edc778f5..08ce75139bd 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -30,6 +30,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index d7a9fb614d0..09f27e4868e 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -4812,6 +4812,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.c = rzalloc(mem_ctx, struct g9p_code); ctx.c->fragment = stage == MESA_SHADER_FRAGMENT; ctx.c->high_bank0 = stage == MESA_SHADER_COMPUTE && out->a9; + ctx.c->a9_stack = stage == MESA_SHADER_COMPUTE && out->a9; ctx.reg = ralloc_array(mem_ctx, int8_t, impl->ssa_alloc * 4); ctx.uni = ralloc_array(mem_ctx, int16_t, impl->ssa_alloc * 4); memset(ctx.uni, -1, impl->ssa_alloc * 4 * sizeof(*ctx.uni)); diff --git a/src/imagination/g9p/g9p_isa.h b/src/imagination/g9p/g9p_isa.h index d9c4c579a03..915429ca13e 100644 --- a/src/imagination/g9p/g9p_isa.h +++ b/src/imagination/g9p/g9p_isa.h @@ -27,6 +27,9 @@ struct g9p_code { /* registers above r31 ("homes") in the fragment bank (bank 0) while the code's own stay in bank 1: A9 compute * (g9p_mov_to_high()) */ bool high_bank0; + /* A9 compute: Apple's A9 mask-stack rule, the stack register's last-read bit only on the program's + * last stack operation (g9p_pop()), not on every pop (g9p_pop_levels()) */ + bool a9_stack; /* the code pushes onto the mask stack (needs g9p_push_init()) */ bool uses_stack; }; @@ -462,10 +465,27 @@ g9p_else(struct g9p_code *c, unsigned st) G9P_EMIT(c, g9p_w0(c, 0x21c8), g9p_st_dst(c, st), g9p_st_src(c, st), 0x46d4); } +/* + * Pop @levels. Bit 7 of the source word is the stack register's last-read bit. G9P (A10) sets it on every + * pop; Apple's A9 compiler sets it only on the program's last stack operation (G5P's fragment port + * g5p_usc_stack() does the same, and so does Apple's captured divergent loop: the loop's pop leaves + * the bit clear, the last pop sets it). An intermediate pop that marks the register read for the last time + * lets the counter go while lanes still wait on it: on A9 compute a loop whose lanes leave at different + * iterations ((g & 3) + 2 rounds) lost the GPU before any store, while uniform loops ran. + */ +#define G9P_SRC_A_LAST_READ 0x0080 +#define G9P_CLASS_STACK 0xc8 /* the mask stack: its register as destination and as source A */ +#define G9P_W0_STACK (G9P_CLASS_STACK | G9P_CONT | G9P_W0_SRC_A_BANK1) +#define G9P_STACK_PUSH 0xd1 +#define G9P_STACK_POP 0xd2 +#define G9P_STACK_CTL 0x0600 /* set in every stack operation's last word (observed role) */ +#define G9P_STACK_LEVELS_SHIFT 14 static inline void g9p_pop_levels(struct g9p_code *c, unsigned st, unsigned levels) { - G9P_EMIT(c, g9p_w0(c, 0x21c8), g9p_st_dst(c, st), 0x0080 | g9p_st_src(c, st), 0x06d2 | levels << 14); + G9P_EMIT(c, g9p_w0(c, G9P_W0_STACK), g9p_st_dst(c, st), + (c->a9_stack ? 0 : G9P_SRC_A_LAST_READ) | g9p_st_src(c, st), + G9P_STACK_CTL | G9P_STACK_POP | levels << G9P_STACK_LEVELS_SHIFT); } /* Loops: start (two levels), break the lanes whose compare passed/failed */ From cd0ec06c6b358b0297cd07938ddd404a86ede03f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:26 +0100 Subject: [PATCH 017/109] pvr/g5p: refuse A9 compute workgroups above Apple's per-pipeline limit Apple's A9 driver caps each compute pipeline's threads per threadgroup by its register allocation. With t bank-0 registers (DOUTU temps) and f bank-1 registers (the record's field): min(32 * floor((3329 - a) / a), 393216 / b), at most 512, a multiple of 32, a = (4t + 28) & ~31, b = (64f + 448) & ~511. For a kernel with t = 64 and f = 8 this gives 384, the limit Metal reported for the same kernel on iOS. Vulkan has no per-pipeline limit, so an A9 compute pipeline whose workgroup exceeds its allocation's limit is now refused by name at creation: the USC cannot hold it. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/vulkan/pvr_arch_g5p_compute.c | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index 08ce75139bd..6cf292e7963 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -31,6 +31,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index d2afcc8b2bc..70433377cdd 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -75,6 +75,30 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( shader.num_regs); goto out; } + /* Apple's per-pipeline limit (AGXMetalA9 ComputeProgramVariant 0x207d2c4b0 -> +0xb08, which + * -[AGXA9FamilyComputePipeline maxTotalThreadsPerThreadgroup] returns), from the allocation the + * kernel task sends: bank 0 (the DOUTU temps, t) and bank 1 (the record's register field, f + * registers). min(32 * floor((3329 - a) / a) with a = (4t + 28) & ~31, 393216 / ((64f + 448) & ~511)), + * at most 512, a multiple of 32 above 32. k_hi48 (t 64, f 8) gives 384, the limit Metal reported for it + * on iOS. Vulkan has no per-pipeline limit, so a larger workgroup is refused by + * name: the USC cannot hold it. */ + if (shader.a9) { + const unsigned t = 8 * DIV_ROUND_UP(MAX2(shader.num_regs, 1), 8); + const unsigned f = 8 * DIV_ROUND_UP(shader.a9_bank1_regs ? shader.a9_bank1_regs : shader.num_regs, 8); + const unsigned a = (4 * t + 28) & ~31u, b = (64 * f + 448) & ~511u; + unsigned max = MIN2(a ? 32 * ((3329 - a) / a) : 512, b ? 393216 / b : 512); + max = MIN2(max, 512); + if (max > 32) + max &= ~31u; + if (threads > max) { + mesa_logw("G5P refuses: a %u-lane workgroup with %u bank-0 and %u bank-1 registers per lane, above " + "the %u lanes Apple's A9 driver allows for that allocation", threads, t, f, max); + result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, + "G5P compute: %u-lane workgroup above the %u-lane limit of its allocation", threads, + max); + goto out; + } + } } result = pvr_gpu_upload_usc(device, shader.code, shader.code_size, 64, From 921ef51b60bc88f37c7e0df9506420fad3ef6b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:26 +0100 Subject: [PATCH 018/109] pvr/g5p: compute bind slots for gl_NumWorkGroups, buffer array elements and sizes The A9 compute compiler emits G9P_NUM_WORKGROUPS slots, array-element bits in a buffer slot's binding and the G9P_BINDING_SIZE flag (arrayLength()), but the A9 bind loop took every slot as a plain (set, binding). A NumWorkGroups slot indexed the sets out of range, and so did an element or size slot's binding. Handle them as G9P's bind loop does (496e44739dc, 1d51ff5cec6): the dispatch's counts go in the slot (refused for an indirect dispatch, whose counts only the GPU holds), elements step by the binding's stride, and the size flag reads the descriptor's size word. A slot past the layout is an error instead of an out-of-range read. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/vulkan/pvr_arch_g5p_compute.c | 27 +++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index 6cf292e7963..e583107a640 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -32,6 +32,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 70433377cdd..4d381fb2798 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -219,6 +219,15 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, * push-constant buffer's address and the thread count. */ for (unsigned k = 0; k < n; k++) { + /* gl_NumWorkGroups (the A9 compiler's G9P_NUM_WORKGROUPS slots, g9p_compile.c): x and y in binding 0's + * slot, z in binding 1's (G9P's 496e44739dc ("pvr/g9p: gl_NumWorkGroups, workgroup ID z, storage buffer array elements in compute")). An indirect dispatch's counts are the GPU's. */ + if (state->buffers[k].set == G9P_NUM_WORKGROUPS) { + if (indirect_va) + return vk_errorf(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY, + "G5P: gl_NumWorkGroups in an indirect dispatch"); + uniforms[2 * k] = group_count[state->buffers[k].binding ? 2 : 0]; + uniforms[2 * k + 1] = state->buffers[k].binding ? 0 : group_count[1]; + } if (state->buffers[k].set == G9P_PUSH_CONSTANTS) { const struct pvr_push_constants *pc = &cmd_buffer->state.push_consts[PVR_STAGE_ALLOCATION_COMPUTE]; @@ -308,15 +317,23 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, src = uniforms_bo->dev_addr.addr + 4 * (tex0 + 8 * i); dest = state->a9_uploads[i].common; dwords = state->a9_uploads[i].dwords; - } else if (k < n && state->buffers[k].set != G9P_PUSH_CONSTANTS) { + } else if (k < n && state->buffers[k].set != G9P_PUSH_CONSTANTS && + state->buffers[k].set != G9P_NUM_WORKGROUPS) { const struct pvr_descriptor_set *set = desc->sets[state->buffers[k].set]; + /* an array's element (Zink keeps every storage buffer in one array) and the size flag (the + * descriptor's size word at byte 8, arrayLength()): G9P's 496e44739dc ("pvr/g9p: gl_NumWorkGroups, workgroup ID z, storage buffer array elements in compute"), 1d51ff5cec6 */ + const unsigned element = (state->buffers[k].binding & ~G9P_BINDING_SIZE) >> G9P_BINDING_ELEMENT_SHIFT; + const unsigned b = state->buffers[k].binding & BITFIELD_MASK(G9P_BINDING_ELEMENT_SHIFT); const struct pvr_descriptor_set_layout_binding *binding; - if (!set) + if (!set || b >= set->layout->binding_count) + return vk_error(cmd_buffer, VK_ERROR_UNKNOWN); + binding = &set->layout->bindings[b]; + if (element >= MAX2(binding->descriptor_count, 1)) return vk_error(cmd_buffer, VK_ERROR_UNKNOWN); - binding = &set->layout->bindings[state->buffers[k].binding]; - /* pvr_buffer_descriptor: the address comes first */ - src = set->dev_addr.addr + binding->offset; + /* pvr_buffer_descriptor: the address comes first, the size at byte 8 */ + src = set->dev_addr.addr + binding->offset + element * binding->stride + + (state->buffers[k].binding & G9P_BINDING_SIZE ? 8 : 0); } else if(k>n && ksets[state->a9_images[i].set]; From f13e53d14b768c6eebaa4067078d979347563f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:27 +0100 Subject: [PATCH 019/109] g9p: A9 compute keeps one task of bank-1 capacity free per workgroup Apple's per-pipeline limit for a bank-1 allocation of f registers is 393216 / ((64f + 448) & ~511) lanes: 768 at 8, 384 at 16, 256 at 24, 192 at 32. With 24 bank-1 registers one 256-lane group fills it, with no room for another group's first task. On the A9 every such kernel lost the GPU at 64 groups before its stores, while a single group ran. Apple's bank-0 term reserves one task, and Apple's kernels ask for only 8 registers in bank 1. A9 compute now caps its bank-1 registers (values and the mask stack) by workgroup size, so one task of that capacity stays free: 16 at 256 lanes, 24 up to 224, 32 up to 160. Values past the cap go to bank-0 homes. The per-pipeline refusal reserves the same task. Kernels already within the cap are unchanged. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/g9p/g9p_compile.c | 31 +++++++++++++++++-- src/imagination/vulkan/pvr_arch_g5p_compute.c | 3 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index e583107a640..df461b73e93 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -33,6 +33,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index 09f27e4868e..8f54f6af10d 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -51,12 +51,32 @@ g9p_num_regs(void) return e ? MIN2(atoi(e), 63) : 30; } #define G9P_NUM_REGS g9p_num_regs() + +/* + * A9 compute: the bank-1 registers per lane (values and the mask stack: the kernel record's register field) + * that a workgroup of @threads lanes may use. Apple's limit for a bank-1 allocation of f registers is + * 393216 / ((64f + 448) & ~511) lanes (AGXMetalA9 ComputeProgramVariant 0x207d2c4b0 -> +0xb08; its bank-0 + * term reserves one task's allocation). Apple's kernels always ask for 8 there, and put the rest in bank 0. + * Ours took up to 32: at 256 lanes and 24 (field 6), one group fills the whole bank-1 capacity, and 64 such + * groups lost the GPU before their stores; one group alone, or 16 registers, ran successfully. + * So a group keeps one task (32 lanes) of that capacity free, as Apple's bank-0 + * term does: 16 at 256 lanes, 24 up to 224, 32 up to 160. What does not fit goes to bank 0 (homes). + */ +static unsigned +g9p_a9_bank1_limit(unsigned threads) +{ + for (unsigned f = 32; f > 8; f -= 8) { + if (393216 / ((64 * f + 448) & ~511u) >= threads + 32) + return f; + } + return 8; +} /* * Registers for values: vertex shaders keep the mask stack in r15 (compile_back()), so * r30-r31 hold values too there (G9P_REGS still overrides, for tests) */ #define VALUE_REGS(ctx) ((ctx)->stage == MESA_SHADER_VERTEX && (ctx)->st == 15 && !getenv("G9P_REGS") \ - ? 32u : G9P_NUM_REGS) + ? 32u : (ctx)->value_cap ? MIN2(G9P_NUM_REGS, (ctx)->value_cap) : G9P_NUM_REGS) #define G9P_MAX_LOOPS 64 /* nir_tex_instr.backend_flags: a seamful cube's gradients, for the hardware */ #define G9P_TEX_NATIVE_GRADIENTS 1 @@ -126,6 +146,8 @@ struct g9p_vs_store { }; struct g9p_ctx { + /* A9 compute: at most this many value registers in bank 1 (0: no cap; g9p_a9_bank1_limit()) */ + unsigned value_cap; /* a sample iterated straight to the sampler has been emitted */ bool straight_done; /* @@ -4813,6 +4835,11 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.c->fragment = stage == MESA_SHADER_FRAGMENT; ctx.c->high_bank0 = stage == MESA_SHADER_COMPUTE && out->a9; ctx.c->a9_stack = stage == MESA_SHADER_COMPUTE && out->a9; + /* A9 compute: values in bank 1 up to the workgroup's limit, the mask stack after them; the rest in + * bank-0 homes (g9p_a9_bank1_limit()) */ + if (stage == MESA_SHADER_COMPUTE && out->a9) + ctx.value_cap = g9p_a9_bank1_limit(MAX2(out->local_size[0], 1) * MAX2(out->local_size[1], 1) * + MAX2(out->local_size[2], 1)) - 1; ctx.reg = ralloc_array(mem_ctx, int8_t, impl->ssa_alloc * 4); ctx.uni = ralloc_array(mem_ctx, int16_t, impl->ssa_alloc * 4); memset(ctx.uni, -1, impl->ssa_alloc * 4 * sizeof(*ctx.uni)); @@ -4901,7 +4928,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, bool victim_decl = false; ctx.oom = false; - if (generate(&ctx, impl, ctx.reserve_st_low ? 15 : G9P_NUM_REGS)) + if (generate(&ctx, impl, ctx.reserve_st_low ? 15 : out->a9 ? VALUE_REGS(&ctx) : G9P_NUM_REGS)) break; if (ctx.oom && stage == MESA_SHADER_VERTEX && (ctx.reserve_low < 2 || ctx.reserve_low == G9P_RESERVE_LOW_PAIR)) { diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 4d381fb2798..0fb65a4b934 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -85,8 +85,9 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( if (shader.a9) { const unsigned t = 8 * DIV_ROUND_UP(MAX2(shader.num_regs, 1), 8); const unsigned f = 8 * DIV_ROUND_UP(shader.a9_bank1_regs ? shader.a9_bank1_regs : shader.num_regs, 8); + /* bank 1 keeps one task (32 lanes) free, as Apple's bank-0 term does (g9p_a9_bank1_limit()) */ const unsigned a = (4 * t + 28) & ~31u, b = (64 * f + 448) & ~511u; - unsigned max = MIN2(a ? 32 * ((3329 - a) / a) : 512, b ? 393216 / b : 512); + unsigned max = MIN2(a ? 32 * ((3329 - a) / a) : 512, b ? 393216 / b - 32 : 512); max = MIN2(max, 512); if (max > 32) max &= ~31u; From 77eb3443fb749eac4a902423196b944751ecfcec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:27 +0100 Subject: [PATCH 020/109] g9p: A9 compute homes within the bank-0 limit of the workgroup Apple's per-pipeline limit for t DOUTU temps is 32 * floor((3329 - a) / a) lanes, a = (4t + 28) & ~31. With up to 64 homes a kernel could compile and then be refused at pipeline creation. The compiler now caps the homes by the workgroup's bank-0 limit (64 up to 192 lanes, 56 at 256, 32 at 384, 16 at 512) and refuses by name a kernel that needs more. The per-pipeline check stays as the guard. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/g9p/g9p_compile.c | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index df461b73e93..6eef3b85cf3 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -34,6 +34,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index 8f54f6af10d..a522900a271 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -62,6 +62,25 @@ g9p_num_regs(void) * So a group keeps one task (32 lanes) of that capacity free, as Apple's bank-0 * term does: 16 at 256 lanes, 24 up to 224, 32 up to 160. What does not fit goes to bank 0 (homes). */ +/* + * A9 compute: the homes (bank-0 registers above r31) a workgroup of @threads lanes may use. Apple's bank-0 + * limit for t DOUTU temps is 32 * floor((3329 - a) / a) lanes, a = (4t + 28) & ~31 (the same +0xb08 value); + * the temps are 32 + homes rounded up to 8. 256 lanes allow 88 temps (56 homes); at most 64 homes as before. + * A kernel that needs more is refused by name here, instead of at pipeline creation. + */ +static unsigned +g9p_a9_max_homes(unsigned threads) +{ + unsigned homes = 64; + while (homes > 0) { + const unsigned t = 8 * DIV_ROUND_UP(32 + homes, 8), a = (4 * t + 28) & ~31u; + if (32 * ((3329 - a) / a) >= threads) + break; + homes -= 8; + } + return homes; +} + static unsigned g9p_a9_bank1_limit(unsigned threads) { @@ -4989,7 +5008,12 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, /* a NIR register's range takes in the loops it is used in (use_scalar()) */ const unsigned end = ctx.last_use[victim]; /* compute: at most 64 homes, bank 0 up to r95 (the DOUTU temps; residency) */ - const unsigned max_homes = stage == MESA_SHADER_COMPUTE ? MIN2(g9p_max_homes, 64) : g9p_max_homes; + const unsigned max_homes = + stage != MESA_SHADER_COMPUTE ? g9p_max_homes + : out->a9 ? MIN2(g9p_max_homes, g9p_a9_max_homes(MAX2(out->local_size[0], 1) * + MAX2(out->local_size[1], 1) * + MAX2(out->local_size[2], 1))) + : MIN2(g9p_max_homes, 64); int home = -1; for (unsigned h = 0; home < 0 && h < max_homes; h++) { From c363255b78356f17f07bf6d127b7570191a5351f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:27 +0100 Subject: [PATCH 021/109] g9p: enforce the A9 compute bank-1 cap in every low-register allocator The bank-1 cap applied to the main allocator only. The low-register range, pair and window allocators could still hand out registers past it, and kernels compiled that way broke the one-free-task rule. Bound every low-register allocator by the cap, keep high-bank eviction inside it, and refuse by name a mask stack that would exceed it. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/g9p/g9p_compile.c | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index 6eef3b85cf3..828dd439183 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -35,6 +35,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index a522900a271..0b6f4476441 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -96,6 +96,7 @@ g9p_a9_bank1_limit(unsigned threads) */ #define VALUE_REGS(ctx) ((ctx)->stage == MESA_SHADER_VERTEX && (ctx)->st == 15 && !getenv("G9P_REGS") \ ? 32u : (ctx)->value_cap ? MIN2(G9P_NUM_REGS, (ctx)->value_cap) : G9P_NUM_REGS) +#define LOW_REGS(ctx) MIN2(16u, VALUE_REGS(ctx)) #define G9P_MAX_LOOPS 64 /* nir_tex_instr.backend_flags: a seamful cube's gradients, for the hardware */ #define G9P_TEX_NATIVE_GRADIENTS 1 @@ -417,7 +418,7 @@ reads_def(nir_src *src, void *data) static bool evict_low(struct g9p_ctx *ctx, unsigned lo, unsigned n) { - uint64_t high_free = ~ctx->busy & BITFIELD64_RANGE(16, VALUE_REGS(ctx) - 16); + uint64_t high_free = VALUE_REGS(ctx) > 16 ? ~ctx->busy & BITFIELD64_RANGE(16, VALUE_REGS(ctx) - 16) : 0; int keys[16]; if (ctx->c->fragment || lo + n > 16 || getenv("G9P_NO_EVICT")) @@ -461,7 +462,7 @@ evict_low(struct g9p_ctx *ctx, unsigned lo, unsigned n) return false; high_free &= high_free - 1; } - high_free = ~ctx->busy & BITFIELD64_RANGE(16, VALUE_REGS(ctx) - 16); + high_free = VALUE_REGS(ctx) > 16 ? ~ctx->busy & BITFIELD64_RANGE(16, VALUE_REGS(ctx) - 16) : 0; for (unsigned r = lo; r < lo + n; r++) { if (keys[r - lo] < 0) continue; @@ -483,7 +484,9 @@ evict_low(struct g9p_ctx *ctx, unsigned lo, unsigned n) static int evict_low_window(struct g9p_ctx *ctx, unsigned n) { - for (int r = 16 - n; r >= 0; r--) { + if (n > LOW_REGS(ctx)) + return -1; + for (int r = LOW_REGS(ctx) - n; r >= 0; r--) { if (evict_low(ctx, r, n)) return r; } @@ -494,7 +497,7 @@ evict_low_window(struct g9p_ctx *ctx, unsigned n) static int alloc_low_temp(struct g9p_ctx *ctx) { - for (unsigned r = 0; r < MIN2(16, G9P_NUM_REGS); r++) { + for (unsigned r = 0; r < LOW_REGS(ctx); r++) { if (!(ctx->busy & BITFIELD64_BIT(r))) { ctx->busy |= BITFIELD64_BIT(r); ctx->temps |= BITFIELD64_BIT(r); @@ -524,7 +527,7 @@ alloc_low_temp(struct g9p_ctx *ctx) static int alloc_low_range(struct g9p_ctx *ctx, unsigned n) { - for (unsigned r = 0; r + n <= MIN2(16, G9P_NUM_REGS); r++) { + for (unsigned r = 0; r + n <= LOW_REGS(ctx); r++) { if (!(ctx->busy & BITFIELD64_RANGE(r, n))) { ctx->busy |= BITFIELD64_RANGE(r, n); ctx->max_reg = MAX2(ctx->max_reg, r + n - 1); @@ -580,7 +583,7 @@ one_src(struct g9p_ctx *ctx, int r) static int alloc_pair(struct g9p_ctx *ctx) { - for (unsigned r = 0; r + 1 < MIN2(16, G9P_NUM_REGS); r++) { + for (unsigned r = 0; r + 1 < LOW_REGS(ctx); r++) { if (!(ctx->busy & BITFIELD64_RANGE(r, 2))) { ctx->busy |= BITFIELD64_RANGE(r, 2); ctx->max_reg = MAX2(ctx->max_reg, r + 1); @@ -5034,6 +5037,11 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, } } st = ctx.max_reg + 1; + if (ctx.c->high_bank0 && ctx.value_cap && st + 1 > ctx.value_cap + 1) { + if (!out->a9_refusal[0]) + snprintf(out->a9_refusal, sizeof(out->a9_refusal), "bank-1 cap exceeded"); + return false; + } /* * Vertex shaders: the mask stack below r16, selects use it too (dEQP vertex * shaders of 17 registers and more drew wrong values or hung the geometry From 6493c42e478d24fc3495685bbbed7654bf811c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 02:47:55 +0100 Subject: [PATCH 022/109] pvr/g5p: write the workgroup ID z on the A9 as Apple does g9p reads the workgroup ID z from threadgroup word 2, which the A10 kernel PDS program writes. The A9 PDS program only writes x and y (words 0-1), and on the A9 word 2 is the software barrier counter. An A9 kernel that reads gl_WorkGroupID.z or gl_GlobalInvocationID.z therefore read the barrier counter in a kernel with barriers and an unwritten word otherwise. Its z was wrong for any grid with more than one group in z, and right for one group only by chance. That includes the driver's own spill of large workgroup memory, which indexes each group's slot by x + nx * (y + ny * z). A nonzero value in that word put a group's slot outside the spill buffer. Apple's A9 Metal driver (AGXMetalA9 PDSEncoderGen1::EncodeComputePDSProgram) writes z only for a kernel that uses it: after the x/y DOUTW, whose control word then drops its last bit, it adds a DOUTW of temps q1's upper half whose control word has the last bit, bit 28, BSIZE 1 and the destination word at bit 13. That is the same form g9p already uses on the A10, so the hardware question is settled by Apple's binary and by the A10 runs. Place z in threadgroup word 3. Words 0-1 hold x and y, word 2 is the barrier counter, word 3 was otherwise unused, and shared variables start at word 4. The compiler records that a kernel reads z, and the dispatch adds the DOUTW for it after the barrier zero pair when there is one. Kernels that do not read z keep the same PDS and USC code. The dispatch no longer refuses a kernel specialized for one Z group: nothing sets that flag since the specialization was removed. This has not yet run on A9 hardware. Assisted-by: LLM --- src/imagination/g9p/g9p_a9.h | 7 ++++- src/imagination/g9p/g9p_a9_intrinsics.h | 2 +- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/g9p/g9p_compile.c | 6 +++- src/imagination/vulkan/pvr_arch_g5p_compute.c | 28 +++++++++++++------ .../vulkan/pvr_g5p_compute_state.h | 2 +- 6 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/imagination/g9p/g9p_a9.h b/src/imagination/g9p/g9p_a9.h index ecb4408bad2..ea7cc18b5d1 100644 --- a/src/imagination/g9p/g9p_a9.h +++ b/src/imagination/g9p/g9p_a9.h @@ -12,7 +12,12 @@ struct g9p_a9_upload { }; /* g9p_shader::a9_bank1_regs: the kernel record's register field allocates bank 1 (g9p_cdm.h) */ #define G9P_A9_HAS_BANK1_REGS 1 -#define G9P_A9_FIELDS bool a9; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; \ +/* A9 threadgroup word of the workgroup ID z (g9p_shader::a9_wgid_z: the kernel reads it). Words 0-1 hold x and y, + * word 2 the software-barrier counter (g5p_barrier.h, k = 2), shared variables start at word 4 (G9P_SHARED_BASE). + * The kernel PDS writes z here from temps q1's upper half, as AGXMetalA9's PDSEncoderGen1::EncodeComputePDSProgram + * (0x207d1ea4c) does after x and y when the kernel uses it (pvr_arch_g5p_compute.c). */ +#define G9P_A9_TG_WGID_Z 3 +#define G9P_A9_FIELDS bool a9; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; \ unsigned a9_num_uploads; struct g9p_a9_upload a9_uploads[G9P_A9_MAX_UPLOADS]; char a9_refusal[192]; #define G9P_A9_END_PAD(ctx) do { if ((ctx)->out->a9 && (2*(ctx)->c->n+6)%64==0) G9P_EMIT((ctx)->c,0xc000); } while(0) /* Keep main's G9P numbering; only the A9 backend uses NIR-order breaks. */ diff --git a/src/imagination/g9p/g9p_a9_intrinsics.h b/src/imagination/g9p/g9p_a9_intrinsics.h index 1b2703cc9ea..e885110bbe1 100644 --- a/src/imagination/g9p/g9p_a9_intrinsics.h +++ b/src/imagination/g9p/g9p_a9_intrinsics.h @@ -3,7 +3,7 @@ #include "../g5p/g5p_atomic.h" #include "../g5p/g5p_image.h" /* Static threadgroup operands: A9 host-compiler forms (compute.md §4). - * Reserve TG0/1 for IDs and TG2/3 for the future software-barrier counters. + * TG0/1 hold the workgroup IDs x/y, TG2 the software-barrier counter, TG3 the ID z (G9P_A9_TG_WGID_Z). * CDM allocation covers system words plus the native shared block. */ /* S and RI address forms from compute-isa-shared-atomic.md §1.3. * RI reads a temporary half index for the last time (short parcel8800). diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index 828dd439183..dbe019eb9e3 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -36,6 +36,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index 0b6f4476441..c9636902ed8 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -2977,7 +2977,11 @@ emit_intrinsic(struct g9p_ctx *ctx, nir_intrinsic_instr *intr) r = alloc_reg(ctx); if (r < 0) return false; - g9p_tg_load(ctx->c, r, c); + /* A9: z is in its own word after the barrier counter (g9p_a9.h G9P_A9_TG_WGID_Z) */ + const bool a9_z = ctx->out->a9 && c == 2; + g9p_tg_load(ctx->c, r, a9_z ? G9P_A9_TG_WGID_Z : c); + if (a9_z) + ctx->out->a9_wgid_z = true; ctx->reg[key(&intr->def, c)] = r; } g9p_wait(ctx->c); diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 0fb65a4b934..e4fdc2714df 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -115,7 +115,7 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( state->num_regs = shader.num_regs; state->a9_bank1_regs = shader.a9_bank1_regs; state->a9_bodyless = shader.a9_bodyless; - state->a9_unit_z = shader.a9_unit_z; + state->a9_wgid_z = shader.a9_wgid_z; state->a9_local_dwords = shader.a9_local_dwords; state->a9_has_barriers = shader.a9_has_barriers; state->a9_device_fence = shader.a9_device_fence; @@ -198,13 +198,6 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, * empty program does not retire under our CDM launch contract, so do not * launch it: the sub-command's ordinary completion tail still establishes * command-buffer ordering and signals the submit fence. */ - /* The compiler's unit-Z specialization is valid only for these actual - * dispatch dimensions. Indirect dimensions cannot be checked here. */ - if (state->a9_unit_z && (indirect_va || group_count[2] != 1)) { - return vk_errorf(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY, - "compute workgroup Z specialization requires a direct dispatch with one Z group"); - } - if (state->a9_bodyless) { /* An indirect bodyless dispatch has no consumer of its dimensions, * so its argument read is unobservable and is omitted like the kernel @@ -375,7 +368,8 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, /* * Kernel task: DOUTU source (USC offset, 0x10), the local-ID control * word (temps q0 -> r0) and the workgroup-ID one (temps q0, 64 bits -> - * threadgroup words 0-1) in the coefficient task. + * threadgroup words 0-1, and z from q1's upper half -> word 3 when the + * kernel reads it) in the coefficient task. */ g9p_pds_kick_data(kernel_data, usc_offset(device, state->shader_bo)); /* the DOUTU temps allocate bank 0 (homes up to r(num_regs - 1)); the record's register field below @@ -406,6 +400,22 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, kernel_code[5] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); kernel_code[6] = g9p_pds_dout(G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC); } + if (state->a9_wgid_z) { + /* Workgroup ID z, only for a kernel that reads it: AGXMetalA9's EncodeComputePDSProgram (0x207d1ea4c, flags + * bit 2) adds a DOUTW of temps q1's upper half (g9p_pds_dout(G9P_DOUTW, 1, G9P_PDS_TEMPS, ...); its source-1 + * word below: last, the common store, the destination word, the upper 32 bits) after the x/y one (all 64 + * bits), which then drops its last bit; G9P's 496e44739dc uses the same form on the A10. + * Our word is G9P_A9_TG_WGID_Z (3): word 2 is the barrier counter, whose zero pair is written first. */ + const unsigned at = state->a9_has_barriers ? 2 : 1; + kernel_data[4] = PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_DEST_COMMON_STORE | + PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_BSIZE_ALL64; + kernel_data[5] = PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_LAST_EN | + PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_DEST_COMMON_STORE | + G9P_A9_TG_WGID_Z << PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_AO_SHIFT | + PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_BSIZE_UPPER; + kernel_code[at] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, 0); + kernel_code[at + 1] = g9p_pds_dout(G9P_DOUTW, 1, G9P_PDS_TEMPS, 5, G9P_PDS_END); + } result = pvr_arch_cmd_buffer_upload_pds(cmd_buffer, kernel_data, kernel_dwords, ROGUE_CDMCTRL_KERNEL1_DATA_ADDR_ALIGNMENT, diff --git a/src/imagination/vulkan/pvr_g5p_compute_state.h b/src/imagination/vulkan/pvr_g5p_compute_state.h index 54a0b27b82d..671de5f87cd 100644 --- a/src/imagination/vulkan/pvr_g5p_compute_state.h +++ b/src/imagination/vulkan/pvr_g5p_compute_state.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: MIT */ #ifndef PVR_G5P_COMPUTE_STATE_H #define PVR_G5P_COMPUTE_STATE_H -#define PVR_G5P_COMPUTE_STATE_FIELDS unsigned a9_bank1_regs; bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ +#define PVR_G5P_COMPUTE_STATE_FIELDS unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ unsigned a9_num_uploads; struct { unsigned char kind, set, binding, element, combined, level, common, dwords; } a9_uploads[7]; #define PVR_G5P_NATIVE_STORAGE(d) PVR_HAS_FEATURE((d), apple_g5p) #endif From a6a1bb2911ead6d888ccf5f055e157d501b159a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:37 +0100 Subject: [PATCH 023/109] pvr/g5p: capture shaders and diagnostic stubs with PVR_DEBUG Use shader_dump with PVR_SHADER_DUMP_DIR, and capture_stub for the explicit opt-in that replaces refused shaders with incorrect pictures. Assisted-by: LLM --- src/imagination/common/pvr_debug.c | 2 + src/imagination/common/pvr_debug.h | 4 + src/imagination/vulkan/meson.build | 1 + src/imagination/vulkan/pvr_arch_g5p_compute.c | 37 ++- src/imagination/vulkan/pvr_arch_g5p_ds.c | 16 +- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 53 +++- src/imagination/vulkan/pvr_g5p_ds.c | 12 +- src/imagination/vulkan/pvr_g5p_shader_dump.c | 294 ++++++++++++++++++ src/imagination/vulkan/pvr_g5p_shader_dump.h | 57 ++++ src/imagination/vulkan/pvr_g5p_stub_spv.h | 57 ++++ 10 files changed, 516 insertions(+), 17 deletions(-) create mode 100644 src/imagination/vulkan/pvr_g5p_shader_dump.c create mode 100644 src/imagination/vulkan/pvr_g5p_shader_dump.h create mode 100644 src/imagination/vulkan/pvr_g5p_stub_spv.h diff --git a/src/imagination/common/pvr_debug.c b/src/imagination/common/pvr_debug.c index 764436641e0..9b6d8d1fd19 100644 --- a/src/imagination/common/pvr_debug.c +++ b/src/imagination/common/pvr_debug.c @@ -38,6 +38,8 @@ static const struct debug_named_value debug_control[] = { "Dump descriptor set and pipeline layouts." }, { "info", PVR_DEBUG_INFO, "Display information about the driver and device." }, + { "shader_dump", PVR_DEBUG_SHADER_DUMP, "Capture shader compilation inputs and results." }, + { "capture_stub", PVR_DEBUG_CAPTURE_STUB, "Replace refused shaders with diagnostic stubs; renders incorrect pixels." }, { "errors", PVR_DEBUG_ERRORS, "Log every Vulkan error the driver returns, with its origin." }, DEBUG_NAMED_VALUE_END }; diff --git a/src/imagination/common/pvr_debug.h b/src/imagination/common/pvr_debug.h index 7d205bcf7c1..98b820fc833 100644 --- a/src/imagination/common/pvr_debug.h +++ b/src/imagination/common/pvr_debug.h @@ -39,6 +39,10 @@ extern uint32_t PVR_DEBUG; #define PVR_DEBUG_VK_DUMP_DESCRIPTOR_SET_LAYOUT BITFIELD_BIT(2) #define PVR_DEBUG_INFO BITFIELD_BIT(3) +#define PVR_DEBUG_SHADER_DUMP BITFIELD_BIT(6) + +#define PVR_DEBUG_CAPTURE_STUB BITFIELD_BIT(7) + #define PVR_DEBUG_ERRORS BITFIELD_BIT(8) void pvr_process_debug_variable(void); diff --git a/src/imagination/vulkan/meson.build b/src/imagination/vulkan/meson.build index dcf1adf7c8d..05cbd5f82b8 100644 --- a/src/imagination/vulkan/meson.build +++ b/src/imagination/vulkan/meson.build @@ -164,6 +164,7 @@ pvr_files += files( 'pvr_g5p_formats.c', 'pvr_g5p_image.c', 'pvr_g5p_nir.c', + 'pvr_g5p_shader_dump.c', 'pvr_g5p_texel_buffer.c', 'pvr_g5p_xfb_query.c', 'pvr_g5p_ds.c', diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index e4fdc2714df..672d36226c5 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -23,6 +23,7 @@ #include "pvr_pipeline.h" #include "pvr_g5p.h" #include "pvr_g5p_tex_encode.h" +#include "pvr_g5p_shader_dump.h" #include "hwdef/rogue_hw_utils.h" #include "pvr_g5p_compute.h" #include "pvr_job_context.h" @@ -47,15 +48,39 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( &create_info->stage, g9p_spirv_options(), g9p_nir_options(), mem_ctx, &nir); - if (result != VK_SUCCESS) + if (result != VK_SUCCESS) { + pvr_g5p_shader_dump(&create_info->stage, create_info->layout, "comp", NULL, 0, "REFUSE", + "SPIR-V to NIR failed"); goto out; + } if (!g9p_a9_compile_compute(nir, &shader, mem_ctx)) { - mesa_logw("pvr/g5p: compute pipeline refused: %s", - shader.a9_refusal[0] ? shader.a9_refusal : "no reason recorded"); - result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, - "G5P compute: shader not supported by the backend yet"); - goto out; + /* Shader capture and diagnostic stubs (pvr_g5p_shader_dump.h). */ + const bool stub = pvr_g5p_capture_stub(); + char reason[sizeof(shader.a9_refusal)]; + snprintf(reason, sizeof(reason), "%s", shader.a9_refusal[0] ? shader.a9_refusal : "no reason recorded"); + pvr_g5p_shader_dump(&create_info->stage, create_info->layout, "comp", NULL, 0, stub ? "STUB" : "REFUSE", + reason); + if (!stub) { + mesa_logw("pvr/g5p: compute pipeline refused: %s", reason); + result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, + "G5P compute: shader not supported by the backend yet"); + goto out; + } + mesa_logw("G5P capture stub: compute shader refused (%s): empty kernel, dispatches omitted", reason); + VkPipelineShaderStageCreateInfo stub_info; + VkShaderModuleCreateInfo stub_module; + pvr_g5p_stub_stage(VK_SHADER_STAGE_COMPUTE_BIT, &stub_info, &stub_module); + result = vk_pipeline_shader_stage_to_nir(&device->vk, pipeline->base.pipeline_flags, &stub_info, + g9p_spirv_options(), g9p_nir_options(), mem_ctx, &nir); + if (result != VK_SUCCESS) + goto out; + if (!g9p_a9_compile_compute(nir, &shader, mem_ctx)) { + result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, "G5P compute: the capture stub was refused"); + goto out; + } + } else { + pvr_g5p_shader_dump(&create_info->stage, create_info->layout, "comp", NULL, 0, "ADMIT", NULL); } /* A barrier above 32 lanes waits for every task of the workgroup * (Apple's software barrier, g5p_barrier.h), so all of them must be diff --git a/src/imagination/vulkan/pvr_arch_g5p_ds.c b/src/imagination/vulkan/pvr_arch_g5p_ds.c index 13f01fdd4c9..e23a74f48b8 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_ds.c +++ b/src/imagination/vulkan/pvr_arch_g5p_ds.c @@ -22,6 +22,7 @@ #include "pvr_cmd_buffer.h" #include "pvr_csb.h" #include "pvr_g5p_ds.h" +#include "pvr_g5p_shader_dump.h" #include "pvr_g5p_tex_encode.h" #include "pvr_hw_pass.h" #include "pvr_image.h" @@ -74,9 +75,18 @@ PVR_PER_ARCH(g5p_ds_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_r " (pvr_g5p_ds_layout())", (uint64_t)image->total_size, plane.size); if (image->vk.samples == VK_SAMPLE_COUNT_4_BIT && (hw_render->depth_init == VK_ATTACHMENT_LOAD_OP_LOAD || hw_render->stencil_init == VK_ATTACHMENT_LOAD_OP_LOAD || - hw_render->depth_store || hw_render->stencil_store)) - return G5P_DS_REFUSE(cmd_buffer, "a load or store of a 4-sample depth/stencil attachment " - "(no multisample ZLS capture: it stays on chip)"); + hw_render->depth_store || hw_render->stencil_store)) { + if (!pvr_g5p_capture_stub()) + return G5P_DS_REFUSE(cmd_buffer, "a load or store of a 4-sample depth/stencil attachment " + "(no multisample ZLS capture: it stays on chip)"); + /* PVR_G5P_CAPTURE_STUB: the pass goes on, its ZLS load/store dropped (pvr_g5p_ds_zls()) */ + static bool told; + if (!told) { + told = true; + fprintf(stderr, "G5P capture stub: 4-sample depth/stencil load/store dropped (depth is not kept between " + "passes)\n"); + } + } return VK_SUCCESS; } diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 13d8b312102..6bd15ccca61 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -55,6 +55,7 @@ #include "pvr_g5p_tex_encode.h" #include "pvr_g5p_upload.h" #include "pvr_g5p_xfb.h" +#include "pvr_g5p_shader_dump.h" #include "pvr_g5p_ds.h" #include "pvr_g5p_msaa.h" #include "pvr_g5p_vertex.h" @@ -594,8 +595,26 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo key.depth_feedback = false; /* the fragment program first: the vertex program is compiled against the * varyings it reads (Apple's VertexProgramKey; Rogue's pco_rev_link_nir) */ - if (!pvr_g5p_compile_stage(fs_info, true, g->samples, NULL, &key, NULL, fs)) - return G5P_REFUSE(device, "fragment shader: %s", fs->refusal); + /* PVR_G5P_SHADER_DUMP / PVR_G5P_CAPTURE_STUB (pvr_g5p_shader_dump.h) */ + VkPipelineShaderStageCreateInfo stub_info, fs_stub_info; + VkShaderModuleCreateInfo stub_module, fs_stub_module; + const uint32_t dump_samples = g->samples; + const struct pvr_g5p_dump_blob fs_blobs[] = { { "fs_key", &key, sizeof(key) }, + { "samples", &dump_samples, sizeof(dump_samples) } }; + if (!pvr_g5p_compile_stage(fs_info, true, g->samples, NULL, &key, NULL, fs)) { + const bool stub = pvr_g5p_capture_stub(); + pvr_g5p_shader_dump(fs_info, create_info->layout, "frag", fs_blobs, ARRAY_SIZE(fs_blobs), + stub ? "STUB" : "REFUSE", fs->refusal); + if (!stub) + return G5P_REFUSE(device, "fragment shader: %s", fs->refusal); + mesa_logw("G5P capture stub: fragment shader refused (%s): magenta stub", fs->refusal); + *fs = (struct pco_g5p_program){ 0 }; + fs_info = pvr_g5p_stub_stage(VK_SHADER_STAGE_FRAGMENT_BIT, &fs_stub_info, &fs_stub_module); + if (!pvr_g5p_compile_stage(fs_info, true, g->samples, NULL, &key, NULL, fs)) + return G5P_REFUSE(device, "fragment shader stub: %s", fs->refusal); + } else { + pvr_g5p_shader_dump(fs_info, create_info->layout, "frag", fs_blobs, ARRAY_SIZE(fs_blobs), "ADMIT", NULL); + } struct pco_g5p_vs_link link = { .valid = true }; /* points (Apple's g3b-points): the point size after the position; the * topology class is fixed per pipeline even when the topology is dynamic */ @@ -605,9 +624,35 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo for (unsigned i = 0; i < fs->num_varyings; i++) link.interp[fs->vary[i].location] = fs->vary[i].interp; /* the transform feedback variant (pvr_g5p_xfb.h): the capturing vertex program */ + /* the vertex input state itself (the layout holds a pointer to it) and the link, for replay */ + const struct pvr_g5p_dump_blob vs_blobs[] = { { "vertex_input", s->vi, s->vi ? sizeof(*s->vi) : 0 }, + { "dynamic_strides", &vertex.dynamic_strides, 1 }, + { "vs_link", &link, sizeof(link) } }; if (g->xfb_variant ? !pvr_g5p_compile_vs_xfb(vs_info, &vertex, &link, &g->xfb_layout, vs) - : !pvr_g5p_compile_stage(vs_info, false, 1, &vertex, NULL, &link, vs)) - return G5P_REFUSE(device, "vertex shader%s: %s", g->xfb_variant ? " (transform feedback)" : "", vs->refusal); + : !pvr_g5p_compile_stage(vs_info, false, 1, &vertex, NULL, &link, vs)) { + const bool stub = pvr_g5p_capture_stub() && !g->xfb_variant; + pvr_g5p_shader_dump(vs_info, create_info->layout, g->xfb_variant ? "vert-xfb" : "vert", vs_blobs, + ARRAY_SIZE(vs_blobs), stub ? "STUB" : "REFUSE", vs->refusal); + if (!stub) + return G5P_REFUSE(device, "vertex shader%s: %s", g->xfb_variant ? " (transform feedback)" : "", vs->refusal); + mesa_logw("G5P capture stub: vertex shader refused (%s): point stub, nothing drawn", vs->refusal); + *vs = (struct pco_g5p_program){ 0 }; + vs_info = pvr_g5p_stub_stage(VK_SHADER_STAGE_VERTEX_BIT, &stub_info, &stub_module); + /* the stub writes no varyings: a fragment program reading some becomes the magenta stub too */ + if (fs->num_varyings) { + free(fs->code); + *fs = (struct pco_g5p_program){ 0 }; + fs_info = pvr_g5p_stub_stage(VK_SHADER_STAGE_FRAGMENT_BIT, &fs_stub_info, &fs_stub_module); + if (!pvr_g5p_compile_stage(fs_info, true, g->samples, NULL, &key, NULL, fs)) + return G5P_REFUSE(device, "fragment shader stub: %s", fs->refusal); + memset(link.interp, 0, sizeof(link.interp)); + } + if (!pvr_g5p_compile_stage(vs_info, false, 1, &vertex, NULL, &link, vs)) + return G5P_REFUSE(device, "vertex shader stub: %s", vs->refusal); + } else { + pvr_g5p_shader_dump(vs_info, create_info->layout, g->xfb_variant ? "vert-xfb" : "vert", vs_blobs, + ARRAY_SIZE(vs_blobs), "ADMIT", NULL); + } if (g->xfb_variant && !g->xfb_layout.buffers) return VK_SUCCESS; /* captures nothing: no variant */ /* point fill rasterizes ISP points of one size per draw (pointlinewidth): a per-vertex diff --git a/src/imagination/vulkan/pvr_g5p_ds.c b/src/imagination/vulkan/pvr_g5p_ds.c index 911ce58e565..151020e74ce 100644 --- a/src/imagination/vulkan/pvr_g5p_ds.c +++ b/src/imagination/vulkan/pvr_g5p_ds.c @@ -21,6 +21,7 @@ #include "csbgen/rogue/ppp.h" #include "pvr_g5p_ds.h" +#include "pvr_g5p_shader_dump.h" static uint32_t g5p_pot(uint32_t v) @@ -129,10 +130,13 @@ pvr_g5p_ds_zls(const struct pvr_g5p_ds_target *t, struct pvr_g5p_zls_words *out) if (!pvr_g5p_ds_layout(t->format, t->width, t->height, t->samples, &plane)) return "a depth/stencil attachment of 0 pixels, or not 1 or 4 samples"; const bool d = t->depth && f->depth, s = t->stencil && f->stencil; - const bool zload = d && t->zload, zstore = d && t->zstore; - const bool sload = s && t->sload, sstore = s && t->sstore; - if (t->samples == 4 && (zload || zstore || sload || sstore)) - return "a load or store of a 4-sample depth/stencil attachment (no multisample ZLS capture: it stays on chip)"; + bool zload = d && t->zload, zstore = d && t->zstore; + bool sload = s && t->sload, sstore = s && t->sstore; + if (t->samples == 4 && (zload || zstore || sload || sstore)) { + if (!pvr_g5p_capture_stub()) + return "a load or store of a 4-sample depth/stencil attachment (no multisample ZLS capture: it stays on chip)"; + zload = zstore = sload = sstore = false; /* PVR_G5P_CAPTURE_STUB: dropped, logged at the pass */ + } const struct ROGUE_CR_ISP_ZLSCTL ctl = { .loadtwiddled = true, diff --git a/src/imagination/vulkan/pvr_g5p_shader_dump.c b/src/imagination/vulkan/pvr_g5p_shader_dump.c new file mode 100644 index 00000000000..a12a21401c1 --- /dev/null +++ b/src/imagination/vulkan/pvr_g5p_shader_dump.c @@ -0,0 +1,294 @@ +/* SPDX-License-Identifier: MIT */ +/* Shader capture controlled by PVR_DEBUG (pvr_g5p_shader_dump.h). */ +#include "pvr_g5p_shader_dump.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/macros.h" +#include "util/log.h" +#include "pvr_debug.h" +#include "util/u_debug.h" +#include "c11/threads.h" + +DEBUG_GET_ONCE_OPTION(pvr_shader_dump_dir, "PVR_SHADER_DUMP_DIR", "/tmp/pvr-shaders") +#include "util/mesa-blake3.h" +#include "util/u_atomic.h" +#include "util/u_call_once.h" +#include "vk_pipeline_layout.h" +#include "vk_shader_module.h" +#include "vk_util.h" +#include "pvr_descriptor_set.h" +#include "pvr_g5p_stub_spv.h" + +static const char *dump_dir; +static bool stub_on; +static util_once_flag env_once = UTIL_ONCE_FLAG_INIT; + +static void +read_env(void) +{ + const char *d = PVR_IS_DEBUG_SET(SHADER_DUMP) ? debug_get_option_pvr_shader_dump_dir() : NULL; + if (d && *d) { + mkdir(d, 0755); /* may exist */ + dump_dir = d; + /* once per process: a sandboxed process may not write there (Firefox: MOZ_DISABLE_GPU_SANDBOX=1) */ + if (access(d, W_OK)) + mesa_logw("G5P shader dump: cannot write %s (pid %ld): %s", d, (long)getpid(), strerror(errno)); + else + mesa_logi("G5P shader dump: %s (pid %ld)", d, (long)getpid()); + } + stub_on = PVR_IS_DEBUG_SET(CAPTURE_STUB); + if (stub_on) + mesa_logw("G5P capture stub: ON (refused shaders are replaced; pictures will be wrong)"); +} + +const char * +pvr_g5p_shader_dump_dir(void) +{ + util_call_once(&env_once, read_env); + return dump_dir; +} + +bool +pvr_g5p_capture_stub(void) +{ + util_call_once(&env_once, read_env); + return stub_on; +} + +static bool +stage_spirv(const VkPipelineShaderStageCreateInfo *info, const uint32_t **words, size_t *bytes) +{ + VK_FROM_HANDLE(vk_shader_module, module, info->module); + const VkShaderModuleCreateInfo *inline_ci = vk_find_struct_const(info->pNext, SHADER_MODULE_CREATE_INFO); + if (module && !module->nir) { + *words = (const uint32_t *)module->data; + *bytes = module->size; + return true; + } + if (inline_ci) { + *words = inline_ci->pCode; + *bytes = inline_ci->codeSize; + return true; + } + return false; /* a driver-internal NIR module */ +} + +/* write @size bytes to @path through a temporary name and a rename */ +static void +write_atomic(const char *path, const void *data, size_t size, bool keep_existing) +{ + static uint32_t serial; + if (keep_existing && access(path, F_OK) == 0) + return; + char tmp[4096]; + snprintf(tmp, sizeof(tmp), "%s.tmp.%ld.%u", path, (long)getpid(), p_atomic_inc_return(&serial)); + int fd = open(tmp, O_WRONLY | O_CREAT | O_EXCL, 0644); + if (fd < 0) + return; + const char *p = data; + size_t left = size; + while (left) { + ssize_t n = write(fd, p, left); + if (n <= 0) { + if (n < 0 && errno == EINTR) + continue; + break; + } + p += n, left -= n; + } + close(fd); + if (left || rename(tmp, path)) + unlink(tmp); +} + +/* a growing string */ +struct sbuf { + char *s; + size_t len, cap; +}; + +static void +sb_printf(struct sbuf *b, const char *fmt, ...) PRINTFLIKE(2, 3); + +static void +sb_printf(struct sbuf *b, const char *fmt, ...) +{ + for (;;) { + va_list ap; + va_start(ap, fmt); + int n = vsnprintf(b->s ? b->s + b->len : NULL, b->s ? b->cap - b->len : 0, fmt, ap); + va_end(ap); + if (n < 0) + return; + if (b->s && b->len + n < b->cap) { + b->len += n; + return; + } + size_t cap = MAX2(b->cap * 2, b->len + n + 256); + char *s = realloc(b->s, cap); + if (!s) + return; + b->s = s, b->cap = cap; + } +} + +static void +sb_json_string(struct sbuf *b, const char *s) +{ + sb_printf(b, "\""); + for (; s && *s; s++) { + const unsigned char c = *s; + if (c == '"' || c == '\\') + sb_printf(b, "\\%c", c); + else if (c < 0x20) + sb_printf(b, "\\u%04x", c); + else + sb_printf(b, "%c", c); + } + sb_printf(b, "\""); +} + +static void +sb_hex(struct sbuf *b, const void *data, size_t size) +{ + sb_printf(b, "\""); + for (size_t i = 0; i < size; i++) + sb_printf(b, "%02x", ((const uint8_t *)data)[i]); + sb_printf(b, "\""); +} + +static const char * +descriptor_type_name(VkDescriptorType t) +{ + switch (t) { + case VK_DESCRIPTOR_TYPE_SAMPLER: return "sampler"; + case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: return "combined_image_sampler"; + case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: return "sampled_image"; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: return "storage_image"; + case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: return "uniform_texel_buffer"; + case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: return "storage_texel_buffer"; + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: return "uniform_buffer"; + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: return "storage_buffer"; + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: return "uniform_buffer_dynamic"; + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: return "storage_buffer_dynamic"; + case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT: return "input_attachment"; + default: return "other"; + } +} + +void +pvr_g5p_shader_dump(const VkPipelineShaderStageCreateInfo *info, VkPipelineLayout layout_handle, + const char *stage, const struct pvr_g5p_dump_blob *blobs, unsigned nblobs, + const char *result, const char *reason) +{ + const char *dir = pvr_g5p_shader_dump_dir(); + const uint32_t *words; + size_t bytes; + if (!dir || !info || !stage_spirv(info, &words, &bytes)) + return; + + /* named by the first 20 bytes (40 hex digits) of the module's BLAKE3 */ + blake3_hash sha; + char hex[BLAKE3_HEX_LEN]; + _mesa_blake3_compute(words, bytes, sha); + _mesa_blake3_format(hex, sha); + hex[40] = 0; + char path[4096]; + snprintf(path, sizeof(path), "%s/%s.spv", dir, hex); + write_atomic(path, words, bytes, true); + + /* everything that decides the compile, then the result */ + struct sbuf b = { 0 }; + sb_printf(&b, "{\n \"spirv\": \"%s.spv\",\n \"stage\": \"%s\",\n \"entry\": ", hex, stage); + sb_json_string(&b, info->pName); + const VkSpecializationInfo *spec = info->pSpecializationInfo; + sb_printf(&b, ",\n \"specialization\": {\"entries\": ["); + for (uint32_t i = 0; spec && i < spec->mapEntryCount; i++) + sb_printf(&b, "%s[%u, %u, %zu]", i ? ", " : "", spec->pMapEntries[i].constantID, spec->pMapEntries[i].offset, + spec->pMapEntries[i].size); + sb_printf(&b, "], \"data\": "); + sb_hex(&b, spec ? spec->pData : NULL, spec ? spec->dataSize : 0); + sb_printf(&b, "},\n \"layout\": ["); + VK_FROM_HANDLE(vk_pipeline_layout, layout, layout_handle); + for (uint32_t s = 0; layout && s < layout->set_count; s++) { + const struct vk_descriptor_set_layout *vs = layout->set_layouts[s]; + sb_printf(&b, "%s\n [", s ? "," : ""); + if (vs) { + const struct pvr_descriptor_set_layout *ps = container_of(vs, struct pvr_descriptor_set_layout, vk); + bool first = true; + for (uint32_t i = 0; i < ps->binding_count; i++) { + const struct pvr_descriptor_set_layout_binding *pb = &ps->bindings[i]; + if (!pb->descriptor_count) + continue; + sb_printf(&b, "%s{\"binding\": %u, \"type\": \"%s\", \"count\": %u, \"stages\": %u}", first ? "" : ", ", + i, descriptor_type_name(pb->type), pb->descriptor_count, pb->stage_flags); + first = false; + } + } + sb_printf(&b, "]"); + } + sb_printf(&b, "],\n \"push_constants\": ["); + for (uint32_t i = 0; layout && i < layout->push_range_count; i++) + sb_printf(&b, "%s[%u, %u, %u]", i ? ", " : "", layout->push_ranges[i].stageFlags, + layout->push_ranges[i].offset, layout->push_ranges[i].size); + sb_printf(&b, "],\n \"keys\": {"); + for (unsigned i = 0; i < nblobs; i++) { + sb_printf(&b, "%s\"%s\": ", i ? ", " : "", blobs[i].name); + sb_hex(&b, blobs[i].data, blobs[i].size); + } + sb_printf(&b, "}"); + if (!b.s) + return; + /* the variant: everything above */ + blake3_hash vsha; + char vhex[BLAKE3_HEX_LEN]; + _mesa_blake3_compute(b.s, b.len, vsha); + _mesa_blake3_format(vhex, vsha); + vhex[12] = 0; + sb_printf(&b, ",\n \"result\": \"%s\",\n \"reason\": ", result); + sb_json_string(&b, reason ? reason : ""); + sb_printf(&b, "\n}\n"); + snprintf(path, sizeof(path), "%s/%s.%s.%s.json", dir, hex, stage, vhex); + if (b.s) + write_atomic(path, b.s, b.len, false); + free(b.s); + + /* one index line, one append */ + char line[1024]; + int n = snprintf(line, sizeof(line), "%lld\t%ld\t%s\t%s\t%s\t%s\t", (long long)time(NULL), (long)getpid(), + hex, stage, vhex, result); + for (const char *r = reason ? reason : ""; *r && n < (int)sizeof(line) - 2; r++) + line[n++] = *r == '\t' || *r == '\n' ? ' ' : *r; + line[MIN2(n, (int)sizeof(line) - 2)] = '\n'; + n = MIN2(n + 1, (int)sizeof(line) - 1); + snprintf(path, sizeof(path), "%s/index.tsv", dir); + int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (fd >= 0) { + ssize_t w = write(fd, line, n); + (void)w; + close(fd); + } +} + +const VkPipelineShaderStageCreateInfo * +pvr_g5p_stub_stage(VkShaderStageFlagBits stage, VkPipelineShaderStageCreateInfo *info, VkShaderModuleCreateInfo *module) +{ + const uint32_t *code = stage == VK_SHADER_STAGE_FRAGMENT_BIT ? pvr_g5p_stub_frag : + stage == VK_SHADER_STAGE_VERTEX_BIT ? pvr_g5p_stub_vert : pvr_g5p_stub_comp; + const size_t size = stage == VK_SHADER_STAGE_FRAGMENT_BIT ? sizeof(pvr_g5p_stub_frag) : + stage == VK_SHADER_STAGE_VERTEX_BIT ? sizeof(pvr_g5p_stub_vert) : sizeof(pvr_g5p_stub_comp); + *module = (VkShaderModuleCreateInfo){ .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, + .codeSize = size, .pCode = code }; + *info = (VkPipelineShaderStageCreateInfo){ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .pNext = module, .stage = stage, .pName = "main" }; + return info; +} diff --git a/src/imagination/vulkan/pvr_g5p_shader_dump.h b/src/imagination/vulkan/pvr_g5p_shader_dump.h new file mode 100644 index 00000000000..cc62afcb9f8 --- /dev/null +++ b/src/imagination/vulkan/pvr_g5p_shader_dump.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Shader capture for compatibility triage (diagnostic; off unless set). + * + * PVR_DEBUG=shader_dump (PVR_SHADER_DUMP_DIR=): every shader stage the G5P backend compiles is + * recorded in : + * .spv the SPIR-V module, once (first 40 hex digits of + * the BLAKE3 of its words) + * ...json the compile: stage, entry point, specialization + * constants, pipeline layout, the backend's key + * structs as hex (fragment key, vertex layout, + * link; enough for an offline admission check + * replay), and the result: accepted, or refused + * with the backend's reason. = a hash of + * everything but the result. + * index.tsv one line per compile: unix time, pid, hash, + * stage, v, ADMIT/REFUSE/STUB, the reason + * Files are written to a temporary name and renamed (several threads and + * processes may compile at once); index lines are single appends. + * + * PVR_DEBUG=capture_stub: a refused stage is replaced by a stub with the + * same interface (fragment: magenta at location 0; vertex: a point at the + * origin, nothing drawn; compute: an empty kernel, the dispatch omitted), so + * the pipeline is created and the application keeps creating the rest. The + * refusal is still logged ("G5P capture stub: ...") and dumped. Never on by + * default: stubbed pipelines draw wrong pictures. + */ +#ifndef PVR_G5P_SHADER_DUMP_H +#define PVR_G5P_SHADER_DUMP_H + +#include +#include "c11/threads.h" +#include +#include + +struct pvr_g5p_dump_blob { + const char *name; + const void *data; + size_t size; +}; + +/* NULL when PVR_DEBUG=shader_dump is unset (read once) */ +const char *pvr_g5p_shader_dump_dir(void); +bool pvr_g5p_capture_stub(void); + +/* Record one compile of @stage_info (no-op without PVR_DEBUG=shader_dump). + * @result: "ADMIT", "REFUSE" or "STUB" (refused, then stubbed). */ +void pvr_g5p_shader_dump(const VkPipelineShaderStageCreateInfo *stage_info, VkPipelineLayout layout, + const char *stage, const struct pvr_g5p_dump_blob *blobs, unsigned nblobs, + const char *result, const char *reason); + +/* A stage create info for the stub of @stage, in caller storage */ +const VkPipelineShaderStageCreateInfo *pvr_g5p_stub_stage(VkShaderStageFlagBits stage, + VkPipelineShaderStageCreateInfo *info, + VkShaderModuleCreateInfo *module); + +#endif diff --git a/src/imagination/vulkan/pvr_g5p_stub_spv.h b/src/imagination/vulkan/pvr_g5p_stub_spv.h new file mode 100644 index 00000000000..4dbf80640ec --- /dev/null +++ b/src/imagination/vulkan/pvr_g5p_stub_spv.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +/* PVR_DEBUG=capture_stub substitutes (pvr_g5p_shader_dump.c), glslangValidator -V --target-env vulkan1.1: + * frag: layout(location=0) out vec4 c; c = vec4(1, 0, 1, 1) (magenta) + * vert: gl_Position = vec4(0, 0, 0, 1) (a point: draws nothing) + * comp: local_size 1, empty (a bodyless dispatch: omitted) */ +#ifndef PVR_G5P_STUB_SPV_H +#define PVR_G5P_STUB_SPV_H +#include +static const uint32_t pvr_g5p_stub_frag[] = { + 0x07230203,0x00010300,0x0008000b,0x0000000d,0x00000000,0x00020011,0x00000001,0x0006000b, + 0x00000001,0x4c534c47,0x6474732e,0x3035342e,0x00000000,0x0003000e,0x00000000,0x00000001, + 0x0006000f,0x00000004,0x00000004,0x6e69616d,0x00000000,0x00000009,0x00030010,0x00000004, + 0x00000007,0x00030003,0x00000002,0x000001c2,0x00040005,0x00000004,0x6e69616d,0x00000000, + 0x00030005,0x00000009,0x00000063,0x00040047,0x00000009,0x0000001e,0x00000000,0x00020013, + 0x00000002,0x00030021,0x00000003,0x00000002,0x00030016,0x00000006,0x00000020,0x00040017, + 0x00000007,0x00000006,0x00000004,0x00040020,0x00000008,0x00000003,0x00000007,0x0004003b, + 0x00000008,0x00000009,0x00000003,0x0004002b,0x00000006,0x0000000a,0x3f800000,0x0004002b, + 0x00000006,0x0000000b,0x00000000,0x0007002c,0x00000007,0x0000000c,0x0000000a,0x0000000b, + 0x0000000a,0x0000000a,0x00050036,0x00000002,0x00000004,0x00000000,0x00000003,0x000200f8, + 0x00000005,0x0003003e,0x00000009,0x0000000c,0x000100fd,0x00010038 +}; +static const uint32_t pvr_g5p_stub_vert[] = { + 0x07230203,0x00010300,0x0008000b,0x00000015,0x00000000,0x00020011,0x00000001,0x0006000b, + 0x00000001,0x4c534c47,0x6474732e,0x3035342e,0x00000000,0x0003000e,0x00000000,0x00000001, + 0x0006000f,0x00000000,0x00000004,0x6e69616d,0x00000000,0x0000000d,0x00030003,0x00000002, + 0x000001c2,0x00040005,0x00000004,0x6e69616d,0x00000000,0x00060005,0x0000000b,0x505f6c67, + 0x65567265,0x78657472,0x00000000,0x00060006,0x0000000b,0x00000000,0x505f6c67,0x7469736f, + 0x006e6f69,0x00070006,0x0000000b,0x00000001,0x505f6c67,0x746e696f,0x657a6953,0x00000000, + 0x00070006,0x0000000b,0x00000002,0x435f6c67,0x4470696c,0x61747369,0x0065636e,0x00070006, + 0x0000000b,0x00000003,0x435f6c67,0x446c6c75,0x61747369,0x0065636e,0x00030005,0x0000000d, + 0x00000000,0x00030047,0x0000000b,0x00000002,0x00050048,0x0000000b,0x00000000,0x0000000b, + 0x00000000,0x00050048,0x0000000b,0x00000001,0x0000000b,0x00000001,0x00050048,0x0000000b, + 0x00000002,0x0000000b,0x00000003,0x00050048,0x0000000b,0x00000003,0x0000000b,0x00000004, + 0x00020013,0x00000002,0x00030021,0x00000003,0x00000002,0x00030016,0x00000006,0x00000020, + 0x00040017,0x00000007,0x00000006,0x00000004,0x00040015,0x00000008,0x00000020,0x00000000, + 0x0004002b,0x00000008,0x00000009,0x00000001,0x0004001c,0x0000000a,0x00000006,0x00000009, + 0x0006001e,0x0000000b,0x00000007,0x00000006,0x0000000a,0x0000000a,0x00040020,0x0000000c, + 0x00000003,0x0000000b,0x0004003b,0x0000000c,0x0000000d,0x00000003,0x00040015,0x0000000e, + 0x00000020,0x00000001,0x0004002b,0x0000000e,0x0000000f,0x00000000,0x0004002b,0x00000006, + 0x00000010,0x00000000,0x0004002b,0x00000006,0x00000011,0x3f800000,0x0007002c,0x00000007, + 0x00000012,0x00000010,0x00000010,0x00000010,0x00000011,0x00040020,0x00000013,0x00000003, + 0x00000007,0x00050036,0x00000002,0x00000004,0x00000000,0x00000003,0x000200f8,0x00000005, + 0x00050041,0x00000013,0x00000014,0x0000000d,0x0000000f,0x0003003e,0x00000014,0x00000012, + 0x000100fd,0x00010038 +}; +static const uint32_t pvr_g5p_stub_comp[] = { + 0x07230203,0x00010300,0x0008000b,0x0000000a,0x00000000,0x00020011,0x00000001,0x0006000b, + 0x00000001,0x4c534c47,0x6474732e,0x3035342e,0x00000000,0x0003000e,0x00000000,0x00000001, + 0x0005000f,0x00000005,0x00000004,0x6e69616d,0x00000000,0x00060010,0x00000004,0x00000011, + 0x00000001,0x00000001,0x00000001,0x00030003,0x00000002,0x000001c2,0x00040005,0x00000004, + 0x6e69616d,0x00000000,0x00040047,0x00000009,0x0000000b,0x00000019,0x00020013,0x00000002, + 0x00030021,0x00000003,0x00000002,0x00040015,0x00000006,0x00000020,0x00000000,0x00040017, + 0x00000007,0x00000006,0x00000003,0x0004002b,0x00000006,0x00000008,0x00000001,0x0006002c, + 0x00000007,0x00000009,0x00000008,0x00000008,0x00000008,0x00050036,0x00000002,0x00000004, + 0x00000000,0x00000003,0x000200f8,0x00000005,0x000100fd,0x00010038 +}; +#endif From 7f5c0a15a032af8b3eac67522555ce511818e421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:38 +0100 Subject: [PATCH 024/109] pvr/g5p: name each refused draw, pass and pipeline; skip them only on request Fail a command buffer with an allowed host-memory error when a draw is unsupported. Applications may opt into skipping unsupported work through pvr_skip_refused_work, at the cost of incorrect pixels. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_cmd_buffer.c | 18 +++++++--- src/imagination/vulkan/pvr_arch_g5p.c | 4 --- src/imagination/vulkan/pvr_arch_g5p_ds.c | 21 +++++------- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 21 ++++-------- src/imagination/vulkan/pvr_arch_g5p_mrt.c | 36 +++++++++----------- src/imagination/vulkan/pvr_arch_g5p_msaa.c | 18 +++++----- src/imagination/vulkan/pvr_arch_g5p_upload.c | 31 ++++++++--------- src/imagination/vulkan/pvr_arch_g5p_vdm.c | 8 ++--- src/imagination/vulkan/pvr_arch_g5p_vertex.c | 4 --- src/imagination/vulkan/pvr_arch_g5p_xfb.c | 25 ++++++-------- src/imagination/vulkan/pvr_drirc_gen.py | 3 +- src/imagination/vulkan/pvr_g5p.h | 17 +++++++++ src/imagination/vulkan/pvr_g5p_draw_hooks.h | 6 ++-- src/imagination/vulkan/pvr_g5p_hooks.h | 2 +- src/imagination/vulkan/pvr_physical_device.c | 3 ++ src/imagination/vulkan/pvr_physical_device.h | 1 + 16 files changed, 108 insertions(+), 110 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_cmd_buffer.c b/src/imagination/vulkan/pvr_arch_cmd_buffer.c index 99a4ae51560..2d9d38cd234 100644 --- a/src/imagination/vulkan/pvr_arch_cmd_buffer.c +++ b/src/imagination/vulkan/pvr_arch_cmd_buffer.c @@ -9679,6 +9679,18 @@ static void pvr_emit_vdm_index_list(struct pvr_cmd_buffer *cmd_buffer, pvr_csb_clear_relocation_mark(csb); } +/* Refusal reasons are logged by G5P_REFUSE before this boundary. */ +static void +g5p_draw_result(struct pvr_cmd_buffer *cmd_buffer, VkResult result) +{ + if (result == VK_SUCCESS) + return; + if (result == VK_ERROR_FEATURE_NOT_PRESENT && + cmd_buffer->device->pdevice->skip_refused_work) + return; + vk_command_buffer_set_error(&cmd_buffer->vk, pvr_refusal_to_api(result)); +} + void PVR_PER_ARCH(CmdDraw)(VkCommandBuffer commandBuffer, uint32_t vertexCount, uint32_t instanceCount, @@ -9714,8 +9726,7 @@ void PVR_PER_ARCH(CmdDraw)(VkCommandBuffer commandBuffer, result = pvr_flush_push_descriptors(cmd_buffer, &cmd_buffer->state.gfx_desc_state); if (result == VK_SUCCESS) result = pvr_arch_g5p_cmd_draw(cmd_buffer, &g5p_draw); - if (result != VK_SUCCESS) - vk_command_buffer_set_error(&cmd_buffer->vk, result); + g5p_draw_result(cmd_buffer, result); goto end_cmd_draw; } if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g9p)) { @@ -9796,8 +9807,7 @@ void PVR_PER_ARCH(CmdDrawIndexed)(VkCommandBuffer commandBuffer, if (result == VK_SUCCESS) result = pvr_arch_g5p_cmd_draw_indexed(cmd_buffer, indexCount, instanceCount, firstIndex, vertexOffset, firstInstance); - if (result != VK_SUCCESS) - vk_command_buffer_set_error(&cmd_buffer->vk, result); + g5p_draw_result(cmd_buffer, result); goto end_cmd_draw_indexed; } if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g9p)) { diff --git a/src/imagination/vulkan/pvr_arch_g5p.c b/src/imagination/vulkan/pvr_arch_g5p.c index 9333d6738b7..2abc6d9b36a 100644 --- a/src/imagination/vulkan/pvr_arch_g5p.c +++ b/src/imagination/vulkan/pvr_arch_g5p.c @@ -533,10 +533,6 @@ g5p_pbe_encode(uint64_t q[3], uint32_t key, bool srgb, uint64_t va, uint32_t wid g5p_pbe_2d(q,key,srgb,va,width,height,row_stride,false); } -/* A refusal, by name, in the log too (vk_errorf() is silent in release builds) */ -#define G5P_REFUSE(cmd_buffer, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((cmd_buffer), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* * The background of a loaded colour attachment (loadOp LOAD): the load diff --git a/src/imagination/vulkan/pvr_arch_g5p_ds.c b/src/imagination/vulkan/pvr_arch_g5p_ds.c index e23a74f48b8..6ef6d750251 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_ds.c +++ b/src/imagination/vulkan/pvr_arch_g5p_ds.c @@ -12,6 +12,7 @@ #include #include "util/log.h" +#include "pvr_g5p.h" #include "vk_format.h" #include "vk_graphics_state.h" #include "vk_log.h" @@ -28,10 +29,6 @@ #include "pvr_image.h" #include "pvr_job_render.h" -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_DS_REFUSE(cmd_buffer, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((cmd_buffer), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) VkResult PVR_PER_ARCH(g5p_ds_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_renderpass_hwsetup_render *hw_render) @@ -46,38 +43,38 @@ PVR_PER_ARCH(g5p_ds_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_r struct pvr_g5p_ds_plane plane; if (!f) - return G5P_DS_REFUSE(cmd_buffer, "depth/stencil format %s (the ZLS stores D16, D32F, S8, D32F_S8)", + return G5P_REFUSE(cmd_buffer, "depth/stencil format %s (the ZLS stores D16, D32F, S8, D32F_S8)", vk_Format_to_str(iview->vk.format)); struct pvr_g5p_ds_chain chain; const bool sampled_chain = pvr_g5p_ds_sampled_chain(image, &chain); if (sampled_chain && f->stencil && (iview->vk.extent.width < 32 || iview->vk.extent.height < 32)) - return G5P_DS_REFUSE(cmd_buffer, "small combined depth/stencil chain attachment needs Apple's temporary ZLS plane"); + return G5P_REFUSE(cmd_buffer, "small combined depth/stencil chain attachment needs Apple's temporary ZLS plane"); if (iview->vk.base_mip_level || iview->vk.base_array_layer) - return G5P_DS_REFUSE(cmd_buffer, "a depth/stencil view of level %u layer %u (level 0 layer 0 are built)", + return G5P_REFUSE(cmd_buffer, "a depth/stencil view of level %u layer %u (level 0 layer 0 are built)", iview->vk.base_mip_level, iview->vk.base_array_layer); /* upstream's own test (pvr_arch_hw_pass.c): a resolve mode; the index is set only with one */ if (hw_render->depth_resolve_mode || hw_render->stencil_resolve_mode) - return G5P_DS_REFUSE(cmd_buffer, "a depth/stencil resolve (no capture)"); + return G5P_REFUSE(cmd_buffer, "a depth/stencil resolve (no capture)"); if (hw_render->eot_surface_count > 1) - return G5P_DS_REFUSE(cmd_buffer, "depth/stencil beside %u colour targets (no capture of the combination)", + return G5P_REFUSE(cmd_buffer, "depth/stencil beside %u colour targets (no capture of the combination)", hw_render->eot_surface_count); if (hw_render->eot_surface_count) { const struct pvr_image_view *c = info->attachments[hw_render->eot_surfaces[0].attachment_idx]; if (c->vk.extent.width != iview->vk.extent.width || c->vk.extent.height != iview->vk.extent.height) - return G5P_DS_REFUSE(cmd_buffer, "a depth/stencil attachment of %ux%u beside a %ux%u colour target " + return G5P_REFUSE(cmd_buffer, "a depth/stencil attachment of %ux%u beside a %ux%u colour target " "(ZLS_PIXELS describe one surface)", iview->vk.extent.width, iview->vk.extent.height, c->vk.extent.width, c->vk.extent.height); } if (!pvr_g5p_ds_layout(iview->vk.format, image->vk.extent.width, image->vk.extent.height, image->vk.samples, &plane) || image->total_size < (sampled_chain ? chain.size : plane.size)) - return G5P_DS_REFUSE(cmd_buffer, "a depth/stencil image of %" PRIu64 " bytes, the ZLS planes take %" PRIu64 + return G5P_REFUSE(cmd_buffer, "a depth/stencil image of %" PRIu64 " bytes, the ZLS planes take %" PRIu64 " (pvr_g5p_ds_layout())", (uint64_t)image->total_size, plane.size); if (image->vk.samples == VK_SAMPLE_COUNT_4_BIT && (hw_render->depth_init == VK_ATTACHMENT_LOAD_OP_LOAD || hw_render->stencil_init == VK_ATTACHMENT_LOAD_OP_LOAD || hw_render->depth_store || hw_render->stencil_store)) { if (!pvr_g5p_capture_stub()) - return G5P_DS_REFUSE(cmd_buffer, "a load or store of a 4-sample depth/stencil attachment " + return G5P_REFUSE(cmd_buffer, "a load or store of a 4-sample depth/stencil attachment " "(no multisample ZLS capture: it stays on chip)"); /* PVR_G5P_CAPTURE_STUB: the pass goes on, its ZLS load/store dropped (pvr_g5p_ds_zls()) */ static bool told; diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 6bd15ccca61..2462abab6b0 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -9,7 +9,7 @@ * static arena are not used, so a draw is only what pvr_* records here plus * the render job of pvr_arch_g5p.c. * - * The first form this drew (docs: orchard-mesa-pr-plan.md, rung A): one + * The first form this drew (the initial triangle path): one * linear colour target, no depth/stencil, one vertex buffer, non-indexed * single-instance draws, scissors and partial render areas as the ICD clips * them (region clip, ISP scissor table); blending as @@ -78,10 +78,6 @@ #define G5P_TILE 32 /* USC programs have byte sizes and need not end on a dword boundary. */ -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_REFUSE(obj, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* ---- PDS programs (pdsc, Apple's DOUT form: the A9 ICD's) ---- */ @@ -392,8 +388,8 @@ g5p_native_bresenham_line(const struct vk_graphics_pipeline_state *s) s->rs->line.mode == VK_LINE_RASTERIZATION_MODE_BRESENHAM; } -/* What rung A draws: refused by name otherwise (the ICD's validate_state() - * with rung A's limits) */ +/* What initial triangle path draws: refused by name otherwise (the ICD's validate_state() + * with initial triangle path's limits) */ static VkResult g5p_check_state(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *create_info, const struct vk_graphics_pipeline_state *s, struct pco_g5p_fs_key *key) @@ -595,7 +591,7 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo key.depth_feedback = false; /* the fragment program first: the vertex program is compiled against the * varyings it reads (Apple's VertexProgramKey; Rogue's pco_rev_link_nir) */ - /* PVR_G5P_SHADER_DUMP / PVR_G5P_CAPTURE_STUB (pvr_g5p_shader_dump.h) */ + /* PVR_DEBUG=shader_dump / PVR_DEBUG=capture_stub (pvr_g5p_shader_dump.h) */ VkPipelineShaderStageCreateInfo stub_info, fs_stub_info; VkShaderModuleCreateInfo stub_module, fs_stub_module; const uint32_t dump_samples = g->samples; @@ -957,12 +953,9 @@ g5p_pipeline_build_punch(struct pvr_device *device, return result; } -/* Every refusal a draw could meet because of its programs is decided here, - * at pipeline creation, never while recording: a refused draw fails the - * whole command buffer (vkEndCommandBuffer), and Zink then drops the batch - * (Aquarium on r39-6: no water/background, 3 fps). The draw selects the - * punch variant from the dynamic depth/stencil state (g5p_draw()); this - * takes every state the pipeline allows: a dynamic enable may be on, a +/* Program variants are checked at creation under the selected refusal + * policy. The draw selects its punch variant from dynamic depth/stencil + * state (g5p_draw()); this takes every state the pipeline allows: a dynamic enable may be on, a * depth write needs the depth test, and a stencil test may write. */ static VkResult g5p_pipeline_check_draw_variants(struct pvr_device *device, diff --git a/src/imagination/vulkan/pvr_arch_g5p_mrt.c b/src/imagination/vulkan/pvr_arch_g5p_mrt.c index 0e726498a59..f51e5e60b73 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_mrt.c +++ b/src/imagination/vulkan/pvr_arch_g5p_mrt.c @@ -25,10 +25,6 @@ #include "pvr_hw_pass.h" #include "pvr_image.h" -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_MRT_REFUSE(obj, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* Two 32-bit integer targets of one format (dEQP's ShaderExecutor: a mat2xN * output's columns, modf's two results): the raw32 output's four words each, @@ -61,19 +57,19 @@ PVR_PER_ARCH(g5p_mrt_pipeline_check)(struct pvr_device *device, unsigned n, cons * whose fourth target stores to the null target (pvr_arch_g5p.c, pvr_g5p_nir.c) */ enum g5p_mrt_refusal r = g5p_mrt_out_plan_make(n == 3 ? 4 : n, &plan); if (r != G5P_MRT_OK) - return G5P_MRT_REFUSE(device, "%u colour attachments: %s", n, g5p_mrt_refusal_str(r)); + return G5P_REFUSE(device, "%u colour attachments: %s", n, g5p_mrt_refusal_str(r)); for (unsigned i = 0; i < n; i++) { const int pair = g5p_mrt_raw32_pair(n, formats[0], formats[i]); if (pair < 0) - return G5P_MRT_REFUSE(device, "colour attachment %u of %u is %s: %s", i, n, vk_Format_to_str(formats[i]), + return G5P_REFUSE(device, "colour attachment %u of %u is %s: %s", i, n, vk_Format_to_str(formats[i]), G5P_MRT_RAW32_PROBE); if (pair) continue; if (n > 2 && pvr_g5p_raw32_kind(formats[i])) - return G5P_MRT_REFUSE(device, "%u colour attachments of %s: %s", n, vk_Format_to_str(formats[i]), + return G5P_REFUSE(device, "%u colour attachments of %s: %s", n, vk_Format_to_str(formats[i]), G5P_MRT_RAW32_WIDE); if (!pvr_g5p_mrt_format_ok(formats[i]) || formats[i] != formats[0]) - return G5P_MRT_REFUSE(device, "colour attachment %u of %u is %s, attachment 0 %s: %s", i, n, + return G5P_REFUSE(device, "colour attachment %u of %u is %s, attachment 0 %s: %s", i, n, vk_Format_to_str(formats[i]), vk_Format_to_str(formats[0]), g5p_mrt_refusal_str(G5P_MRT_MIXED_FORMATS)); } @@ -104,33 +100,33 @@ PVR_PER_ARCH(g5p_mrt_pass_check)(struct pvr_cmd_buffer *cmd_buffer, return VK_SUCCESS; /* three: Apple's 4-target pass with a padding fourth target (pvr_arch_g5p.c) */ if (n > G5P_MRT_MAX_TARGETS) - return G5P_MRT_REFUSE(cmd_buffer, "%u colour targets: %s", n, g5p_mrt_refusal_str(G5P_MRT_TOO_MANY_TARGETS)); + return G5P_REFUSE(cmd_buffer, "%u colour targets: %s", n, g5p_mrt_refusal_str(G5P_MRT_TOO_MANY_TARGETS)); for (unsigned t = 0; t < n; t++) { const uint32_t idx = hw_render->eot_surfaces[t].attachment_idx; if (idx >= info->attachment_count || !info->attachments[idx]) - return G5P_MRT_REFUSE(cmd_buffer, "colour target %u of %u has no image view", t, n); + return G5P_REFUSE(cmd_buffer, "colour target %u of %u has no image view", t, n); iview[t] = info->attachments[idx]; const struct pvr_image *image = pvr_image_view_get_image(iview[t]); const int pair = g5p_mrt_raw32_pair(n, iview[0]->vk.format, iview[t]->vk.format); if (pair < 0) - return G5P_MRT_REFUSE(cmd_buffer, "colour target %u of %u is %s: %s", t, n, + return G5P_REFUSE(cmd_buffer, "colour target %u of %u is %s: %s", t, n, vk_Format_to_str(iview[t]->vk.format), G5P_MRT_RAW32_PROBE); if (!pair && n > 2 && pvr_g5p_raw32_kind(iview[t]->vk.format)) - return G5P_MRT_REFUSE(cmd_buffer, "%u colour targets of %s: %s", n, vk_Format_to_str(iview[t]->vk.format), + return G5P_REFUSE(cmd_buffer, "%u colour targets of %s: %s", n, vk_Format_to_str(iview[t]->vk.format), G5P_MRT_RAW32_WIDE); if (!pair && (!pvr_g5p_mrt_format_ok(iview[t]->vk.format) || iview[t]->vk.format != iview[0]->vk.format)) - return G5P_MRT_REFUSE(cmd_buffer, "colour target %u of %u is %s, target 0 %s: %s", t, n, + return G5P_REFUSE(cmd_buffer, "colour target %u of %u is %s, target 0 %s: %s", t, n, vk_Format_to_str(iview[t]->vk.format), vk_Format_to_str(iview[0]->vk.format), g5p_mrt_refusal_str(G5P_MRT_MIXED_FORMATS)); if (iview[t]->vk.base_mip_level || iview[t]->vk.layer_count != 1 || image->vk.samples != VK_SAMPLE_COUNT_1_BIT) - return G5P_MRT_REFUSE(cmd_buffer, "colour target %u of %u: mip %u, %u layers, %u samples (one level, one layer, " + return G5P_REFUSE(cmd_buffer, "colour target %u of %u: mip %u, %u layers, %u samples (one level, one layer, " "one sample)", t, n, iview[t]->vk.base_mip_level, iview[t]->vk.layer_count, image->vk.samples); if (iview[t]->vk.extent.width != iview[0]->vk.extent.width || iview[t]->vk.extent.height != iview[0]->vk.extent.height) - return G5P_MRT_REFUSE(cmd_buffer, "colour target %u is %ux%u, target 0 %ux%u (one size per pass)", t, + return G5P_REFUSE(cmd_buffer, "colour target %u is %ux%u, target 0 %ux%u (one size per pass)", t, iview[t]->vk.extent.width, iview[t]->vk.extent.height, iview[0]->vk.extent.width, iview[0]->vk.extent.height); va[t] = g5p_mrt_target_va(iview[t], &bytes[t]); @@ -140,7 +136,7 @@ PVR_PER_ARCH(g5p_mrt_pass_check)(struct pvr_cmd_buffer *cmd_buffer, * an N-target LOAD background (none exists) */ for (unsigned i = 0; i < hw_render->color_init_count; i++) if (hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD) - return G5P_MRT_REFUSE(cmd_buffer, "LOAD of a colour attachment with %u colour targets (no N-target LOAD " + return G5P_REFUSE(cmd_buffer, "LOAD of a colour attachment with %u colour targets (no N-target LOAD " "background)", n); /* The clear and end-of-tile programs cover whole tiles of every target: a @@ -148,7 +144,7 @@ PVR_PER_ARCH(g5p_mrt_pass_check)(struct pvr_cmd_buffer *cmd_buffer, const VkRect2D *area = &info->render_area; if (area->offset.x || area->offset.y || area->extent.width != iview[0]->vk.extent.width || area->extent.height != iview[0]->vk.extent.height) - return G5P_MRT_REFUSE(cmd_buffer, "%u colour targets in a partial render area (%d,%d %ux%u of %ux%u)", n, + return G5P_REFUSE(cmd_buffer, "%u colour targets in a partial render area (%d,%d %ux%u of %ux%u)", n, area->offset.x, area->offset.y, area->extent.width, area->extent.height, iview[0]->vk.extent.width, iview[0]->vk.extent.height); @@ -156,7 +152,7 @@ PVR_PER_ARCH(g5p_mrt_pass_check)(struct pvr_cmd_buffer *cmd_buffer, for (unsigned a = 0; a < n; a++) for (unsigned c = a + 1; c < n; c++) if (va[a] < va[c] + bytes[c] && va[c] < va[a] + bytes[a]) - return G5P_MRT_REFUSE(cmd_buffer, "colour targets %u and %u overlap in memory", a, c); + return G5P_REFUSE(cmd_buffer, "colour targets %u and %u overlap in memory", a, c); return VK_SUCCESS; } @@ -168,13 +164,13 @@ PVR_PER_ARCH(g5p_mrt_draw_check)(struct pvr_cmd_buffer *cmd_buffer, unsigned pip const unsigned pn = MAX2(pipeline_targets, 1); if (!info->pass && !info->dr_info) - return G5P_MRT_REFUSE(cmd_buffer, "a draw outside a render pass"); + return G5P_REFUSE(cmd_buffer, "a draw outside a render pass"); /* the render of the current subpass (dynamic rendering has one) */ const struct pvr_renderpass_hwsetup_render *hw_render = pvr_arch_pass_info_get_hw_render(info, info->current_hw_subpass); /* a 3-target pass runs, and its pipelines compile, as four targets */ const unsigned rn = hw_render->eot_surface_count == 3 ? 4 : hw_render->eot_surface_count; if ((rn > 1 || pn > 1) && pn != rn) - return G5P_MRT_REFUSE(cmd_buffer, "the pipeline writes %u colour targets, the pass has %u", pn, rn); + return G5P_REFUSE(cmd_buffer, "the pipeline writes %u colour targets, the pass has %u", pn, rn); return VK_SUCCESS; } diff --git a/src/imagination/vulkan/pvr_arch_g5p_msaa.c b/src/imagination/vulkan/pvr_arch_g5p_msaa.c index 15c46265a62..f47929e50dd 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_msaa.c +++ b/src/imagination/vulkan/pvr_arch_g5p_msaa.c @@ -8,6 +8,7 @@ #include #include "util/log.h" +#include "pvr_g5p.h" #include "vk_format.h" #include "vk_log.h" @@ -16,9 +17,6 @@ #include "pvr_hw_pass.h" #include "pvr_image.h" -#define G5P_MSAA_REFUSE(cmd_buffer, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((cmd_buffer), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* * The captured 4x pass: one colour target of 4-byte samples, cleared or @@ -36,14 +34,14 @@ PVR_PER_ARCH(g5p_msaa_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr if (hw_render->sample_count <= 1) return VK_SUCCESS; if (hw_render->sample_count != 4) - return G5P_MSAA_REFUSE(cmd_buffer, "%u samples (1 and 4 are built)", hw_render->sample_count); + return G5P_REFUSE(cmd_buffer, "%u samples (1 and 4 are built)", hw_render->sample_count); if (hw_render->eot_surface_count != 1) - return G5P_MSAA_REFUSE(cmd_buffer, "4x with %u end-of-tile surfaces (one target, stored or resolved: a " + return G5P_REFUSE(cmd_buffer, "4x with %u end-of-tile surfaces (one target, stored or resolved: a " "STORE with a resolve needs the CPU resolve after the job)", hw_render->eot_surface_count); for (unsigned i = 0; i < hw_render->color_init_count; i++) if (hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD) - return G5P_MSAA_REFUSE(cmd_buffer, "LOAD of a 4-sample attachment (no multisample background loader)"); + return G5P_REFUSE(cmd_buffer, "LOAD of a 4-sample attachment (no multisample background loader)"); const struct pvr_renderpass_hwsetup_eot_surface *eot = &hw_render->eot_surfaces[0]; const struct pvr_image_view *iview = info->attachments[eot->need_resolve ? eot->src_attachment_idx @@ -51,20 +49,20 @@ PVR_PER_ARCH(g5p_msaa_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr const struct pvr_image *image = pvr_image_view_get_image(iview); if (image->vk.samples != VK_SAMPLE_COUNT_4_BIT || vk_format_get_blocksize(iview->vk.format) != 4 || vk_format_is_depth_or_stencil(iview->vk.format)) - return G5P_MSAA_REFUSE(cmd_buffer, "a 4x colour target %s of %u samples (4-byte samples are built)", + return G5P_REFUSE(cmd_buffer, "a 4x colour target %s of %u samples (4-byte samples are built)", vk_Format_to_str(iview->vk.format), image->vk.samples); if (iview->vk.base_mip_level || iview->vk.base_array_layer) - return G5P_MSAA_REFUSE(cmd_buffer, "a 4-sample view of level %u layer %u", iview->vk.base_mip_level, + return G5P_REFUSE(cmd_buffer, "a 4-sample view of level %u layer %u", iview->vk.base_mip_level, iview->vk.base_array_layer); if (eot->need_resolve) { const struct pvr_image_view *r = info->attachments[eot->attachment_idx]; const struct pvr_image *ri = pvr_image_view_get_image(r); if (eot->resolve_type != PVR_RESOLVE_TYPE_PBE) - return G5P_MSAA_REFUSE(cmd_buffer, "a 4x resolve outside the PBE (transfer resolves are lane g's)"); + return G5P_REFUSE(cmd_buffer, "a 4x resolve outside the PBE (transfer resolves are lane g's)"); if (ri->vk.samples != VK_SAMPLE_COUNT_1_BIT || ri->memlayout != PVR_MEMLAYOUT_LINEAR || r->vk.format != iview->vk.format || r->vk.extent.width != iview->vk.extent.width || r->vk.extent.height != iview->vk.extent.height || r->vk.base_mip_level) - return G5P_MSAA_REFUSE(cmd_buffer, "a resolve view that is not a %ux%u one-sample linear %s view", + return G5P_REFUSE(cmd_buffer, "a resolve view that is not a %ux%u one-sample linear %s view", iview->vk.extent.width, iview->vk.extent.height, vk_Format_to_str(iview->vk.format)); } return VK_SUCCESS; diff --git a/src/imagination/vulkan/pvr_arch_g5p_upload.c b/src/imagination/vulkan/pvr_arch_g5p_upload.c index 1a0d8c19f18..6b27795936d 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_upload.c +++ b/src/imagination/vulkan/pvr_arch_g5p_upload.c @@ -13,6 +13,7 @@ #include #include "util/log.h" +#include "pvr_g5p.h" #include "vk_alloc.h" #include "vk_enum_to_str.h" #include "vk_log.h" @@ -34,10 +35,6 @@ #include "pvr_g5p_upload.h" #include "pvr_g5p_vtex.h" -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_UP_REFUSE(obj, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* The loader: DOUTD c(2i):c(2i+1) -> common store with the control word in * c(2n+2i), the last one ends, then HALT (the fragment loader's shape) */ @@ -100,7 +97,7 @@ PVR_PER_ARCH(g5p_vs_uploads_create)(struct pvr_device *device, const struct pco_ return pvr_arch_g5p_vtex_create(device, vs, out); } if (vs->num_uploads > PVR_G5P_MAX_STAGE_UPLOADS) - return G5P_UP_REFUSE(device, "%u vertex-stage uploads (at most %u)", vs->num_uploads, + return G5P_REFUSE(device, "%u vertex-stage uploads (at most %u)", vs->num_uploads, PVR_G5P_MAX_STAGE_UPLOADS); for (unsigned i = 0; i < vs->num_uploads; i++) { @@ -115,11 +112,11 @@ PVR_PER_ARCH(g5p_vs_uploads_create)(struct pvr_device *device, const struct pco_ default: /* textures and samplers in a vertex program: lane b's descriptor * words exist, the vertex stage does not take them yet */ - return G5P_UP_REFUSE(device, "vertex-stage upload %u: %s", i, g5p_up_kind_str(u->kind)); + return G5P_REFUSE(device, "vertex-stage upload %u: %s", i, g5p_up_kind_str(u->kind)); } if (!u->dwords || u->dwords > 0xfff || u->set >= PVR_MAX_DESCRIPTOR_SETS || u->binding > UINT8_MAX || u->element > UINT16_MAX || u->offset > UINT16_MAX || u->common > UINT16_MAX) - return G5P_UP_REFUSE(device, + return G5P_REFUSE(device, "vertex-stage upload %u: set %u binding %u[%u] offset %u, %u dwords at common %u", i, u->set, u->binding, u->element, u->offset, u->dwords, u->common); } @@ -168,10 +165,10 @@ g5p_up_buffer(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_stage_uplo /* a set without a layout: a pushed set not instantiated (never dereferenced) */ if (!set || !set->layout || u->binding >= set->layout->binding_count) - return G5P_UP_REFUSE(cmd_buffer, "upload: set %u binding %u is not bound", u->set, u->binding); + return G5P_REFUSE(cmd_buffer, "upload: set %u binding %u is not bound", u->set, u->binding); const struct pvr_descriptor_set_layout_binding *lb = &set->layout->bindings[u->binding]; if (u->element >= lb->descriptor_count) - return G5P_UP_REFUSE(cmd_buffer, "upload: set %u binding %u element %u past %u descriptors", u->set, + return G5P_REFUSE(cmd_buffer, "upload: set %u binding %u element %u past %u descriptors", u->set, u->binding, u->element, lb->descriptor_count); struct pvr_buffer_descriptor d; switch (lb->type) { @@ -185,11 +182,11 @@ g5p_up_buffer(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_stage_uplo d = set->dynamic_buffers[lb->dynamic_buffer_idx + u->element]; break; default: - return G5P_UP_REFUSE(cmd_buffer, "upload: set %u binding %u is a %s, not a buffer", u->set, + return G5P_REFUSE(cmd_buffer, "upload: set %u binding %u is a %s, not a buffer", u->set, u->binding, vk_DescriptorType_to_str(lb->type)); } if (!d.addr) - return G5P_UP_REFUSE(cmd_buffer, "upload: set %u binding %u holds no buffer", u->set, u->binding); + return G5P_REFUSE(cmd_buffer, "upload: set %u binding %u holds no buffer", u->set, u->binding); /* a dynamic buffer's range starts at its dynamic offset (VkDescriptorBufferInfo.range from there) */ *addr = d.addr + d.offset; *size = d.size; @@ -203,7 +200,7 @@ PVR_PER_ARCH(g5p_upload_buffer)(struct pvr_cmd_buffer *cmd_buffer, unsigned set, { const struct pvr_g5p_stage_upload u = { .set = set, .binding = binding, .element = element }; if (set >= PVR_MAX_DESCRIPTOR_SETS) - return G5P_UP_REFUSE(cmd_buffer, "upload: set %u", set); + return G5P_REFUSE(cmd_buffer, "upload: set %u", set); return g5p_up_buffer(cmd_buffer, &u, addr, size); } @@ -229,7 +226,7 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc case PCO_G5P_UPLOAD_PUSH: /* the bytes the program reads, as they are now (a copy per draw) */ if (u->offset + bytes > PVR_MAX_PUSH_CONSTANTS_SIZE) - return G5P_UP_REFUSE(cmd_buffer, "vertex push constants [%u, %u) past %u bytes", u->offset, + return G5P_REFUSE(cmd_buffer, "vertex push constants [%u, %u) past %u bytes", u->offset, u->offset + bytes, PVR_MAX_PUSH_CONSTANTS_SIZE); result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, push->data + u->offset, bytes, &bo); if (result != VK_SUCCESS) @@ -242,7 +239,7 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc if (result != VK_SUCCESS) return result; if (u->offset + bytes > size) - return G5P_UP_REFUSE(cmd_buffer, "vertex uniform buffer at set %u binding %u: [%u, %u) past its %" PRIu64 + return G5P_REFUSE(cmd_buffer, "vertex uniform buffer at set %u binding %u: [%u, %u) past its %" PRIu64 " bytes", u->set, u->binding, u->offset, u->offset + bytes, size); src[i] = addr + u->offset; break; @@ -253,7 +250,7 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc if (result != VK_SUCCESS) return result; if (u->dwords != 2 || (u->kind == PCO_G5P_UPLOAD_UBO_ADDR && u->offset >= size)) - return G5P_UP_REFUSE(cmd_buffer, "vertex %s: %u dwords at offset %u of %" PRIu64 " bytes", + return G5P_REFUSE(cmd_buffer, "vertex %s: %u dwords at offset %u of %" PRIu64 " bytes", g5p_up_kind_str(u->kind), u->dwords, u->offset, size); const uint64_t va = addr + (u->kind == PCO_G5P_UPLOAD_UBO_ADDR ? u->offset : 0); result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, &va, sizeof(va), &bo); @@ -268,7 +265,7 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc uint32_t zero[PCO_G5P_XFB_DWORDS]; memset(zero, 0, sizeof(zero)); if (u->dwords != PCO_G5P_XFB_DWORDS || out->xfb_block) - return G5P_UP_REFUSE(cmd_buffer, "a transform feedback block of %u dwords", u->dwords); + return G5P_REFUSE(cmd_buffer, "a transform feedback block of %u dwords", u->dwords); result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, zero, sizeof(zero), &bo); if (result != VK_SUCCESS) return result; @@ -278,7 +275,7 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc break; } default: - return G5P_UP_REFUSE(cmd_buffer, "vertex upload %u: %s", i, g5p_up_kind_str(u->kind)); + return G5P_REFUSE(cmd_buffer, "vertex upload %u: %s", i, g5p_up_kind_str(u->kind)); } } diff --git a/src/imagination/vulkan/pvr_arch_g5p_vdm.c b/src/imagination/vulkan/pvr_arch_g5p_vdm.c index a2e1b0f6a22..3127e155308 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vdm.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vdm.c @@ -34,6 +34,7 @@ #include #include "util/log.h" +#include "pvr_g5p.h" #include "vk_log.h" #include "pvr_bo.h" @@ -43,9 +44,6 @@ #include "winsys/pvr_winsys.h" #include "pvr_g5p_vdm.h" -#define G5P_VDM_REFUSE(cmd_buffer, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((cmd_buffer), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* The bytes of a block before its chaining link to `next`, or -1 when the * block holds no such link */ @@ -91,13 +89,13 @@ PVR_PER_ARCH(g5p_vdm_stream)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr const struct pvr_bo *n = list_entry(b->link.next, struct pvr_bo, link); const int64_t len = block_commands(b, n->vma->dev_addr); if (len < 0) - return G5P_VDM_REFUSE(cmd_buffer, "control stream block %u of %u has no link to block %u", i, blocks, + return G5P_REFUSE(cmd_buffer, "control stream block %u of %u has no link to block %u", i, blocks, i + 1); total += (uint64_t)len; } const uint32_t limit = PVR_G5P_MAX_VDM_BYTES; if (total > limit) - return G5P_VDM_REFUSE(cmd_buffer, "a control stream of %" PRIu64 " bytes in %u blocks (at most %u)", + return G5P_REFUSE(cmd_buffer, "a control stream of %" PRIu64 " bytes in %u blocks (at most %u)", total, blocks, limit); const size_t size = total + ROGUE_VDMCTRL_GUARD_SIZE_DEFAULT; diff --git a/src/imagination/vulkan/pvr_arch_g5p_vertex.c b/src/imagination/vulkan/pvr_arch_g5p_vertex.c index e3f0651a393..4d5eef84018 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vertex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vertex.c @@ -26,10 +26,6 @@ #include "pvr_pipeline.h" #include "pvr_g5p_vertex.h" -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_REFUSE(obj, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) VkResult PVR_PER_ARCH(g5p_vertex_draw_check)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline *g, diff --git a/src/imagination/vulkan/pvr_arch_g5p_xfb.c b/src/imagination/vulkan/pvr_arch_g5p_xfb.c index 566a5857c19..955cb77bb33 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_xfb.c +++ b/src/imagination/vulkan/pvr_arch_g5p_xfb.c @@ -15,6 +15,7 @@ #include #include "util/log.h" +#include "pvr_g5p.h" #include "util/os_time.h" #include "util/u_dynarray.h" #include "vk_command_buffer.h" @@ -31,10 +32,6 @@ #include "pvr_query.h" #include "pvr_queue.h" -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_XFB_REFUSE(obj, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* A buffer range named by a command (a binding or a counter) */ struct g5p_xfb_range { @@ -93,7 +90,7 @@ g5p_xfb_rec_new(struct pvr_cmd_buffer *cmd_buffer, struct pvr_g5p_xfb_cmd *st, e const struct pvr_sub_cmd *sub = cmd_buffer->state.current_sub_cmd; if ((!sub || sub->type != PVR_SUB_CMD_TYPE_GRAPHICS) && kind <= G5P_XFB_DRAW) { vk_command_buffer_set_error(&cmd_buffer->vk, - G5P_XFB_REFUSE(cmd_buffer, "transform feedback outside a graphics sub-command")); + pvr_refusal_to_api(G5P_REFUSE(cmd_buffer, "transform feedback outside a graphics sub-command"))); return NULL; } struct g5p_xfb_rec *r = util_dynarray_grow(&st->recs, struct g5p_xfb_rec, 1); @@ -143,8 +140,8 @@ PVR_PER_ARCH(g5p_xfb_begin)(struct pvr_cmd_buffer *cmd_buffer, uint32_t first, u return; if (cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT) { vk_command_buffer_set_error(&cmd_buffer->vk, - G5P_XFB_REFUSE(cmd_buffer, "transform feedback in a SIMULTANEOUS_USE command " - "buffer (its per-draw blocks are filled per submission)")); + pvr_refusal_to_api(G5P_REFUSE(cmd_buffer, "transform feedback in a SIMULTANEOUS_USE command " + "buffer (its per-draw blocks are filled per submission)"))); return; } struct g5p_xfb_rec *r = g5p_xfb_rec_new(cmd_buffer, st, G5P_XFB_BEGIN); @@ -192,7 +189,7 @@ PVR_PER_ARCH(g5p_xfb_draw_check)(struct pvr_cmd_buffer *cmd_buffer, VkPrimitiveT /* the stores follow the vertex invocations: they are the primitive * stream only for lists (GLES 3.0 captures lists only) */ if (!g5p_xfb_vpp(topology)) - return G5P_XFB_REFUSE(cmd_buffer, "transform feedback with topology %d (point, line and triangle lists)", + return G5P_REFUSE(cmd_buffer, "transform feedback with topology %d (point, line and triangle lists)", topology); return VK_SUCCESS; } @@ -205,7 +202,7 @@ PVR_PER_ARCH(g5p_xfb_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct g5p_x if (!st) return VK_ERROR_OUT_OF_HOST_MEMORY; if (!block) - return G5P_XFB_REFUSE(cmd_buffer, "a capturing vertex program without its transform feedback block"); + return G5P_REFUSE(cmd_buffer, "a capturing vertex program without its transform feedback block"); struct g5p_xfb_rec *r = g5p_xfb_rec_new(cmd_buffer, st, G5P_XFB_DRAW); if (!r) return VK_ERROR_OUT_OF_HOST_MEMORY; @@ -232,8 +229,8 @@ g5p_xfb_query_event(struct pvr_cmd_buffer *cmd_buffer, enum g5p_xfb_kind kind, c return; if (pool->result_elem_stride < 2 * sizeof(uint64_t)) { vk_command_buffer_set_error(&cmd_buffer->vk, - G5P_XFB_REFUSE(cmd_buffer, "a stream query pool of %u bytes per query", - pool->result_elem_stride)); + pvr_refusal_to_api(G5P_REFUSE(cmd_buffer, "a stream query pool of %u bytes per query", + pool->result_elem_stride))); return; } struct g5p_xfb_rec *r = g5p_xfb_rec_new(cmd_buffer, st, kind); @@ -250,7 +247,7 @@ PVR_PER_ARCH(g5p_xfb_query_begin)(struct pvr_cmd_buffer *cmd_buffer, const struc { if (index) { vk_command_buffer_set_error(&cmd_buffer->vk, - G5P_XFB_REFUSE(cmd_buffer, "a stream query of stream %u (stream 0 only)", index)); + pvr_refusal_to_api(G5P_REFUSE(cmd_buffer, "a stream query of stream %u (stream 0 only)", index))); return; } g5p_xfb_query_event(cmd_buffer, G5P_XFB_QUERY_BEGIN, pool, query, 1); @@ -355,10 +352,10 @@ g5p_xfb_run(struct pvr_device *device, struct pvr_g5p_xfb_cmd *st, unsigned end) r->e.query.pool = &r->qview; } if (!ok) - return G5P_XFB_REFUSE(device, "a transform feedback buffer without a CPU mapping"); + return G5P_REFUSE(device, "a transform feedback buffer without a CPU mapping"); const char *why = g5p_xfb_execute(x, &r->e); if (why) - return G5P_XFB_REFUSE(device, "transform feedback: %s", why); + return G5P_REFUSE(device, "transform feedback: %s", why); } return VK_SUCCESS; } diff --git a/src/imagination/vulkan/pvr_drirc_gen.py b/src/imagination/vulkan/pvr_drirc_gen.py index aef105ca8e3..34b7e4f4167 100644 --- a/src/imagination/vulkan/pvr_drirc_gen.py +++ b/src/imagination/vulkan/pvr_drirc_gen.py @@ -17,7 +17,8 @@ def declare_options(): debug_options = [] features_options = [] performance_options = [] - misc_options = [] + misc_options = [drirc_gen.DrircBool("pvr_skip_refused_work", False, + "Skip unsupported work, allowing incorrect rendering", c_name="pvr_skip_refused_work")] drirc_gen.add_common_vk_options(debug_options, features_options, misc_options, valid_options=VALID_COMMON_VK_OPTIONS) diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index 67403ad76e5..74abe76faf1 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -27,6 +27,23 @@ static inline VkResult pvr_refusal_to_api(VkResult result) return result == VK_ERROR_FEATURE_NOT_PRESENT ? VK_ERROR_OUT_OF_HOST_MEMORY : result; } +#include "util/log.h" +#include "util/u_atomic.h" +#include "vk_log.h" + +/* Refusal kinds are keyed by their call site (pipeline validation or pass/draw + * validation). Count atomically across recording threads; warn on occurrences + * 1, 2, 4, ... so tolerant workloads cannot emit one warning per draw. Keep + * vk_errorf for debug callbacks and preserve the internal refusal result. */ +#define G5P_REFUSE(obj, ...) \ + ({ \ + static uint32_t refusal_count; \ + uint32_t occurrence = p_atomic_inc_return(&refusal_count); \ + if (occurrence && !(occurrence & (occurrence - 1))) \ + mesa_logw("G5P refuses: " __VA_ARGS__); \ + vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__); \ + }) + #include "pvr_common.h" #include "pvr_limits.h" #include "pvr_macros.h" diff --git a/src/imagination/vulkan/pvr_g5p_draw_hooks.h b/src/imagination/vulkan/pvr_g5p_draw_hooks.h index 8e2ee175711..2f0c62a2336 100644 --- a/src/imagination/vulkan/pvr_g5p_draw_hooks.h +++ b/src/imagination/vulkan/pvr_g5p_draw_hooks.h @@ -22,8 +22,7 @@ result = pvr_flush_push_descriptors(cmd_buffer, &state->gfx_desc_state); \ if (result == VK_SUCCESS) \ result = pvr_arch_g5p_cmd_draw_indirect(cmd_buffer, buffer, offset, drawCount, stride, true); \ - if (result != VK_SUCCESS) \ - vk_command_buffer_set_error(&cmd_buffer->vk, result); \ + g5p_draw_result(cmd_buffer, result); /* a refusal skips the draw */ \ goto end_cmd_draw_indexed_indirect; \ } #define PVR_G5P_HOOK_g5p_indirect() \ @@ -31,8 +30,7 @@ result = pvr_flush_push_descriptors(cmd_buffer, &state->gfx_desc_state); \ if (result == VK_SUCCESS) \ result = pvr_arch_g5p_cmd_draw_indirect(cmd_buffer, buffer, offset, drawCount, stride, false); \ - if (result != VK_SUCCESS) \ - vk_command_buffer_set_error(&cmd_buffer->vk, result); \ + g5p_draw_result(cmd_buffer, result); /* a refusal skips the draw */ \ goto end_cmd_draw_indirect; \ } diff --git a/src/imagination/vulkan/pvr_g5p_hooks.h b/src/imagination/vulkan/pvr_g5p_hooks.h index b151eab9b1a..27b38d91392 100644 --- a/src/imagination/vulkan/pvr_g5p_hooks.h +++ b/src/imagination/vulkan/pvr_g5p_hooks.h @@ -294,7 +294,7 @@ if (PVR_HAS_FEATURE(&(cmd_buffer)->device->pdevice->dev_info, apple_g5p)) { \ VkResult r_ = pvr_arch_g5p_clear_attachments(cmd_buffer, count, attachments, rect_count, rects); \ if (r_ != VK_SUCCESS) \ - pvr_cmd_buffer_set_error_unwarned(cmd_buffer, r_); \ + pvr_cmd_buffer_set_error_unwarned(cmd_buffer, pvr_refusal_to_api(r_)); \ return; \ } diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index 3ba24affc6a..1ec6f23e03f 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -1360,6 +1360,9 @@ VkResult pvr_physical_device_init(struct pvr_physical_device *pdevice, goto err_vk_physical_device_finish; } + pdevice->skip_refused_work = PVR_HAS_FEATURE(&pdevice->dev_info, apple_g5p) && + instance->drirc.misc.pvr_skip_refused_work; + pdevice->sync_type = ws->syncobj_type; if (ws->job_sync_size) { pdevice->sync_type.size = MAX2(pdevice->sync_type.size, ws->job_sync_size); diff --git a/src/imagination/vulkan/pvr_physical_device.h b/src/imagination/vulkan/pvr_physical_device.h index 05e532f81f3..d17cd312128 100644 --- a/src/imagination/vulkan/pvr_physical_device.h +++ b/src/imagination/vulkan/pvr_physical_device.h @@ -56,6 +56,7 @@ struct pvr_physical_device { char *render_path; char *display_path; bool has_primary; + bool skip_refused_work; /* primary node (cardN) of the render device * Only valid when has_primary is true. From 3b559c10cd9229f3d8f5bae96047a989b61ba3d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:39 +0100 Subject: [PATCH 025/109] pvr/g5p: driconf pvr_skip_refused_work for applications that cannot survive a refusal Pipeline creation and render-pass recording fail by default with an allowed host-memory error. Per-application driconf can skip refused work instead, with a warning for each skipped pipeline and incorrect pixels. Internal pipelines are always required to compile. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_cmd_buffer.c | 7 ++- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 55 +++++++++++++++----- src/imagination/vulkan/pvr_arch_g5p_meta.c | 3 ++ src/imagination/vulkan/pvr_arch_queue.c | 10 ++++ src/imagination/vulkan/pvr_cmd_buffer.h | 3 ++ src/imagination/vulkan/pvr_g5p.h | 1 + src/imagination/vulkan/pvr_g5p_shader_dump.c | 2 + src/imagination/vulkan/pvr_g5p_shader_dump.h | 4 ++ 8 files changed, 72 insertions(+), 13 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_cmd_buffer.c b/src/imagination/vulkan/pvr_arch_cmd_buffer.c index 2d9d38cd234..7efc5ccfff3 100644 --- a/src/imagination/vulkan/pvr_arch_cmd_buffer.c +++ b/src/imagination/vulkan/pvr_arch_cmd_buffer.c @@ -2696,8 +2696,13 @@ VkResult pvr_arch_cmd_buffer_end_sub_cmd(struct pvr_cmd_buffer *cmd_buffer) result = pvr_sub_cmd_gfx_job_init(&device->pdevice->dev_info, cmd_buffer, gfx_sub_cmd); + if (result == VK_ERROR_FEATURE_NOT_PRESENT && + device->pdevice->skip_refused_work) { + gfx_sub_cmd->g5p_refused = true; + break; + } if (result != VK_SUCCESS) - return pvr_cmd_buffer_set_error_unwarned(cmd_buffer, result); + return pvr_cmd_buffer_set_error_unwarned(cmd_buffer, pvr_refusal_to_api(result)); } if (pvr_sub_cmd_gfx_requires_split_submit(gfx_sub_cmd)) { diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 2462abab6b0..112d286f12f 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -45,6 +45,7 @@ #include "pvr_cmd_buffer.h" #include "pvr_csb.h" #include "pvr_device.h" +#include "pvr_physical_device.h" #include "pvr_framebuffer.h" #include "pvr_g5p.h" #include "pvr_g5p_vdm.h" @@ -929,8 +930,10 @@ g5p_pipeline_build_punch(struct pvr_device *device, * in Aquarium with it, cause not yet bisected. Without the variant, * g5p_pipeline_check_draw_variants() refuses the pipeline at creation * when a depth/stencil state could need it, the non-fatal form r39-3 had. */ - if (g->fs_blend_mode == 4 /* general */) + if (g->fs_blend_mode == 4 /* general */) { + g->punch_why = "general blend epilogue"; return VK_SUCCESS; + } struct pvr_g5p_pipeline *p = vk_zalloc(&device->vk.alloc, sizeof(*p), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!p) @@ -947,8 +950,10 @@ g5p_pipeline_build_punch(struct pvr_device *device, g5p_pipeline_free(device, p); /* A normal colour-only shader remains valid. A draw that requires the * absent punch variant refuses by name before emitting any draw state. */ - if (result == VK_ERROR_FEATURE_NOT_PRESENT) + if (result == VK_ERROR_FEATURE_NOT_PRESENT) { + g->punch_why = "the variant's compile was refused (see the refusal above)"; return VK_SUCCESS; + } } return result; } @@ -965,8 +970,13 @@ g5p_pipeline_check_draw_variants(struct pvr_device *device, if (state->rs && !BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_RS_RASTERIZER_DISCARD_ENABLE) && state->rs->rasterizer_discard_enable) return VK_SUCCESS; - const bool depth = !state->rp || state->rp->depth_attachment_format != VK_FORMAT_UNDEFINED; - const bool stencil = !state->rp || state->rp->stencil_attachment_format != VK_FORMAT_UNDEFINED; + /* VkRenderPass supplies attachment bits without formats; dynamic rendering + * supplies both. Include both representations when deciding which + * depth/stencil variants this pipeline can select. */ + const bool depth = !state->rp || (state->rp->attachments & MESA_VK_RP_ATTACHMENT_DEPTH_BIT) || + state->rp->depth_attachment_format != VK_FORMAT_UNDEFINED; + const bool stencil = !state->rp || (state->rp->attachments & MESA_VK_RP_ATTACHMENT_STENCIL_BIT) || + state->rp->stencil_attachment_format != VK_FORMAT_UNDEFINED; const struct vk_depth_stencil_state *ds = state->ds; #define G5P_DYN_DS(x) BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_DS_##x) const bool depth_test = depth && (G5P_DYN_DS(DEPTH_TEST_ENABLE) || (ds && ds->depth.test_enable)); @@ -975,10 +985,21 @@ g5p_pipeline_check_draw_variants(struct pvr_device *device, #undef G5P_DYN_DS const bool testing = depth_test || stencil_test, writing = depth_write || stencil_test; const bool needs_punch = (testing && g->fs_discard_tokens > 1) || (writing && g->fs_discard_select); - if (needs_punch && !g->punch) - return G5P_REFUSE(device, "pipeline: discard under depth/stencil requires an unsupported punch-through variant"); - if (testing && (g->fs_discard || needs_punch) && g->samples != 1) - return G5P_REFUSE(device, "pipeline: discard under multisample depth/stencil is not hardware-qualified"); + /* The draws decide (g5p_cmd_draw()): a draw whose depth/stencil state needs the missing form is + * refused and skipped there, the command buffer valid, while the pipeline's other draws run. + * Refusing the pipeline instead loses a WebGPU device (wgpu-hal: any creation error but OOM). + * the default policy refuses at creation. */ + const char *why = needs_punch && !g->punch ? "discard under depth/stencil requires an unsupported punch-through variant" + : testing && (g->fs_discard || needs_punch) && g->samples != 1 + ? "discard under multisample depth/stencil is not hardware-qualified" : NULL; + if (!why) + return VK_SUCCESS; + if (!device->pdevice->skip_refused_work) + return G5P_REFUSE(device, "pipeline: %s", why); + mesa_logw("G5P: pipeline admitted; its draws under depth/stencil will be refused: %s (discard tokens %u, " + "select %u, blend mode %u, write mask %#x, samples %u%s%s)", why, g->fs_discard_tokens, + g->fs_discard_select, g->fs_blend_mode, g->fs_write_mask, g->samples, + g->punch_why ? "; no punch variant: " : "", g->punch_why ? g->punch_why : ""); return VK_SUCCESS; } @@ -1000,14 +1021,22 @@ PVR_PER_ARCH(g5p_graphics_pipeline_compile)(struct pvr_device *device, free(fs.code); if (result != VK_SUCCESS) { g5p_pipeline_free(device, g); - return result; + /* Unsupported programs fail creation unless the application explicitly + * opts into incorrect pixels. Internal pipelines always fail. */ + if (result == VK_ERROR_FEATURE_NOT_PRESENT && !pvr_g5p_internal_pipeline && + device->pdevice->skip_refused_work) { + mesa_logw("pvr/g5p: pipeline admitted without programs after a compile refusal"); + pipeline->g5p = NULL; + return VK_SUCCESS; + } + return pvr_refusal_to_api(result); } result = g5p_pipeline_build_punch(device, create_info, vk_state, g); if (result == VK_SUCCESS) result = g5p_pipeline_check_draw_variants(device, vk_state, g); if (result != VK_SUCCESS) { g5p_pipeline_free(device, g); - return result; + return pvr_refusal_to_api(result); } pipeline->g5p = g; @@ -1038,7 +1067,7 @@ PVR_PER_ARCH(g5p_graphics_pipeline_compile)(struct pvr_device *device, if (result != VK_SUCCESS) { g5p_pipeline_free(device, g); pipeline->g5p = NULL; - return result; + return pvr_refusal_to_api(result); } } } @@ -1125,7 +1154,9 @@ g5p_clear_pipeline(struct pvr_cmd_buffer *cmd_buffer, VkFormat format, uint8_t w .pStages = stages, .pVertexInputState = &vi, .pInputAssemblyState = &ia, .pViewportState = &vp, .pRasterizationState = &rs, .pMultisampleState = &ms, .pColorBlendState = &cb, .pDynamicState = &dy, .layout = gd->clear_layout }; + pvr_g5p_internal_pipeline++; result = device->vk.dispatch_table.CreateGraphicsPipelines(vk_dev, VK_NULL_HANDLE, 1, &gci, NULL, &pipe); + pvr_g5p_internal_pipeline--; pvr_g5p_clear_nir_free(vs); pvr_g5p_clear_nir_free(fs); if (result != VK_SUCCESS) { @@ -1609,7 +1640,7 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g bool lines; if (!g) - return G5P_REFUSE(cmd_buffer, "a draw without a G5P graphics pipeline"); + return G5P_REFUSE(cmd_buffer, "a draw of a pipeline without G5P programs (refused at creation, named there)"); /* transform feedback: while capturing, the pipeline's capturing variant * (pvr_g5p_xfb.h); a pipeline that captures nothing draws as it is */ const bool xfb = g->xfb && pvr_arch_g5p_xfb_capturing(cmd_buffer); diff --git a/src/imagination/vulkan/pvr_arch_g5p_meta.c b/src/imagination/vulkan/pvr_arch_g5p_meta.c index 441b12e90b6..ea09c845e3a 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_meta.c +++ b/src/imagination/vulkan/pvr_arch_g5p_meta.c @@ -24,6 +24,7 @@ #include "pvr_buffer.h" #include "pvr_cmd_buffer.h" +#include "pvr_g5p_shader_dump.h" #include "pvr_descriptor_set.h" #include "pvr_device.h" #include "pvr_g5p.h" @@ -183,7 +184,9 @@ PVR_PER_ARCH(g5p_meta_blit)(struct pvr_cmd_buffer *cmd_buffer, const VkBlitImage struct list_head *mark = cmd_buffer->sub_cmds.prev; g5p_meta_begin(cmd_buffer, save); + pvr_g5p_internal_pipeline++; vk_meta_blit_image2(&cmd_buffer->vk, &cmd_buffer->device->g5p->meta, info); + pvr_g5p_internal_pipeline--; g5p_meta_end(cmd_buffer, save); free(save); diff --git a/src/imagination/vulkan/pvr_arch_queue.c b/src/imagination/vulkan/pvr_arch_queue.c index 2f7765b01a5..4b230c0b9a6 100644 --- a/src/imagination/vulkan/pvr_arch_queue.c +++ b/src/imagination/vulkan/pvr_arch_queue.c @@ -1199,6 +1199,16 @@ static VkResult pvr_process_cmd_buffer(struct pvr_device *device, const int64_t g9p_t0 = getenv("G9P_TIME") ? os_time_get_nano() : 0; switch (sub_cmd->type) { case PVR_SUB_CMD_TYPE_GRAPHICS: { + /* a refused render pass (pvr_arch_cmd_buffer.c): nothing to submit */ + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p) && + sub_cmd->gfx.g5p_refused) { + if (!sub_cmd->is_resume) { + suspended_data->rts = NULL; + suspended_data->query_pool = NULL; + } + result = VK_SUCCESS; + break; + } /* If the fragment job utilizes queries, for data integrity * it needs to wait for the query to be processed. */ diff --git a/src/imagination/vulkan/pvr_cmd_buffer.h b/src/imagination/vulkan/pvr_cmd_buffer.h index ac93575aa4b..682a1e64d03 100644 --- a/src/imagination/vulkan/pvr_cmd_buffer.h +++ b/src/imagination/vulkan/pvr_cmd_buffer.h @@ -153,6 +153,9 @@ struct pvr_dynamic_render_info { struct pvr_sub_cmd_gfx { /* a vk_meta blit's render pass: transfer work for barriers (pvr_arch_g5p_meta.c) */ bool g5p_transfer; + /* a render pass G5P refuses (pvr_arch_g5p_job_init(): a shape it cannot run yet, named on stderr): + * not submitted, its draws lost, the command buffer valid (pvr_arch_queue.c) */ + bool g5p_refused; struct pvr_dynamic_render_info *dr_info; struct pvr_renderpass_hwsetup_render *hw_render; uint32_t attachment_count; diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index 74abe76faf1..e0a7131023b 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -259,6 +259,7 @@ struct pvr_g5p_pipeline { * Original colour-only programs remain byte-identical. */ struct pvr_g5p_pipeline *punch; bool punch_variant; + const char *punch_why; /* no punch variant: why (static string; g5p_pipeline_build_punch()) */ /* the program writes memory (pco_g5p_program::side_effects): a fragment * program's draws bound the shaded pixels with the ISP scissor; a vertex * program's draws run even when no pixel is left (pvr_arch_g5p_gfx.c) */ diff --git a/src/imagination/vulkan/pvr_g5p_shader_dump.c b/src/imagination/vulkan/pvr_g5p_shader_dump.c index a12a21401c1..2432ce40fcb 100644 --- a/src/imagination/vulkan/pvr_g5p_shader_dump.c +++ b/src/imagination/vulkan/pvr_g5p_shader_dump.c @@ -292,3 +292,5 @@ pvr_g5p_stub_stage(VkShaderStageFlagBits stage, VkPipelineShaderStageCreateInfo .pNext = module, .stage = stage, .pName = "main" }; return info; } + +thread_local unsigned pvr_g5p_internal_pipeline; diff --git a/src/imagination/vulkan/pvr_g5p_shader_dump.h b/src/imagination/vulkan/pvr_g5p_shader_dump.h index cc62afcb9f8..609139a0e3d 100644 --- a/src/imagination/vulkan/pvr_g5p_shader_dump.h +++ b/src/imagination/vulkan/pvr_g5p_shader_dump.h @@ -54,4 +54,8 @@ const VkPipelineShaderStageCreateInfo *pvr_g5p_stub_stage(VkShaderStageFlagBits VkPipelineShaderStageCreateInfo *info, VkShaderModuleCreateInfo *module); +/* Pipelines the driver makes for itself (clear quads, vk_meta blits) are created with this + * raised: a refused one fails as before instead of being admitted without programs */ +extern thread_local unsigned pvr_g5p_internal_pipeline; + #endif From 7adf3e6f08d98e7b61498bcd3e3ebe05118adb08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 03:40:59 +0100 Subject: [PATCH 026/109] pvr: generate the error-result wrappers from vk.xml Generate device-level VkResult wrappers at build time. Save original entrypoints per device so multiple architectures can coexist in a process. Install wrappers only with PVR_DEBUG=errors; the origin-logging change is independent and remains useful if these result wrappers are dropped. Assisted-by: LLM --- src/imagination/vulkan/meson.build | 11 ++++- src/imagination/vulkan/pvr_arch_device.c | 7 ++++ src/imagination/vulkan/pvr_device.h | 1 + src/imagination/vulkan/pvr_errlog.c | 47 +++++++++++++++++++++ src/imagination/vulkan/pvr_errlog.h | 7 ++++ src/imagination/vulkan/pvr_errlog_gen.py | 52 ++++++++++++++++++++++++ 6 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 src/imagination/vulkan/pvr_errlog.c create mode 100644 src/imagination/vulkan/pvr_errlog.h create mode 100644 src/imagination/vulkan/pvr_errlog_gen.py diff --git a/src/imagination/vulkan/meson.build b/src/imagination/vulkan/meson.build index 05cbd5f82b8..540d3ab098a 100644 --- a/src/imagination/vulkan/meson.build +++ b/src/imagination/vulkan/meson.build @@ -16,6 +16,14 @@ pvr_entrypoints = custom_target( depend_files : vk_entrypoints_gen_depend_files, ) +pvr_errlog_gen = custom_target( + 'pvr_errlog_gen', + input : [files('pvr_errlog_gen.py'), vk_api_xml], + output : 'pvr_errlog_gen.h', + command : [prog_python, '@INPUT0@', '--xml', '@INPUT1@', '--out', '@OUTPUT@'], + depend_files : vk_entrypoints_gen_depend_files, +) + pvr_drirc = custom_target( 'pvr_drirc', input : [files('pvr_drirc_gen.py'), drirc_gen, '00-pvr-defaults.conf'], @@ -165,6 +173,7 @@ pvr_files += files( 'pvr_g5p_image.c', 'pvr_g5p_nir.c', 'pvr_g5p_shader_dump.c', + 'pvr_errlog.c', 'pvr_g5p_texel_buffer.c', 'pvr_g5p_xfb_query.c', 'pvr_g5p_ds.c', @@ -219,7 +228,7 @@ endforeach libvulkan_powervr_mesa = shared_library( 'vulkan_powervr_mesa', - [pvr_files, pvr_entrypoints, pvr_drirc, u_format_pack_h], + [pvr_files, pvr_entrypoints, pvr_drirc, pvr_errlog_gen, u_format_pack_h], include_directories : [ pvr_includes, inc_imagination, diff --git a/src/imagination/vulkan/pvr_arch_device.c b/src/imagination/vulkan/pvr_arch_device.c index a409d1319be..5c1e3a4f185 100644 --- a/src/imagination/vulkan/pvr_arch_device.c +++ b/src/imagination/vulkan/pvr_arch_device.c @@ -35,6 +35,7 @@ #include "pvr_g9p.h" # include "pvr_g5p.h" #include "pvr_utrace.h" +#include "pvr_errlog.h" #include "util/os_misc.h" #define PVR_GLOBAL_FREE_LIST_INITIAL_SIZE (2U * 1024U * 1024U) @@ -829,6 +830,10 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, if (result != VK_SUCCESS) goto err_free_device; + result = pvr_errlog_install(device); + if (result != VK_SUCCESS) + goto err_vk_device_finish; + device->instance = instance; device->pdevice = pdevice; device->ws = ws; @@ -1061,6 +1066,7 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, pvr_bo_store_destroy(device); err_vk_device_finish: + vk_free(&device->vk.alloc, device->errlog_orig); vk_device_finish(&device->vk); err_free_device: @@ -1131,6 +1137,7 @@ void PVR_PER_ARCH(destroy_device)(struct pvr_device *device, pvr_bo_store_destroy(device); pvr_winsys_destroy(device->ws); p_atomic_dec(&device->instance->active_device_count); + vk_free(&device->vk.alloc, device->errlog_orig); vk_device_finish(&device->vk); vk_free(&device->vk.alloc, device); } diff --git a/src/imagination/vulkan/pvr_device.h b/src/imagination/vulkan/pvr_device.h index b22b767561d..ca2d5b760f1 100644 --- a/src/imagination/vulkan/pvr_device.h +++ b/src/imagination/vulkan/pvr_device.h @@ -70,6 +70,7 @@ struct pvr_compute_query_shader { struct pvr_device { struct vk_device vk; + struct vk_device_dispatch_table *errlog_orig; struct pvr_instance *instance; struct pvr_physical_device *pdevice; diff --git a/src/imagination/vulkan/pvr_errlog.c b/src/imagination/vulkan/pvr_errlog.c new file mode 100644 index 00000000000..9a87d7953c5 --- /dev/null +++ b/src/imagination/vulkan/pvr_errlog.c @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: MIT */ +#include "pvr_errlog.h" +#include "pvr_device.h" +#include "pvr_debug.h" +#include "pvr_g5p_shader_dump.h" +#include "util/log.h" +#include "vk_alloc.h" +#include "vk_log.h" +#include "vk_command_buffer.h" +#include "vk_queue.h" +#include "vk_enum_to_str.h" +#include +#include +#include +#include + +static void pvr_errlog_result(const char *entry, VkResult result) +{ + mesa_logw("pvr: %s returned %s", entry, vk_Result_to_str(result)); + const char *dir = pvr_g5p_shader_dump_dir(); + if (!dir) + return; + char path[4096], line[640]; + snprintf(path, sizeof(path), "%s/errors.tsv", dir); + int n = snprintf(line, sizeof(line), "%lld\t%ld\tresult\t%s\t%s\n", + (long long)time(NULL), (long)getpid(), vk_Result_to_str(result), entry); + int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (fd >= 0) { + ssize_t written = write(fd, line, MIN2(n, (int)sizeof(line) - 1)); + (void)written; + close(fd); + } +} + +#include "pvr_errlog_gen.h" + +VkResult pvr_errlog_install(struct pvr_device *device) +{ + if (!PVR_IS_DEBUG_SET(ERRORS)) + return VK_SUCCESS; + device->errlog_orig = vk_alloc(&device->vk.alloc, sizeof(*device->errlog_orig), + 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); + if (!device->errlog_orig) + return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + pvr_errlog_patch(&device->vk.dispatch_table, device->errlog_orig); + return VK_SUCCESS; +} diff --git a/src/imagination/vulkan/pvr_errlog.h b/src/imagination/vulkan/pvr_errlog.h new file mode 100644 index 00000000000..8d290249a88 --- /dev/null +++ b/src/imagination/vulkan/pvr_errlog.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef PVR_ERRLOG_H +#define PVR_ERRLOG_H +#include +struct pvr_device; +VkResult pvr_errlog_install(struct pvr_device *device); +#endif diff --git a/src/imagination/vulkan/pvr_errlog_gen.py b/src/imagination/vulkan/pvr_errlog_gen.py new file mode 100644 index 00000000000..743d96da361 --- /dev/null +++ b/src/imagination/vulkan/pvr_errlog_gen.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Generate per-device error-result wrappers from the Vulkan registry.""" +import argparse +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'vulkan' / 'util')) +from vk_entrypoints import get_entrypoints_from_xml + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--xml', required=True) + parser.add_argument('--out', required=True) + args = parser.parse_args() + entries = [e for e in get_entrypoints_from_xml([args.xml], beta=False) + if e.is_device_entrypoint() and e.return_type == 'VkResult' + and e.alias is None] + out = ['/* SPDX-License-Identifier: MIT */', + '/* Generated from vk.xml; do not edit. */', ''] + for e in entries: + if e.guard: + out += [f'#ifdef {e.guard}'] + p = e.params[0] + device = {'VkDevice': f'vk_device_from_handle({p.name})', + 'VkQueue': f'vk_queue_from_handle({p.name})->base.device', + 'VkCommandBuffer': f'vk_command_buffer_from_handle({p.name})->base.device'}[p.type] + out += ['static VKAPI_ATTR VkResult VKAPI_CALL', + f'pvr_errlog_{e.name}({e.decl_params()})', '{', + f' struct vk_device *pvr_errlog_vk = {device};', + ' struct pvr_device *pvr_errlog_device = container_of(pvr_errlog_vk, struct pvr_device, vk);', + f' VkResult r = pvr_errlog_device->errlog_orig->{e.name}({e.call_params()});', + ' if (r < 0)', f' pvr_errlog_result("vk{e.name}", r);', + ' return r;', '}'] + if e.guard: + out += ['#endif'] + out += [''] + out += ['static void', 'pvr_errlog_patch(struct vk_device_dispatch_table *t,', + ' struct vk_device_dispatch_table *orig)', '{', ' *orig = *t;'] + for e in entries: + if e.guard: + out += [f'#ifdef {e.guard}'] + out += [f' if (t->{e.name})', f' t->{e.name} = pvr_errlog_{e.name};'] + if e.guard: + out += ['#endif'] + out += ['}', ''] + Path(args.out).write_text('\n'.join(out)) + + +if __name__ == '__main__': + main() From 0d94475141c209620b28b5ee0b20cd4416af2144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:40 +0100 Subject: [PATCH 027/109] pvr/g5p: eight and more storage buffers per stage, dynamic offsets, vertex loads WebGPU's core limits want 8 storage buffers per shader stage; wgpu derives the count as maxDescriptorSetStorageBuffers / 2. The graphics backend took two storage buffers (Apple's fixed-load bases common0/common2), loaded them only at a constant element index, refused vertex-stage loads, and refused blend constants with storage buffers. - Up to 16 storage buffers per stage. Two or fewer without blend constants keep today's layout and bytes; otherwise the pointer pairs follow the stage's other common dwords. - Loads the fixed form cannot encode (a register offset, a pointer above common2, any vertex load) use addr.add_shift and the addressed 4/16-byte load of the register-indexed UBO path. A bank-0 vertex offset is copied into bank 1 first. addr.add_shift takes a pointer pair at any even common dword below 256. - Storage loads split into scalars or 16-byte-aligned vec4 loads, and stores into scalar stores (WGSL vec2/vec3 members). - The vertex upload loader takes 24 uploads. - maxDescriptorSetStorageBuffers goes from 12 to 16. The G9P liveness loop limit is spelled per backend (A9: G9P_MAX_LOOPS, A10: 64), so the A9's can be raised alone. Every shader of a 578-shader differential corpus is byte-identical. Assisted-by: LLM --- src/imagination/g5p/g5p_usc.c | 33 +++- src/imagination/g5p/pco_g5p_general.c | 150 ++++++++++++++++--- src/imagination/g5p/pco_g5p_general.h | 2 + src/imagination/g9p/g9p_compile.c | 2 +- src/imagination/vulkan/pvr_g5p_nir.c | 42 ++++++ src/imagination/vulkan/pvr_g5p_upload.h | 5 +- src/imagination/vulkan/pvr_physical_device.c | 2 +- 7 files changed, 210 insertions(+), 26 deletions(-) diff --git a/src/imagination/g5p/g5p_usc.c b/src/imagination/g5p/g5p_usc.c index 8ada20f53a8..86f8282913f 100644 --- a/src/imagination/g5p/g5p_usc.c +++ b/src/imagination/g5p/g5p_usc.c @@ -742,6 +742,16 @@ unsigned g5p_usc_mov_shared(uint8_t *out, unsigned dst_half, unsigned src_dword, return finish(out, raw, 12); } +/* Byte fields of the address add (g5p_usc_address_add_shift()) and later encoders. */ +#define G5P_HEADER_FRAGMENT 0x01 /* the header's continuation into the destination parcel, nothing else */ +#define G5P_HEADER_SRC_B_BANK1 0x10 /* source B in register bank 1 (g9p_w0()'s W0 bit 12; vertex code) */ +#define G5P_PARCEL_CONTINUES 0x01 +#define G5P_SRC_LAST_READ 0x80 /* a register source read for the last time (in the group bit's place) */ +#define G5P_CLASS_ADDR_ADD 0x89 /* the address add's class byte (the binary class with the 16-bit result) */ +#define G5P_ADDR_ADD_ROUTING_NONE 0x00 +#define G5P_ADDR_ADD_CTL 0x06 /* set in every witnessed address add (observed role) */ +#define G5P_ADDR_ADD_SHIFT_BIT 5 /* the shift, bits 5..7 of the control byte */ + /* ---- ISA round 2: inline shared sources (g5p_usc_resources.py) ---- */ static unsigned alu_header(uint8_t *raw, unsigned cls, unsigned dst_half, unsigned bank, unsigned header, unsigned source_extension, unsigned dst_control) @@ -1037,7 +1047,7 @@ unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t) raw[9] = 0; if (coupled) { /* Complete native operand/result recipe, reconstructed against the - * paired Metal programs and exact in r1c's non-cube device rungs. + * paired Metal programs and exact in the non-cube device cases. * Individual destination/coordinate continuation bits are unknown; * keep this suffix indivisible rather than guessing mixed forms. */ raw[9] = (1u << 7) | 1u; @@ -1190,10 +1200,25 @@ unsigned g5p_usc_address_add_shift(uint8_t *out, unsigned pair_half, unsigned ba CHECK(half_register(pair_half, "pair_half requires a 64-bit aligned register pair") && !(pair_half & 2), "pair_half requires a 64-bit aligned register pair"); CHECK(half_register(index_half, "index_half requires a measured 32-bit register"), g5p_usc_error); - CHECK(base_common_dword < 32 && !(base_common_dword & 1), "pointer base must be an even common dword below 32"); + CHECK(base_common_dword < 256 && !(base_common_dword & 1), "pointer base must be an even common dword below 256"); CHECK(shift <= 4, "only shifts 0-4 are corpus-verified"); - uint8_t raw[12] = {0x89, vs ? 0x11 : 0x01, pair_half, vs ? 0x03 : 0x01, 0x81 | 4 * base_common_dword, - (kill ? 0x80 : 0) | 1 | index_half << 1, 0x00, 0x41, 0xa1, 0x01, 0x00, 0x06 | shift << 5}; + /* The layout of g5p_usc_alu_shared(): alu_header() (vertex code: destination bank 1, and B's bank bit in + * the header), A = the pointer base (shared_src(), group 1; above dword 31 its high bits are the field that + * the vertex XFB record's address add and every shared ALU operand use), B = the index with its last read, + * the integer add, then the last parcel: no routing, and the control byte with the shift. */ + uint8_t raw[12], a[4]; + if (!alu_header(raw, G5P_CLASS_ADDR_ADD, pair_half, vs, G5P_HEADER_FRAGMENT | (vs ? G5P_HEADER_SRC_B_BANK1 : 0), + 0, 0) || + !shared_src(a, base_common_dword, 1)) + return 0; + raw[4] = a[0]; + raw[5] = (index_half << 1) | (kill ? G5P_SRC_LAST_READ : 0) | G5P_PARCEL_CONTINUES; + raw[6] = a[2]; + raw[7] = a[3]; + raw[8] = binop_code[G5P_BIN_IADD]; + raw[9] = G5P_PARCEL_CONTINUES; + raw[10] = G5P_ADDR_ADD_ROUTING_NONE; + raw[11] = G5P_ADDR_ADD_CTL | shift << G5P_ADDR_ADD_SHIFT_BIT; return finish(out, raw, 12); } diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index c0a425b3728..fd743b36542 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -106,8 +106,8 @@ enum { BLEND_NONE, BLEND_PREMULT, BLEND_STRAIGHT_OVER, BLEND_REPLACE, BLEND_GENE /* The general blend epilogue (g5p_blend.c, hyprland-gpu.md §11.2) for * settings without a measured epilogue. Premultiplied, straight-over and * replace keep their paths. By default only the settings of the device - * blend ladder (linux-a9 blend-ladder.c, release7 2026-11-02: 45 rungs, - * every pixel within 1 LSB) and of the bench ladder's three blend rungs + * blend ladder (45 cases on the A9, + * every pixel within 1 LSB) and of the bench ladder's three blend cases * take their original byte-identical shape. All other core equations use * the same tile protocol with canonical native-f16 ALU lowering. */ bool pco_g5p_blend_general(void) @@ -759,6 +759,7 @@ static bool alu_supported(struct gctx *c, nir_alu_instr *a) /* Register-indexed UBO loads (defined with the common-store plan). */ static bool dyn_offset_alu(nir_alu_instr *a); static bool dyn_ubo_load(nir_intrinsic_instr *in); +static bool eval_offset(nir_def *d, unsigned comp, uint32_t *v); static bool dyn_ubo_offset(nir_intrinsic_instr *in, struct ref *idx, unsigned *shift, uint32_t *base); /* Vertex iadd that only feeds UBO/push offsets folds into the upload plan. */ @@ -1179,7 +1180,8 @@ static void scan_cf(struct gctx *c, struct exec_list *list, struct unsupported * ok = in->src[0].ssa->bit_size==32 && in->num_components==1 && nir_intrinsic_write_mask(in)==1; break; case nir_intrinsic_load_ssbo: - ok = !c->vs; + /* 4- or 16-byte loads (g5p_nir_prepare splits the rest) */ + ok = in->def.bit_size == 32 && (in->def.num_components == 1 || in->def.num_components == 4); break; case nir_intrinsic_load_vertex_id: case nir_intrinsic_load_instance_id: @@ -1723,6 +1725,12 @@ static void liveness(struct gctx *c) mark(c, ix, idx); continue; } + if (in->intrinsic == nir_intrinsic_load_ssbo) { + uint32_t off; + if (!eval_offset(in->src[1].ssa, 0, &off)) + mark(c, resolve_value(c, in->src[1].ssa, 0), idx); + continue; + } if (in->intrinsic == nir_intrinsic_store_global) { unsigned b, k; struct ref off; @@ -2121,25 +2129,33 @@ static void plan_common(struct gctx *c) for (unsigned i = 0; i < c->nssbo; i++) found |= c->ssbo_set[i] == set && c->ssbo_binding[i] == binding; if (!found) { - if (c->nssbo == 2) - refuse(c, "load_ssbo: only pointer bases common0/common2 are witnessed (two buffers)"); + if (c->nssbo == PCO_G5P_MAX_SSBOS) + refuse(c, "more than %u storage buffers", PCO_G5P_MAX_SSBOS); c->ssbo_set[c->nssbo] = set; c->ssbo_binding[c->nssbo++] = binding; } } } unsigned next = 0; - if (!c->vs && c->blend == BLEND_GENERAL && c->binfo.uses_constants) { - /* Apple's layout: the blend colour in common 0..3, everything else after it */ - if (c->nssbo) - refuse(c, "blend constants with storage buffers (both want common dwords 0..3)"); + bool blend_constants = !c->vs && c->blend == BLEND_GENERAL && c->binfo.uses_constants; + /* Apple's layout: the blend colour in common 0..3, everything else after + * it. Storage buffers keep common0/common2 only when they are at most two + * and the blend colour does not want those dwords; otherwise their + * pointers follow the stage's other common dwords (below). */ + c->ssbo_high = c->nssbo > 2 || (c->nssbo && blend_constants); + if (blend_constants) { add_upload(c, PCO_G5P_UPLOAD_BLEND_CONSTANTS, 0, 0, 0, 0, 4, G5P_BLEND_CONST_BASE); next = 4; } - for (unsigned i = 0; i < c->nssbo; i++) + for (unsigned i = 0; !c->ssbo_high && i < c->nssbo; i++) { + c->ssbo_common[i] = 2 * i; add_upload(c, PCO_G5P_UPLOAD_SSBO_ADDR, c->ssbo_set[i], c->ssbo_binding[i], 0, 0, 2, 2 * i); - if (c->nssbo) + } + if (c->nssbo && !c->ssbo_high) next = MAX2(next, 4); + /* High storage pointers: an even dword each after the data, so the + * plan reserves their dwords (plus one for alignment) at its end. */ + const unsigned ssbo_reserve = c->ssbo_high ? 2 * c->nssbo + 1 : 0; bool qualified = c->vs && g5p_vtex_qualified(); /* Preserve the flown <=4-texture layout. Larger fragment programs use * Apple's f16 capacity layout: all eight-dword images, then samplers. @@ -2167,7 +2183,7 @@ static void plan_common(struct gctx *c) } if (!g5p_stage_common_plan(next, c->ntex, sampler_count + fetch_sampler, compact, c->nuptr, c->nsize, ubos, - ubo_dwords, push_dwords, &common_plan)) + ubo_dwords, push_dwords + ssbo_reserve, &common_plan)) refuse(c, "common layout cannot fit pointers below 32 and data below 256 dwords"); compact = common_plan.compact; if (common_plan.low_pointers) { @@ -2266,7 +2282,16 @@ static void plan_common(struct gctx *c) (r->hi - r->lo) / 4, r->common); } if (planned) - assert(next == common_plan.end); + assert(next + ssbo_reserve == common_plan.end); + /* addr.add_shift reads a pointer pair at any even common dword below 256 + * (the shared-operand field's high bits, g5p_usc_address_add_shift). */ + for (unsigned i = 0; c->ssbo_high && i < c->nssbo; i++) { + c->ssbo_common[i] = next = align(next, 2); + next += 2; + if (next > limit) + refuse(c, "storage-buffer pointers exceed %u common-store dwords", limit); + add_upload(c, PCO_G5P_UPLOAD_SSBO_ADDR, c->ssbo_set[i], c->ssbo_binding[i], 0, 0, 2, c->ssbo_common[i]); + } /* Transform feedback block: pointer pairs at even dwords (address add). */ if (xfb) { c->xfb_common = align(next, 2); @@ -3431,7 +3456,7 @@ static void emit_isub(struct gctx *c, nir_alu_instr *a) * tools/mesa-a9-vtx-re/probes.metal, byte-equal to these encoders). The * fragment form (bank 0, header 1) in a vertex program reads and writes * other registers: x >> 2 then read again gave x on hardware (vertex-input's - * gl_VertexIndex rungs). Dynamic amounts have no vertex probe. */ + * gl_VertexIndex cases). Dynamic amounts have no vertex probe. */ static void emit_shift(struct gctx *c, nir_alu_instr *a) { struct ref x = alu_src(c, a, 0), amt = alu_src(c, a, 1); @@ -4343,7 +4368,7 @@ static void stage_ssbo_store(struct gctx *c,nir_intrinsic_instr *in) nir_intrinsic_offset_shift(in)!=0 || nir_intrinsic_align_mul(in)<4 || nir_intrinsic_align_offset(in)%4) refuse(c,"store_ssbo: aligned scalar32-bit byte offsets only"); unsigned set,binding,base=0;block_binding(in->src[1].ssa,&set,&binding); - for(unsigned i=0;inssbo;i++)if(c->ssbo_set[i]==set && c->ssbo_binding[i]==binding)base=2*i; + for(unsigned i=0;inssbo;i++)if(c->ssbo_set[i]==set && c->ssbo_binding[i]==binding)base=c->ssbo_common[i]; struct ref offset=resolve_value(c,in->src[2].ssa,0),value=resolve_value(c,in->src[0].ssa,0); unsigned index=reg_of(c,offset,"store_ssbo byte offset"),data=reg_of(c,value,"store_ssbo value"); if(c->vs)vs_bank1_only(c,"vertex SSBO store",index,data,-1,-1); @@ -4365,6 +4390,87 @@ static void stage_ssbo_store(struct gctx *c,nir_intrinsic_instr *in) c->p->evidence|=c->vs?PCO_G5P_EV_VS_STORE:PCO_G5P_EV_FS_STORE; } +/* A vertex address operand from bank 0 is copied into bank 1 (the only + * witnessed vertex addr.add_shift form reads bank 1). */ +static unsigned vs_bank1_index(struct gctx *c, unsigned reg) +{ + if (!c->vs || vbank(c, reg)) + return reg; + uint8_t raw[G5P_USC_MAX]; + bool bank0 = c->vs_bank0; + c->vs_bank0 = false; + unsigned b1 = alloc_reg(c, -1); + c->vs_bank0 = bank0; + put(c, raw, g5p_usc_move(raw, H(c, b1), H(c, reg), 1, 0, vhdr(c, 0x21, reg, -1, -1)), + "bank0 address offset into bank1"); + c->scratch |= BITFIELD64_BIT(b1); + vs_bank1_only(c, "vertex storage-buffer address offset", b1, -1, -1, -1); + return b1; +} + +/* A storage-buffer load through its pointer pair: pair = pointer + byte + * offset (addr.add_shift, shift 0), then the addressed 4/16-byte load that + * Apple's register-indexed constant-buffer programs use (emit_dyn_ubo). + * The offset is the load's own register, or a scratch immediate for a + * constant offset that the fixed form cannot encode. g5p_nir_prepare() + * leaves only scalar loads and 16-byte-aligned vec4 loads. */ +static void emit_address_ssbo_load(struct gctx *c, nir_intrinsic_instr *in, unsigned base) +{ + uint8_t raw[G5P_USC_MAX]; + const unsigned n = in->def.num_components, bytes = 4 * n; + uint32_t off; + bool constant = eval_offset(in->src[1].ssa, 0, &off); + if (constant && off % 4) + refuse(c, "load_ssbo: unaligned constant offset %u", off); + bool kill; + unsigned index; + struct ref ix = {0}; + if (constant) { + index = scratch_imm(c, off); + kill = true; + } else { + if (n == 4 && nir_intrinsic_align(in) < 16) + refuse(c, "load_ssbo: 16-byte load without 16-byte alignment"); + ix = resolve_value(c, in->src[1].ssa, 0); + index = reg_of(c, ix, "load_ssbo byte offset"); + kill = dies(c, ix); + } + unsigned raw_index = index; + index = vs_bank1_index(c, index); + if (index != raw_index) + kill = true; /* the bank-1 copy is scratch */ + if (!constant && kill && index == raw_index) + drop(c, V(c, ix.def, ix.comp)); + unsigned pair; + for (pair = 0; pair + 2 <= NUM_DWORDS; pair += 2) + if (!((c->busy | c->reserved | c->tex_coord_pending | c->tex_pending) & BITFIELD64_RANGE(pair, 2))) + break; + if (pair + 2 > NUM_DWORDS) + refuse(c, "load_ssbo: register pressure for an address pair"); + c->busy |= BITFIELD64_RANGE(pair, 2); + c->max_dword = MAX2(c->max_dword, pair + 2); + c->max_written = MAX2(c->max_written, pair + 2); + put(c, raw, g5p_usc_address_add_shift(raw, 2 * pair, base, H(c, index), 0, kill, c->vs), + "storage-buffer address"); + const unsigned dst = alloc_run(c, n, "load_ssbo"); + if (c->vs) { + vs_bank1_only(c, "vertex storage-buffer load", dst, -1, -1, -1); + put(c, raw, g5p_usc_vertex_address_load(raw, 2 * dst, bytes, 2 * pair), "vertex storage-buffer load"); + } else { + put(c, raw, g5p_usc_address_load(raw, 2 * dst, bytes, 2 * pair, 0), "fragment storage-buffer load"); + } + put(c, raw, g5p_usc_wait(raw, 0), "wait"); + c->busy &= ~BITFIELD64_RANGE(pair, 2); + free_scratch(c); + for (unsigned k = 0; k < n; k++) { + struct val *v = V(c, &in->def, k); + v->state = VAL_REG; + v->reg = dst + k; + v->held = true; + } + c->p->evidence |= PCO_G5P_EV_BUFFER_LOAD; +} + static void emit_intrinsic(struct gctx *c, nir_intrinsic_instr *in) { uint8_t raw[G5P_USC_MAX]; @@ -4456,11 +4562,17 @@ static void emit_intrinsic(struct gctx *c, nir_intrinsic_instr *in) unsigned base = 0; for (unsigned i = 0; i < c->nssbo; i++) if (c->ssbo_set[i] == set && c->ssbo_binding[i] == binding) - base = 2 * i; - unsigned off = load_offset(c, in, 1, 0); + base = c->ssbo_common[i]; + uint32_t off = 0; unsigned n = in->def.num_components, bytes = n == 1 ? 4 : n == 4 ? 16 : 0; - if (!bytes || off % bytes || off / bytes > 255) - refuse(c, "load_ssbo: only 4/16-byte element loads at a constant element index <= 255 are witnessed"); + if (!bytes || in->def.bit_size != 32) + refuse(c, "load_ssbo: only 4/16-byte 32-bit loads"); + bool fixed = eval_offset(in->src[1].ssa, 0, &off) && !c->vs && base <= 2 && + !(off % bytes) && off / bytes <= 255; + if (!fixed) { + emit_address_ssbo_load(c, in, base); + break; + } unsigned dst = alloc_run(c, n, "load_ssbo"); put(c, raw, g5p_usc_buffer_load(raw, 2 * dst, bytes, base, off / bytes, 0), "fixed buffer load"); put(c, raw, g5p_usc_wait(raw, 0), "wait"); diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index 6515bcefa7b..4a946dd4b26 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -16,6 +16,8 @@ * ICD's job arena is what bounds a draw's programs, see pco_g5p_general.c). */ #define PCO_G5P_PROGRAM_MAX (1u << 20) #define PCO_G5P_MAX_UPLOADS 40 +/* storage buffers per stage (maxPerStageDescriptorStorageBuffers) */ +#define PCO_G5P_MAX_SSBOS 16 #define PCO_G5P_MAX_ATTRIBS 16 #define PCO_G5P_COMMON_DWORDS 64 /* Every vertex program that has flown needs <= 12 temps; 16 and 20 wedged diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index c9636902ed8..a95c0675531 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -4880,7 +4880,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.ip = 0; walk_liveness(&ctx, &impl->body, false); - if (ctx.loop_index > G9P_MAX_LOOPS) { + if (ctx.loop_index > (out->a9 ? G9P_MAX_LOOPS : 64)) { fprintf(stderr, "g9p: too many loops\n"); return false; } diff --git a/src/imagination/vulkan/pvr_g5p_nir.c b/src/imagination/vulkan/pvr_g5p_nir.c index 3547f5dcee4..ba2c39efff0 100644 --- a/src/imagination/vulkan/pvr_g5p_nir.c +++ b/src/imagination/vulkan/pvr_g5p_nir.c @@ -289,6 +289,47 @@ clamp_dyn_ubo(nir_shader *nir) static nir_shader *g5p_nir_prepare(nir_shader *nir); +/* Storage-buffer accesses the backend encodes: a 32-bit scalar, or a vec4 + * load at a 16-byte-aligned offset (one 16-byte addressed load). Wider or + * misaligned vectors become scalars at offset + 4k (WGSL's vec2/vec3 and + * struct members), and stores become one scalar store per written + * component. A constant offset keeps a vec4 that the fixed load already + * encodes, so earlier programs are unchanged. */ +static bool +split_ssbo_access(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + if (in->intrinsic == nir_intrinsic_load_ssbo) { + const unsigned n = in->def.num_components; + if (in->def.bit_size != 32 || n == 1) + return false; + if (n == 4 && (nir_intrinsic_align(in) >= 16 || + (nir_src_is_const(in->src[1]) && !(nir_src_as_uint(in->src[1]) % 16)))) + return false; + b->cursor = nir_before_instr(&in->instr); + nir_def *comp[NIR_MAX_VEC_COMPONENTS]; + for (unsigned k = 0; k < n; k++) + comp[k] = nir_load_ssbo(b, 1, 32, in->src[0].ssa, nir_iadd_imm(b, in->src[1].ssa, 4 * k), + .access = nir_intrinsic_access(in), .align_mul = 4, .align_offset = 0); + nir_def_replace(&in->def, nir_vec(b, comp, n)); + return true; + } + if (in->intrinsic == nir_intrinsic_store_ssbo) { + const unsigned n = in->src[0].ssa->num_components; + const unsigned mask = nir_intrinsic_write_mask(in); + if (in->src[0].ssa->bit_size != 32 || (n == 1 && mask == 1)) + return false; + b->cursor = nir_before_instr(&in->instr); + u_foreach_bit(k, mask) + nir_store_ssbo(b, nir_channel(b, in->src[0].ssa, k), in->src[1].ssa, + nir_iadd_imm(b, in->src[2].ssa, 4 * k), .write_mask = 1, + .access = nir_intrinsic_access(in), .align_mul = 4, .align_offset = 0); + nir_instr_remove(&in->instr); + return true; + } + return false; +} + + /* SPIR-V -> optimised NIR (the ICD's g5p_nir()) */ static nir_shader * g5p_nir(const uint32_t *spv, size_t words, mesa_shader_stage stage, const char *entry) @@ -339,6 +380,7 @@ g5p_nir_prepare(nir_shader *nir) nir_shader_gather_xfb_info(nir); nir_lower_io_passes(nir, false); nir_lower_explicit_io(nir, nir_var_mem_ssbo | nir_var_mem_ubo, nir_address_format_32bit_index_offset); + nir_shader_intrinsics_pass(nir, split_ssbo_access, nir_metadata_control_flow, NULL); bool progress; do { progress = false; diff --git a/src/imagination/vulkan/pvr_g5p_upload.h b/src/imagination/vulkan/pvr_g5p_upload.h index f535e124e70..909420a2fb0 100644 --- a/src/imagination/vulkan/pvr_g5p_upload.h +++ b/src/imagination/vulkan/pvr_g5p_upload.h @@ -33,7 +33,10 @@ struct pco_g5p_program; struct pvr_cmd_buffer; struct pvr_device; -#define PVR_G5P_MAX_STAGE_UPLOADS 16 +/* 8+ storage-buffer pointers, 13 UBOs, push constants and the XFB block + * (WebGPU core per-stage maxima together). The loader's data is one 16-byte + * unit per upload; VDMCTRL_PDS_STATE0.pds_data_size holds up to 63. */ +#define PVR_G5P_MAX_STAGE_UPLOADS 24 /* One upload of a stage: what the backend's pco_g5p_upload names */ struct pvr_g5p_stage_upload { diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index 1ec6f23e03f..8577f0218e7 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -695,7 +695,7 @@ static bool pvr_physical_device_get_properties( .maxDescriptorSetSamplers = 3U * 16U, .maxDescriptorSetUniformBuffers = 3U * 12U, .maxDescriptorSetUniformBuffersDynamic = 8U, - .maxDescriptorSetStorageBuffers = 3U * 4U, + .maxDescriptorSetStorageBuffers = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 16U : 3U * 4U, .maxDescriptorSetStorageBuffersDynamic = 4U, .maxDescriptorSetSampledImages = 3U * 16U, .maxDescriptorSetStorageImages = 3U * 4U, From 94430e89850e060517a9215d06c8934bd228215b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:40 +0100 Subject: [PATCH 028/109] pvr/g5p: 64 KiB uniform buffers; register-indexed vec2/vec3 UBO reads WebGPU's maxUniformBufferBindingSize is 64 KiB; the A9 reported 16 KiB. Nothing in the A9 paths depends on 16 KiB: a UBO range that does not fit the 256-dword common store is read through its pointer at any 32-bit offset, and a register-indexed read adds the index to the pointer. So maxUniformBufferRange goes from 16 to 64 KiB. WGSL reads vec2/vec3 members of UBO arrays at a register index, which the backend refused. A vec2/vec3 at the start of a 16-byte element now loads the whole element with the 16-byte form and frees the unused registers after the wait. Any other register-indexed vec2/vec3 UBO load is split into scalars after constant folding. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 25 +++++++++++++------- src/imagination/vulkan/pvr_g5p_nir.c | 25 ++++++++++++++++++++ src/imagination/vulkan/pvr_physical_device.c | 2 +- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index fd743b36542..a89fdc21f57 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -4202,11 +4202,14 @@ static void emit_dyn_ubo(struct gctx *c, nir_intrinsic_instr *in) uint32_t base; block_binding_element(in->src[0].ssa, &set, &binding, &element); dyn_ubo_offset(in, &ix, &shift, &base); - unsigned bytes = 4 * n; /* Witnessed: 16-byte loads (Apple's v_cexpr, vec4 elements at shift 4) and * 4-byte loads (G9P's, at any byte offset); the shift field is - * corpus-verified 0-4. vec2/vec3 (8-byte, 12-byte) loads are not. */ - if (in->def.bit_size != 32 || !((n == 4 && shift == 4) || (n == 1 && shift != 1)) || base % bytes) + * corpus-verified 0-4. vec2/vec3 (8-byte, 12-byte) loads are not: a + * vec2/vec3 at the start of a 16-byte element (nir_opt_shrink_vectors of + * a vec4 read) loads the whole element and drops the unused registers. */ + const unsigned loaded = (n == 2 || n == 3) && shift == 4 && !(base % 16) ? 4 : n; + unsigned bytes = 4 * loaded; + if (in->def.bit_size != 32 || !((loaded == 4 && shift == 4) || (n == 1 && shift != 1)) || base % bytes) refuse(c, "load_ubo: register-indexed %u x %u-bit at shift %u, base %u (witnessed: vec4 at shift 4, " "scalar at shift 0/2-4, element-aligned base)", n, in->def.bit_size, shift, base); /* Apple's form for an unsigned index is this one: umin, then addr.add_shift @@ -4246,18 +4249,22 @@ static void emit_dyn_ubo(struct gctx *c, nir_intrinsic_instr *in) c->max_dword = MAX2(c->max_dword, pair + 2); put(c, raw, g5p_usc_address_add_shift(raw, 2 * pair, common, 2 * index, shift, kill, c->vs), "addr.add_shift"); unsigned dst = 0; - for (dst = 0; dst + n <= NUM_DWORDS; dst += n) - if (!((c->busy | c->reserved | c->tex_coord_pending | c->tex_pending) & BITFIELD64_RANGE(dst, n))) + for (dst = 0; dst + loaded <= NUM_DWORDS; dst += loaded) + if (!((c->busy | c->reserved | c->tex_coord_pending | c->tex_pending) & BITFIELD64_RANGE(dst, loaded))) break; - if (dst + n > NUM_DWORDS) - refuse(c, "register pressure: load_ubo needs %u aligned dword(s)", n); - c->busy |= BITFIELD64_RANGE(dst, n); - c->max_dword = MAX2(c->max_dword, dst + n); + if (dst + loaded > NUM_DWORDS) + refuse(c, "register pressure: load_ubo needs %u aligned dword(s)", loaded); + c->busy |= BITFIELD64_RANGE(dst, loaded); + c->max_dword = MAX2(c->max_dword, dst + loaded); + if (loaded != n) /* the new whole-element form; hardware-tested forms keep their bytes */ + c->max_written = MAX2(c->max_written, dst + loaded); if (c->vs) put(c, raw, g5p_usc_vertex_address_load(raw, 2 * dst, bytes, 2 * pair), "vertex address load"); else put(c, raw, g5p_usc_address_load(raw, 2 * dst, bytes, 2 * pair, 0), "fragment address load"); put(c, raw, g5p_usc_wait(raw, 0), "wait"); + /* the load has completed (wait): the element's unused tail is free */ + c->busy &= ~BITFIELD64_RANGE(dst + n, loaded - n); for (unsigned k = 0; k < n; k++) { struct val *v = V(c, &in->def, k); v->state = VAL_REG; diff --git a/src/imagination/vulkan/pvr_g5p_nir.c b/src/imagination/vulkan/pvr_g5p_nir.c index ba2c39efff0..af0c8de972d 100644 --- a/src/imagination/vulkan/pvr_g5p_nir.c +++ b/src/imagination/vulkan/pvr_g5p_nir.c @@ -329,6 +329,26 @@ split_ssbo_access(nir_builder *b, nir_intrinsic_instr *in, void *data) return false; } +/* After constant folding: a register-indexed vec2/vec3 UBO load becomes + * scalars at offset + 4k (the backend encodes scalar and vec4 forms; + * vec4_dyn_ubo() then merges complete quads). Constant-offset loads are + * preloaded and stay as they are. */ +static bool +split_dyn_ubo(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + const unsigned n = in->def.num_components; + if (in->intrinsic != nir_intrinsic_load_ubo || in->def.bit_size != 32 || (n != 2 && n != 3) || + nir_src_is_const(in->src[1])) + return false; + b->cursor = nir_before_instr(&in->instr); + nir_def *comp[NIR_MAX_VEC_COMPONENTS]; + for (unsigned k = 0; k < n; k++) + comp[k] = nir_load_ubo(b, 1, 32, in->src[0].ssa, nir_iadd_imm(b, in->src[1].ssa, 4 * k), + .access = nir_intrinsic_access(in), .align_mul = 4, .align_offset = 0, + .range_base = 0, .range = ~0u); + nir_def_replace(&in->def, nir_vec(b, comp, n)); + return true; +} /* SPIR-V -> optimised NIR (the ICD's g5p_nir()) */ static nir_shader * @@ -393,6 +413,11 @@ g5p_nir_prepare(nir_shader *nir) NIR_PASS(progress, nir, nir_opt_dead_cf); } while (progress); progress = false; + if (nir_shader_intrinsics_pass(nir, split_dyn_ubo, nir_metadata_control_flow, NULL)) { + NIR_PASS(_, nir, nir_opt_constant_folding); + NIR_PASS(_, nir, nir_opt_algebraic); + NIR_PASS(_, nir, nir_opt_cse); + } NIR_PASS(progress, nir, vec4_dyn_ubo); if (stage == MESA_SHADER_FRAGMENT && pco_g5p_fs_dyn_ubo()) NIR_PASS(progress, nir, clamp_dyn_ubo); diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index 8577f0218e7..c0d6114d478 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -677,7 +677,7 @@ static bool pvr_physical_device_get_properties( .maxImageDimensionCube = 4096U, .maxImageArrayLayers = rogue_get_render_size_max_z(dev_info), .maxTexelBufferElements = 64U * 1024U, - .maxUniformBufferRange = 16U * 1024U, + .maxUniformBufferRange = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 64U * 1024U : 16U * 1024U, .maxStorageBufferRange = 128U * 1024U * 1024U, .maxPushConstantsSize = 128U, .maxMemoryAllocationCount = 4096U, From 2729022cafc80cce09fb56b2379e1a191738e92e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:40 +0100 Subject: [PATCH 029/109] pvr/g5p: up to 13 uniform buffers in a stage's common-store plan The driver reports 13 UBOs per stage and WebGPU wants 12, but the plan refused more than 8, and the compiler tracked at most 8 uniform ranges. Thirteen spilled UBO pointers plus four dwords of blend constants still fit the 32 dwords that addr.add_shift's low pointers use (4 + 2 * 13 = 30). A plan that does not fit is still refused by name. Assisted-by: LLM --- src/imagination/g5p/g5p_fs_common.h | 2 +- src/imagination/g5p/pco_g5p_general.c | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/imagination/g5p/g5p_fs_common.h b/src/imagination/g5p/g5p_fs_common.h index d731dbd7f88..eab1f4661df 100644 --- a/src/imagination/g5p/g5p_fs_common.h +++ b/src/imagination/g5p/g5p_fs_common.h @@ -24,7 +24,7 @@ g5p_stage_common_plan(unsigned start, unsigned textures, unsigned samplers, uint64_t push_dwords, struct g5p_fs_common_plan *p) { if (start > 4 || textures > 16 || samplers > 17 || - dynamic_ptrs > 16 || size_queries > 4 || ubos > 8) + dynamic_ptrs > 16 || size_queries > 4 || ubos > 13) return false; *p = (struct g5p_fs_common_plan){0}; /* Each retry only enables one of two monotone choices. A layout which diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index a89fdc21f57..7f5d1087177 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -244,10 +244,15 @@ struct gctx { bool cube_gradient; /* fixed native bank0 r0..r8 operand tuple */ struct { unsigned set, binding, element, lod, common; } tsize[4]; /* textureSize/QueryLevels */ unsigned nsize; - struct range ranges[8]; + struct range ranges[16]; /* 13 UBOs per stage (maxPerStageDescriptorUniformBuffers) + push constants */ unsigned nranges; - unsigned ssbo_set[2], ssbo_binding[2]; + /* Storage buffers: one pointer pair each. Up to two take Apple's witnessed + * fixed-load bases common0/common2 (the hardware-tested layout, byte-identical); + * more, or any with blend constants, sit after the stage's other common + * dwords (ssbo_high) and every access goes through addr.add_shift. */ + unsigned ssbo_set[PCO_G5P_MAX_SSBOS], ssbo_binding[PCO_G5P_MAX_SSBOS], ssbo_common[PCO_G5P_MAX_SSBOS]; unsigned nssbo; + bool ssbo_high; /* register-indexed UBO loads: one pointer (UBO VA + constant base) each. * The pairs sit at even common dwords below 32 (addr.add_shift's base * field), so at most 16; plan_common() refuses a pair past dword 30 and From c8fbd42c92df4daf31ffb672eb13a97553ae0196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:41 +0100 Subject: [PATCH 030/109] pvr/g5p: accept a register UBO index whose bound NIR has proven Each fragment register UBO index is wrapped in umin(i, last). When the index is already provably at most `last` (u[x & 3] in uvec4[4], a common WGSL shape), nir_opt_algebraic drops the umin, and the backend then refused the load as unclamped. Accept an index that nir_unsigned_upper_bound() proves below 4096, the 16-byte elements of a 64 KiB block. An index without a proven bound is still refused. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 7f5d1087177..c17b98a5289 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -4190,13 +4190,24 @@ bool pco_g5p_fuse_ffma(void) return true; } -/* The index is clamp_dyn_ubo's umin(i, last), a constant bound inside the block. */ +/* The index is clamp_dyn_ubo's umin(i, last), a constant bound inside the + * block; or nir_opt_algebraic removed that umin because the index is + * already proven to be at most `last` (x & 3 into uvec4[4]). The second form + * is accepted only with a proven bound inside the largest UBO + * (maxUniformBufferRange 64 KiB = 4096 elements of 16 bytes), never a + * wild 32-bit index. */ static bool clamped_index(struct ref ix) { nir_alu_instr *a = alu_of(ix.def); - if (!a || a->op != nir_op_umin) - return false; - return nir_src_is_const(a->src[0].src) || nir_src_is_const(a->src[1].src); + if (a && a->op == nir_op_umin && + (nir_src_is_const(a->src[0].src) || nir_src_is_const(a->src[1].src))) + return true; + nir_instr *ins = nir_def_instr(ix.def); + nir_shader *shader = nir_cf_node_get_function(&ins->block->cf_node)->function->shader; + struct hash_table *ht = _mesa_pointer_hash_table_create(NULL); + uint32_t bound = nir_unsigned_upper_bound(shader, ht, nir_get_scalar(ix.def, ix.comp)); + _mesa_hash_table_destroy(ht, NULL); + return bound < 4096; } static void emit_dyn_ubo(struct gctx *c, nir_intrinsic_instr *in) From ce7eebaaec3510cf889f217e0f4a82cbdfda559e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:03:41 +0100 Subject: [PATCH 031/109] pvr/g5p: compile 28 fragment varying locations The 16-location cap was an admission choice, not the hardware's. The A9 interpolates 124 components exactly (tested on the device), and the vertex side already handles 32 locations. The fragment compiler now admits 28 locations (112 components: 4 + 112 + 1 + 8 = 125 of the TA's 128-dword vertex, with the position, a point size and 8 clip distances). The varying PDS program is sized by the same constant. The advertised limit follows in a later commit, once Zink keeps 16 GL varyings at 112 components. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 4 ++-- src/imagination/g5p/pco_g5p_general.h | 15 ++++++++------- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 3 +++ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index c17b98a5289..89c89d9be6d 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -1535,8 +1535,8 @@ static void plan_varyings(struct gctx *c) mask[n] |= BITFIELD64_MASK(in->def.num_components) << nir_intrinsic_component(in); } if (util_bitcount(read) > PCO_G5P_MAX_FS_VARYINGS) - refuse(c, "%u varying locations read (%u vec4: the reported 64 components)", util_bitcount(read), - PCO_G5P_MAX_FS_VARYINGS); + refuse(c, "%u varying locations read (%u vec4: the reported %u components)", util_bitcount(read), + PCO_G5P_MAX_FS_VARYINGS, 4 * PCO_G5P_MAX_FS_VARYINGS); bool smooth = false; u_foreach_bit(n, read) smooth |= interp[n] == PCO_G5P_INTERP_SMOOTH; diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index 4a946dd4b26..f0ec3dac8c4 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -138,13 +138,14 @@ enum pco_g5p_upload_kind { /* Varyings (docs/gpu/experiments/varyings.md). Apple's A9 compiler takes at * most 60 interpolated components ("Number of varying components(64) * exceeds the limit (60)"): 15 vec4 fragment inputs. */ -/* Plan row R2 (desktop-gl-varyings.md section 5): 16 vec4, the 64 components - * the driver reports (maxVertexOutputComponents, maxFragmentInputComponents) - * and stock Zink's IMG GL limit at 64 (16 varyings). Beyond Apple's per-target - * 60, inside the hardware-proven range (varyings-capacity-1: exact to 124 on - * the A9, flat included). With the position, a point size and 8 clip - * distances the vertex is 4 + 64 + 1 + 8 = 77 dwords, inside the TA's 128. */ -#define PCO_G5P_MAX_FS_VARYINGS 16 +/* 28 vec4, the 112 components the driver reports (maxVertexOutputComponents, + * maxFragmentInputComponents): WebGPU core wants 16 inter-stage locations + * (wgpu: components / 4 - 1). With the position, a point size + * and 8 clip distances the vertex is 4 + 112 + 1 + 8 = 125 dwords, inside the + * TA's 128. Beyond Apple's per-target 60, inside the hardware-proven range + * (varying-capacity and geometry-shader probes: exact to 124 on the A9, flat + * included). Before: 16 vec4, 64 components. */ +#define PCO_G5P_MAX_FS_VARYINGS 28 #define PCO_G5P_VARYING_SLOTS 32 /* VARYING_SLOT_VAR0..VAR31 */ enum pco_g5p_interp { diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 112d286f12f..7c7dd55b093 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -331,6 +331,9 @@ launch_program(uint32_t *dest) * the TA vertex, which drops x, y, z: dword d is TA dword d - 3, in 16-bit * units). Returns the data dwords; *code_dwords the code's. */ +/* pvr_physical_device.c reports what the fragment compiler admits */ +static_assert(4 * PCO_G5P_MAX_FS_VARYINGS == 112, "G5P varying components"); + static unsigned varying_program(const struct pco_g5p_program *fs, const struct pco_g5p_program *vs, uint32_t *data, uint32_t *code, unsigned *code_dwords) From 7fa38bac19bbc625209702de3f59664ba1f01724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:07 +0100 Subject: [PATCH 032/109] pvr/g5p: five to eight colour targets, compact UNORM8 WebGPU core wants 8 colour attachments; the A9 reported 4. For 5 to 8 targets of one format, R8G8B8A8_UNORM or B8G8R8A8_UNORM: - each target's RGBA is packed into one dword at its output store (saturate, round to even), written with one emitpix of N words at pixel output width 8, and stored through a physical R32UI PBE (no second conversion); - CLEAR converts the f32 clear colours in the background program; LOAD texel-fetches every target (one texture state each); the end-of-tile state-load selector gets its high bits; - a partial write mask, blending, a CLEAR/LOAD mix in one pass, and wider or sRGB formats at 5 to 8 are refused by name. 1, 2 and 4 targets keep their programs byte for byte. maxColorAttachments, maxFragmentOutputAttachments and maxFragmentCombinedOutputResources go from 4 to 8. Tested on the A9 before the powervr interface: 104 images at 5, 6, 7 and 8 targets (clear, draw, LOAD, fractional colours), every pixel of every target. Assisted-by: LLM --- src/imagination/g5p/g5p_mrt_out.h | 17 ++-- src/imagination/g5p/g5p_mrt_programs.h | 63 +++++++++++--- src/imagination/g5p/g5p_usc.c | 68 +++++++++++++++ src/imagination/g5p/g5p_usc.h | 7 ++ src/imagination/g5p/pco_g5p_general.c | 70 ++++++++++++++- src/imagination/g5p/pco_g5p_general.h | 4 + src/imagination/vulkan/pvr_arch_g5p.c | 90 ++++++++++++++------ src/imagination/vulkan/pvr_arch_g5p_blend.c | 17 +++- src/imagination/vulkan/pvr_arch_g5p_dgl.c | 2 +- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 4 + src/imagination/vulkan/pvr_arch_g5p_mrt.c | 18 +++- src/imagination/vulkan/pvr_g5p.h | 14 ++- src/imagination/vulkan/pvr_g5p_formats.c | 8 ++ src/imagination/vulkan/pvr_g5p_formats.h | 2 + src/imagination/vulkan/pvr_g5p_mrt.h | 5 +- src/imagination/vulkan/pvr_g5p_nir.c | 36 ++++++++ src/imagination/vulkan/pvr_physical_device.c | 6 +- 17 files changed, 372 insertions(+), 59 deletions(-) diff --git a/src/imagination/g5p/g5p_mrt_out.h b/src/imagination/g5p/g5p_mrt_out.h index 144cf91f826..c02e2161fec 100644 --- a/src/imagination/g5p/g5p_mrt_out.h +++ b/src/imagination/g5p/g5p_mrt_out.h @@ -23,7 +23,9 @@ #include #include -#define G5P_MRT_MAX_TARGETS 4 +/* 1..4: packed-f16 targets (two dwords each); 5..8: compact UNORM8 targets + * (one packed dword each, verified on the A9 over 104 test images) */ +#define G5P_MRT_MAX_TARGETS 8 #define G5P_MRT_FS_SLOTS 8 /* FRAG_RESULT_DATA0..DATA7 */ #define G5P_MRT_HALVES_PER_TARGET 4 #define G5P_MRT_MAX_HALVES (G5P_MRT_MAX_TARGETS * G5P_MRT_HALVES_PER_TARGET) @@ -49,7 +51,7 @@ static inline const char *g5p_mrt_refusal_str(enum g5p_mrt_refusal r) case G5P_MRT_NO_TARGETS: return "no colour targets"; case G5P_MRT_TOO_MANY_TARGETS: - return "more than four colour targets"; + return "more than eight colour targets"; case G5P_MRT_THREE_TARGETS: return "three colour targets: Apple's compiler was only observed for 1, 2 and 4 " "(the pixel output width 6 is not a kernel width)"; @@ -106,7 +108,8 @@ static inline unsigned g5p_mrt_emit_words(unsigned field) /* Word counts for which an Apple A9 witness exists for a homogeneous set. */ static inline bool g5p_mrt_emit_words_witnessed(unsigned words) { - return words == 1 || words == 2 || words == 4 || words == 8; + /* 1, 2, 4, 8 (homogeneous 8-bit RGBA and raw sets), 5..7 (compact UNORM8) */ + return words >= 1 && words <= 8 && words != 3; } static inline enum g5p_mrt_refusal g5p_mrt_out_plan_make(unsigned targets, struct g5p_mrt_out_plan *p) @@ -119,11 +122,13 @@ static inline enum g5p_mrt_refusal g5p_mrt_out_plan_make(unsigned targets, struc if (targets == 3) return G5P_MRT_THREE_TARGETS; p->targets = targets; - p->words = targets * G5P_MRT_WORDS_PER_TARGET; + /* 5..8: one packed UNORM8 dword per target (the compact layout), the + * pass at the kernel's widest pixel output width 8 */ + p->words = targets > 4 ? targets : targets * G5P_MRT_WORDS_PER_TARGET; p->emit_field = g5p_mrt_emit_field(p->words); - p->pixel_output_width = p->words; /* [D] 2/4/8, matches the kernel table; mrt2/mrt4 capture confirms */ + p->pixel_output_width = targets > 4 ? 8 : p->words; /* [D] 2/4/8, matches the kernel table; mrt2/mrt4 capture confirms */ for (unsigned t = 0; t < targets; t++) - p->first_half[t] = g5p_mrt_half(t, 0); + p->first_half[t] = targets > 4 ? 2 * t : g5p_mrt_half(t, 0); return G5P_MRT_OK; } diff --git a/src/imagination/g5p/g5p_mrt_programs.h b/src/imagination/g5p/g5p_mrt_programs.h index cbecff1198c..73810cb749f 100644 --- a/src/imagination/g5p/g5p_mrt_programs.h +++ b/src/imagination/g5p/g5p_mrt_programs.h @@ -27,24 +27,28 @@ #include #include -#define G5P_MRT_MAX 4 +#include "g5p_usc.h" + +#define G5P_MRT_MAX 8 /* 5..8: compact UNORM8 targets */ #define G5P_MRT_PBE_DWORDS 7 /* per target */ #define G5P_MRT_PBE_STRIDE 8 /* dwords per target in the common store */ +#define G5P_MRT_PBE_HALF_OFFSET 6 /* a target's output-half offset in its PBE block */ #define G5P_MRT_CLEAR_DWORDS 4 /* f32 clear colour per target */ #define G5P_MRT_CLEAR_USC_MAX (16 * 4 * G5P_MRT_MAX + 8) -#define G5P_MRT_EOT_USC_MAX 130 +#define G5P_MRT_EOT_USC_MAX 194 -/* Target counts with an Apple witness for the whole pass: 1, 2 and 4. */ +/* Target counts with an Apple witness for the whole pass: 1, 2 and 4; 5..8 + * as compact UNORM8 (verified on the A9 over 104 test images). */ static inline bool g5p_mrt_count_ok(unsigned n) { - return n == 1 || n == 2 || n == 4; + return n == 1 || n == 2 || (n >= 4 && n <= 8); } /* Pixel output width of the pass (UAPI pixel_output_width): two output * registers per 8-bit RGBA target. iOS record+0x6dc: 2, 4, 8. */ static inline uint32_t g5p_mrt_pixel_width(unsigned n) { - return 2 * n; + return n > 4 ? 8 : 2 * n; /* compact: one dword per target, the kernel's width 8 */ } /* PBE dwords the end-of-tile DOUTD moves: 7, 15, 31. */ @@ -58,7 +62,8 @@ static inline unsigned g5p_mrt_pbe_dwords(unsigned n) * 0x402, 0x402, 0x802; Apple's sizes are twice DIV_ROUND_UP(dwords, 16)). */ static inline uint32_t g5p_mrt_eot_info(unsigned n) { - return n == 4 ? 0x802 : 0x402; + /* the same values for 1, 2, 4; 0xc02 and 0x1002 for 5..8 (39..63 dwords) */ + return ((g5p_mrt_pbe_dwords(n) + 15) / 16) * 0x400 | 2; } /* Partial-render restore SIZEINFO (restore[1], record+0x51c): usc_sharedsize @@ -92,9 +97,39 @@ static inline unsigned g5p_mrt_pbe_block(uint32_t *out, unsigned n, const uint64 return g5p_mrt_pbe_block_halves(out, n, pbe, 4); } +/* 5..8 UNORM targets: one packed dword per target; BGRA8 + * byte order is B,G,R,A. Each f32 clear channel in common dword 4t + c is + * converted to UNORM8 into one byte of an output half (g5p_usc_unorm8_byte(), + * the packing form recovered from controlled probes), then one emitpix + * writes the N packed dwords and the program ends. Returns its size, + * 16 * 4N + 6, or 0 when an encoder refuses (never for 5..8 targets). */ +static inline unsigned g5p_mrt_clear_compact(uint8_t *out, unsigned n, bool bgra) +{ + unsigned o = 0, len; + for (unsigned t = 0; t < n; t++) + for (unsigned h = 0; h < 2; h++) + for (unsigned byte = 0; byte < 2; byte++) { + const unsigned c = h * 2 + byte; + const unsigned channel = bgra && !(c & 1) ? 2 - c : c; + const unsigned src = 4 * t + channel, dst = 2 * t + h; + len = g5p_usc_unorm8_byte(out + o, dst, byte, src); + if (!len) + return 0; + o += len; + } + /* emitpix of the N packed dwords from output half 0, then END */ + len = g5p_usc_emit_mrt(out + o, n, 0); + if (!len) + return 0; + o += len; + return o + g5p_usc_special(out + o, true); +} + /* The clear program. Returns its size, 16 * 4N + 6. */ static inline unsigned g5p_mrt_clear_usc(uint8_t *out, unsigned n) { + if (n > 4) + return g5p_mrt_clear_compact(out, n, true); unsigned o = 0; for (unsigned t = 0; t < n; t++) for (int c = 3; c >= 0; c--) { @@ -111,16 +146,17 @@ static inline unsigned g5p_mrt_clear_usc(uint8_t *out, unsigned n) } /* The end-of-tile program. Returns its size: 66 for one or two targets, 130 - * for four. */ + * for four; 0 when an encoder refuses (never for the counts g5p_mrt_count_ok() accepts). */ static inline unsigned g5p_mrt_eot_usc_mode(uint8_t *out, unsigned n, bool integer) { unsigned o = 0; for (unsigned k = 0; k < n; k++) { - uint8_t t = n - 1 - k; - const uint8_t ins[12] = {0x88, 0x01, (uint8_t)(2 * t), 0x01, (uint8_t)(0x99 + 0x20 * k), 0x01, - 0x00, 0x41, 0x28, 0x01, 0x39, 0x00}; - memcpy(out + o, ins, sizeof(ins)); - o += sizeof(ins); + /* output half 2 * (n - 1 - k) = target k's output-half offset (in its PBE block) */ + const unsigned len = g5p_usc_mov_shared_half(out + o, 2 * (n - 1 - k), + G5P_MRT_PBE_STRIDE * k + G5P_MRT_PBE_HALF_OFFSET); + if (!len) + return 0; + o += len; } for (unsigned k = 0; k < n; k++) { bool last = k + 1 == n; @@ -149,6 +185,7 @@ static inline unsigned g5p_mrt_eot_usc_mode(uint8_t *out, unsigned n, bool integ */ static inline unsigned g5p_mrt_eot_usc(uint8_t *out, unsigned n) { - return g5p_mrt_eot_usc_mode(out, n, false); + /* 5..8: packed bytes through a physical R32UI store, the integer source mode */ + return g5p_mrt_eot_usc_mode(out, n, n > 4); } #endif diff --git a/src/imagination/g5p/g5p_usc.c b/src/imagination/g5p/g5p_usc.c index 86f8282913f..261157f5726 100644 --- a/src/imagination/g5p/g5p_usc.c +++ b/src/imagination/g5p/g5p_usc.c @@ -793,6 +793,74 @@ static unsigned add_route(uint8_t *raw, unsigned n, int route) return n; } +/* Names shared by the fragment encoders. */ +#define G5P_CLASS_TERNARY_RESULT_HALF 0x8a /* g5p_usc_literal_ternary() with result_half */ +#define G5P_TERNARY_CTL_RESULT_HALF 0x02 + +/* One byte of a packed UNORM8 colour, for the clear of 5..8 colour targets (g5p_mrt_programs.h): the f32 in + * common-store dword src_dword, converted to an unsigned normalized byte, into byte 0 or byte 1 of the 16-bit + * dst_half, bank 0. Two of them fill one half: the byte-1 form reads dst_half as B and keeps the byte that the + * byte-0 form wrote. It is the float-to-integer conversion in the three-source class with the 16-bit result, + * laid out as literal_prefix() followed by a common-store C: + * A a short literal, the conversion's target format: G5P_SMALL_UN8, the format code of + * g5p_usc_convert_small() (g5p_usc_f2u16_half() puts G5P_SMALL_U16 in the same place); + * B byte 0: unused (selector 0); byte 1: dst_half, group 1; + * C the common-store dword (shared_src(), group 1); + * opcode the float-to-integer conversion (convert_code[], as g5p_usc_f2u16_half()); + * control bit 0 the continuation into the last parcel; bit 1 the 16-bit result; bit 2 set by every + * conversion form (g5p_usc_convert2(), g5p_usc_f2u16_half(), g5p_usc_convert_small()); the byte-1 + * form adds bits 4 and 6, which were only observed together (bit 6 also selects the source's high + * byte in g5p_usc_convert_small()); + * routing the low byte of the last parcel, observed per byte; its high byte, the source control, is 0. + * The form was recovered from controlled probes; the clear was tested on the A9 at 5, 6, 7 and 8 targets. */ +#define G5P_CONVERT_CTL 0x04 +#define G5P_UNORM8_CTL_BYTE1 0x50 +#define G5P_UNORM8_ROUTING_BYTE0 0x10 +#define G5P_UNORM8_ROUTING_BYTE1 0x1c +#define G5P_SOURCE_CONTROL_NONE 0x00 + +unsigned g5p_usc_unorm8_byte(uint8_t *out, unsigned dst_half, unsigned byte, unsigned src_dword) +{ + CHECK(byte <= 1, "a half has bytes 0 and 1"); + const unsigned dst_bank = 0, c_group = 1; + const unsigned b_half = byte ? dst_half : 0, b_group = byte ? 1 : 0; + uint8_t raw[G5P_USC_MAX]; + unsigned n = literal_prefix(raw, G5P_CLASS_TERNARY_RESULT_HALF, dst_half, dst_bank, G5P_SMALL_UN8, b_half, + b_group, G5P_HEADER_FRAGMENT, 0); + if (!n) + return 0; + if (!shared_src(raw + n, src_dword, c_group)) + return 0; + n += 4; + raw[n++] = convert_code[G5P_F2U32]; + raw[n++] = G5P_TERNARY_CTL_RESULT_HALF | G5P_CONVERT_CTL | (byte ? G5P_UNORM8_CTL_BYTE1 : 0) | + G5P_PARCEL_CONTINUES; + raw[n++] = byte ? G5P_UNORM8_ROUTING_BYTE1 : G5P_UNORM8_ROUTING_BYTE0; + raw[n++] = G5P_SOURCE_CONTROL_NONE; + return finish(out, raw, n); +} + +/* A move of a common-store dword into a 16-bit half of bank 0 (the end-of-tile program's state moves, + * g5p_mrt_programs.h): alu_header() with the 16-bit move class (G5P_CLASS_MOVE_HALF, as g5p_usc_half_immediate()) + * and the fragment header, A = the common-store dword (shared_src(), group 1), then the half move's opcode and + * control (G5P_HALF_MOVE_OP, G5P_HALF_MOVE_CTL: the bytes of g5p_usc_half_immediate()'s last parcel, named by + * that observed role) and no source control. */ +#define G5P_CLASS_MOVE_HALF 0x88 +#define G5P_HALF_MOVE_OP 0x28 +#define G5P_HALF_MOVE_CTL 0x39 +unsigned g5p_usc_mov_shared_half(uint8_t *out, unsigned dst_half, unsigned src_dword) +{ + uint8_t raw[12]; + if (!alu_header(raw, G5P_CLASS_MOVE_HALF, dst_half, 0, G5P_HEADER_FRAGMENT, 0, 0) || + !shared_src(raw + 4, src_dword, 1)) + return 0; + raw[8] = G5P_HALF_MOVE_OP; + raw[9] = G5P_PARCEL_CONTINUES; + raw[10] = G5P_HALF_MOVE_CTL; + raw[11] = G5P_SOURCE_CONTROL_NONE; + return finish(out, raw, 12); +} + unsigned g5p_usc_alu_shared(uint8_t *out, enum g5p_binop op, unsigned dst_half, unsigned a_dword, unsigned b_half, unsigned bank, unsigned b_group, unsigned header, unsigned source_extension, unsigned dst_control, bool negate_b, diff --git a/src/imagination/g5p/g5p_usc.h b/src/imagination/g5p/g5p_usc.h index dfa571ea8b3..82d98a13609 100644 --- a/src/imagination/g5p/g5p_usc.h +++ b/src/imagination/g5p/g5p_usc.h @@ -217,6 +217,13 @@ unsigned g5p_usc_cube_gradient_tuple(uint8_t *out); /* f32 -> u16 (truncating) into one half: Apple's array-layer conversion * `89 01 DH 01 09 SS 20 01 60 06`. */ unsigned g5p_usc_f2u16_half(uint8_t *out, unsigned dst_half, unsigned src_half, unsigned group); +/* One byte of a packed UNORM8 colour: the f32 in common-store dword src_dword as an unsigned normalized byte in + * byte 0 or 1 of dst_half, bank 0; the byte-1 form keeps byte 0 of dst_half (the fields are listed at the + * encoder). The clear of 5..8 colour targets (g5p_mrt_programs.h). */ +/* Common-store dword src_dword (group 1) into the 16-bit dst_half of bank 0 (fields at the encoder): the + * end-of-tile program's state moves (g5p_mrt_programs.h). */ +unsigned g5p_usc_mov_shared_half(uint8_t *out, unsigned dst_half, unsigned src_dword); +unsigned g5p_usc_unorm8_byte(uint8_t *out, unsigned dst_half, unsigned byte, unsigned src_dword); /* length 0 = minimum */ unsigned g5p_usc_buffer_load(uint8_t *out, unsigned dst_half, unsigned byte_count, unsigned base_common_dword, unsigned index, unsigned length); diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 89c89d9be6d..0b6f95a98aa 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -5379,9 +5379,43 @@ static bool out_const(struct gctx *c, unsigned k, uint16_t *h) return true; } +static bool compact_output(const struct pco_g5p_fs_key *key) +{ + return key->color_output == PCO_G5P_COLOR_PACKED_BGRA8 || + key->color_output == PCO_G5P_COLOR_PACKED_RGBA8; +} + +/* Semantic UNORM8 conversion, the same saturate/round-even rule as + * nir_format_float_to_unorm(), lowered through existing G5P ALU encoders. + * Packing occurs at each output store, so eight targets keep eight dwords + * rather than 32 live float components. */ +static bool lower_compact_store(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + const struct pco_g5p_fs_key *key = data; + if (in->intrinsic != nir_intrinsic_store_output) + return false; + if (nir_intrinsic_component(in) || in->src[0].ssa->num_components != 4 || nir_intrinsic_write_mask(in) != 15) + return false; /* refused in fs_output: no partial packed output contract */ + b->cursor = nir_before_instr(&in->instr); + nir_def *rgba = in->src[0].ssa; + nir_def *packed = nir_imm_int(b, 0); + for (unsigned c = 0; c < 4; c++) { + const unsigned channel = key->color_output == PCO_G5P_COLOR_PACKED_BGRA8 && !(c & 1) ? 2 - c : c; + nir_def *v = nir_fsat(b, nir_channel(b, rgba, channel)); + v = nir_f2u32(b, nir_fround_even(b, nir_fmul_imm(b, v, 255.0f))); + packed = nir_ior(b, packed, nir_ishl_imm(b, v, 8 * c)); + } + nir_src_rewrite(&in->src[0], packed); + nir_intrinsic_set_write_mask(in, 1); + in->num_components = 1; + return true; +} + /* Output-plan decisions that need use counts. */ static void plan_fs_output(struct gctx *c) { + if (compact_output(c->key)) + return; /* g5p_mrt_out_plan_make(): one word per target, width 8 */ if (c->key->color_output != PCO_G5P_COLOR_F16) { /* four raw words per target: 4, or 8 for two targets (emitpix field 0, Apple's 4-target width) */ if (c->mrt.targets > 1) { /* one emitpix writes both targets: both need their stores */ @@ -5519,6 +5553,32 @@ static void fs_output(struct gctx *c) } fs_discards(c); c->now = LAST_END; + if (compact_output(c->key)) { + /* target t's packed dword: output word t (c->out[4t], lower_compact_store()) */ + struct pvr_g5p_dgl_source sources[8] = {0}; + for (unsigned t = 0; t < c->mrt.targets; t++) { + if (c->slot_written[t] != 1) + refuse(c, "compact MRT requires a full four-channel store to each target"); + struct ref r = c->out[4 * t]; + sources[t].immediate = is_const(c, r); + if (sources[t].immediate) { + sources[t].value = V(c, r.def, r.comp)->imm; + } else { + sources[t].value = reg_of(c, r, "compact MRT output"); + sources[t].group = vgrp(c, r); + for (unsigned j = t + 1; j < c->mrt.targets; j++) + if (c->out[4 * j].def == r.def && c->out[4 * j].comp == r.comp) + sources[t].group = 0; + } + } + uint8_t code[10 * G5P_USC_MAX]; + size_t size = pvr_g5p_dgl_output_words(code, sizeof(code), sources, c->mrt.words, 0xf); + if (!size) + refuse(c, "compact MRT output source cannot be encoded"); + put(c, code, size, "compact MRT packed output and emitpix"); + free_scratch(c); + return; + } if (c->key->color_output != PCO_G5P_COLOR_F16) { /* target t's channels: output words 4t..4t+3 (c->out[] is indexed 4 * slot + component) */ const unsigned words = c->mrt.words; @@ -7794,7 +7854,15 @@ static bool compile_general_link(nir_shader *nir, bool half_output, const uint8_ mr = g5p_mrt_check_key(&c->mrt, c->key->blend, c->key->write_mask); if (mr != G5P_MRT_OK) refuse(c, "%s", g5p_mrt_refusal_str(mr)); - if (c->key->color_output != PCO_G5P_COLOR_F16) { + if (compact_output(c->key) != (c->mrt.targets > 4)) + refuse(c, "five to eight colour targets take the compact UNORM8 output, fewer do not"); + if (compact_output(c->key)) { + if (c->key->blend || c->key->write_mask != 15 || c->key->mrt_blend_state) + refuse(c, "five to eight colour targets: blending and partial write masks are not built " + "(compact MRT is full-write and unblended)"); + NIR_PASS(_, nir, nir_shader_intrinsics_pass, lower_compact_store, nir_metadata_control_flow, + (void *)c->key); + } else if (c->key->color_output != PCO_G5P_COLOR_F16) { if (c->key->mrt_blend_state) refuse(c, "raw32 MRT blend or masks are not qualified"); if (c->key->color_output != PCO_G5P_COLOR_UINT32 && diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index f0ec3dac8c4..51992c327eb 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -82,6 +82,10 @@ enum pco_g5p_color_output { PCO_G5P_COLOR_F16 = 0, PCO_G5P_COLOR_UINT32, PCO_G5P_COLOR_SINT32, + /* 5..8 UNORM8 targets: each target's RGBA packed into one dword (BGRA8: + * bytes B,G,R,A), stored through a physical R32UI PBE */ + PCO_G5P_COLOR_PACKED_BGRA8, + PCO_G5P_COLOR_PACKED_RGBA8, }; struct pco_g5p_rt_blend { diff --git a/src/imagination/vulkan/pvr_arch_g5p.c b/src/imagination/vulkan/pvr_arch_g5p.c index 2abc6d9b36a..536773eb1cb 100644 --- a/src/imagination/vulkan/pvr_arch_g5p.c +++ b/src/imagination/vulkan/pvr_arch_g5p.c @@ -56,7 +56,7 @@ #include "pvr_query.h" #include "pvr_rogue_pds_encode.h" -static const unsigned g5p_mrt_variant_targets[PVR_G5P_MRT_VARIANTS] = { 1, 2, 4 }; +static const unsigned g5p_mrt_variant_targets[PVR_G5P_MRT_VARIANTS] = { 1, 2, 4, 5, 6, 7, 8, 5, 6, 7, 8 }; /* The DOUTU temps of every G5P program (G5P_USC_TEMPS of the A9 ICD) */ #define G5P_TEMPS 8 @@ -230,6 +230,12 @@ g5p_free_programs(struct pvr_device *device, struct pvr_g5p_programs *p) vk_free(&device->vk.alloc, p->raw_load[i]); } } + for (unsigned i = 0; i < ARRAY_SIZE(p->mrt_load); i++) { + if (p->mrt_load[i]) { + g5p_free_programs(device, p->mrt_load[i]); + vk_free(&device->vk.alloc, p->mrt_load[i]); + } + } for (unsigned i = 0; i < ARRAY_SIZE(p->raw_clear_usc); i++) { pvr_bo_suballoc_free(p->raw_clear_usc[i]); pvr_bo_suballoc_free(p->raw_clear_kick[i].pvr_bo); @@ -263,7 +269,9 @@ g5p_upload_load(struct pvr_device *device, struct pvr_g5p_programs *p, unsigned struct pco_g5p_program prog; VkResult result = VK_SUCCESS; - if (!(raw == 10 || raw == 11 ? pvr_g5p_compile_load_scalar8(&prog, raw == 11) : (raw == 8 || raw == 9) ? pvr_g5p_compile_load_packed_sint(&prog, raw == 8 ? 8 : 16) : raw == 7 ? pvr_g5p_compile_load_rgb10a2ui(&prog) : raw == 6 ? pvr_g5p_compile_load_rgba8ui(&prog) : raw == 5 ? pvr_g5p_compile_load_rgba16ui(&prog) : raw ? pvr_g5p_compile_load_raw(&prog, raw == 2) : pvr_g5p_compile_load(&prog))) { + /* raw 12..19: the compact LOAD of 5..8 targets, BGRA8 (12..15) then RGBA8 (16..19) */ + if (!(raw >= 12 ? pvr_g5p_compile_load_mrt(&prog, 5 + (raw - 12) % 4, raw < 16) : + raw == 10 || raw == 11 ? pvr_g5p_compile_load_scalar8(&prog, raw == 11) : (raw == 8 || raw == 9) ? pvr_g5p_compile_load_packed_sint(&prog, raw == 8 ? 8 : 16) : raw == 7 ? pvr_g5p_compile_load_rgb10a2ui(&prog) : raw == 6 ? pvr_g5p_compile_load_rgba8ui(&prog) : raw == 5 ? pvr_g5p_compile_load_rgba16ui(&prog) : raw ? pvr_g5p_compile_load_raw(&prog, raw == 2) : pvr_g5p_compile_load(&prog))) { snprintf(p->load_refusal, sizeof(p->load_refusal), "%s", prog.refusal); free(prog.code); return VK_SUCCESS; @@ -276,13 +284,14 @@ g5p_upload_load(struct pvr_device *device, struct pvr_g5p_programs *p, unsigned for (unsigned i = 0; i < prog.num_uploads; i++) { const struct pco_g5p_upload *u = &prog.uploads[i]; if ((u->kind != PCO_G5P_UPLOAD_IMAGE && u->kind != PCO_G5P_UPLOAD_SAMPLER && - u->kind != PCO_G5P_UPLOAD_IMAGE_SIZE) || u->set || u->binding) { + u->kind != PCO_G5P_UPLOAD_IMAGE_SIZE) || u->set || u->binding >= (raw >= 12 ? 8u : 1u)) { snprintf(p->load_refusal, sizeof(p->load_refusal), "a load program upload of kind %u at %u.%u", u->kind, u->set, u->binding); free(prog.code); return VK_SUCCESS; } p->load_upload[i].kind = u->kind; + p->load_upload[i].binding = u->binding; p->load_upload[i].dwords = u->dwords; p->load_upload[i].common = u->common; } @@ -397,7 +406,8 @@ g5p_upload_programs(struct pvr_device *device, struct pvr_g5p_programs *p) for (unsigned i = 0; i < PVR_G5P_MRT_VARIANTS; i++) { const unsigned n = g5p_mrt_variant_targets[i]; uint8_t code[MAX2(G5P_MRT_CLEAR_USC_MAX, G5P_MRT_EOT_USC_MAX)]; - unsigned size = g5p_mrt_clear_usc(code, n); + /* variants 7..10: the RGBA8 byte order of the compact clear */ + unsigned size = i >= 7 ? g5p_mrt_clear_compact(code, n, false) : g5p_mrt_clear_usc(code, n); result = pvr_gpu_upload_usc(device, code, size, 64, &p->clear_usc[i]); if (result != VK_SUCCESS) @@ -545,12 +555,9 @@ g5p_pbe_encode(uint64_t q[3], uint32_t key, bool srgb, uint64_t va, uint32_t wid * them. Apple's LOAD program is not used. */ static VkResult -g5p_load_background(struct pvr_cmd_buffer *cmd_buffer, const uint64_t state[4], uint32_t width, uint32_t height, - uint64_t bgnd[4], unsigned raw) +g5p_load_background(struct pvr_cmd_buffer *cmd_buffer, const uint64_t state[][4], uint32_t width, uint32_t height, + uint64_t bgnd[4], const struct pvr_g5p_programs *p) { - const struct pvr_g5p_programs *p = &cmd_buffer->device->g5p->programs; - if (raw) - p = p->raw_load[raw - 1]; const VkSamplerCreateInfo sci = { .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, .magFilter = VK_FILTER_NEAREST, @@ -568,7 +575,12 @@ g5p_load_background(struct pvr_cmd_buffer *cmd_buffer, const uint64_t state[4], /* textureSize()'s block (PCO_G5P_UPLOAD_IMAGE_SIZE): width, height, layers, levels */ const uint32_t size[4] = { width, height, 1, 1 }; struct pvr_suballoc_bo *state_bo, *sampler_bo, *size_bo; - VkResult result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, state, 4 * sizeof(state[0]), &state_bo); + /* one 32-byte texture state per target an IMAGE upload reads (5..8 targets: binding = target) */ + unsigned views = 1; + for (unsigned i = 0; i < p->load_uploads; i++) + if (p->load_upload[i].kind == PCO_G5P_UPLOAD_IMAGE) + views = MAX2(views, p->load_upload[i].binding + 1); + VkResult result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, state, views * sizeof(state[0]), &state_bo); if (result != VK_SUCCESS) return result; result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, sampler, sizeof(sampler), &sampler_bo); @@ -582,7 +594,7 @@ g5p_load_background(struct pvr_cmd_buffer *cmd_buffer, const uint64_t state[4], uint32_t data[4 * ARRAY_SIZE(p->load_upload)] = { 0 }; for (unsigned i = 0; i < n; i++) { const unsigned kind = p->load_upload[i].kind; - const uint64_t src = kind == PCO_G5P_UPLOAD_IMAGE ? state_bo->dev_addr.addr + const uint64_t src = kind == PCO_G5P_UPLOAD_IMAGE ? state_bo->dev_addr.addr + 32 * p->load_upload[i].binding : kind == PCO_G5P_UPLOAD_SAMPLER ? sampler_bo->dev_addr.addr : size_bo->dev_addr.addr; if (p->load_upload[i].dwords > (kind == PCO_G5P_UPLOAD_IMAGE ? 8u : kind == PCO_G5P_UPLOAD_SAMPLER ? 2u : 4u)) @@ -638,7 +650,8 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd case 1: v = 0; break; case 2: v = 1; break; case 4: v = 2; break; - default: return G5P_REFUSE(cmd_buffer, "%u colour targets (1, 2, 3 or 4 are built)", n); + case 5: case 6: case 7: case 8: v = nt - 2; break; /* compact UNORM8; RGBA8 adds 4 below */ + default: return G5P_REFUSE(cmd_buffer, "%u colour targets (1 to 8 are built)", n); } /* several targets: Apple's pass shape (pvr_g5p_mrt.h) */ VkResult mrt = pvr_arch_g5p_mrt_pass_check(cmd_buffer, hw_render); @@ -669,7 +682,7 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd return vdm; /* targets: PBE words and clear colours */ - uint64_t pbe[4][3]; + uint64_t pbe[G5P_MRT_MAX][3]; unsigned raw = 0; const VkFormat first_format = null_target ? VK_FORMAT_UNDEFINED : info->attachments[hw_render->eot_surfaces[0].attachment_idx]->vk.format; @@ -705,9 +718,14 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd g->raw_words = raw_total; g->raw_pixel_width = raw >= 5 ? source.pixel_output_width : raw_total; } - float clear[4][4] = { { 0 } }; - /* a loaded target: its texture state (q0, q1, q2, 0), the load program's image upload */ - uint64_t load_state[4] = { 0 }; + /* 5..8 targets: one packed UNORM8 dword each (pvr_g5p_mrt.h admits RGBA8 or BGRA8 only) */ + const bool compact = nt > 4; + if (compact && first_format == VK_FORMAT_R8G8B8A8_UNORM) + v += 4; + float clear[G5P_MRT_MAX][4] = { { 0 } }; + /* a loaded target: its texture state (q0, q1, q2, 0), the load program's image upload (every target's + * for 5..8 targets: the compact LOAD reads each) */ + uint64_t load_state[G5P_MRT_MAX][4] = { { 0 } }; if (null_target) { uint32_t key; bool srgb; @@ -757,6 +775,12 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd * the original signed image key. Blending is rejected for integers. */ uint32_t store_key = raw ? key & ~(0x80u << 5) : key; + /* Compact outputs already hold the exact four destination bytes. A + * physical R32UI store keeps them; the UNORM PBE key expects f16 + * source channels and would convert the packed bits. The logical + * sampling key and the four-byte pixel/row layout stay. */ + if (compact) + store_key = pvr_g5p_packed_word_pbe_key(); if (raw >= 5 && raw <= 11) { /* Equal-size raw physical view; logical sampling stays typed. * Native witness-r16 supplies the narrow16-bit source layout. */ @@ -780,7 +804,7 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd g->msaa_resolve = true; } } - if (!t) { + if (!t || compact) { if(twiddled) { struct pvr_image_descriptor desc; /* A one-layer array attachment is still a 2D store/restore @@ -790,10 +814,10 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd load_view.vk.view_type=VK_IMAGE_VIEW_TYPE_2D; const char *why=pvr_arch_g5p_pack_image_view(&load_view,&desc); if(why)return G5P_REFUSE(cmd_buffer,"twiddled LOAD view: %s",why); - load_state[0]=desc.words[0];load_state[1]=desc.words[1]; - load_state[2]=(uint64_t)desc.meta[0]|(uint64_t)desc.meta[1]<<32; + load_state[t][0]=desc.words[0];load_state[t][1]=desc.words[1]; + load_state[t][2]=(uint64_t)desc.meta[0]|(uint64_t)desc.meta[1]<<32; } else - pvr_g5p_tex_linear(load_state, key, srgb, addr, iview->vk.extent.width, iview->vk.extent.height,level->pitch); + pvr_g5p_tex_linear(load_state[t], key, srgb, addr, iview->vk.extent.width, iview->vk.extent.height,level->pitch); } if (!t) { g->width = iview->vk.extent.width; @@ -830,8 +854,9 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd "%ux%u): no clear of the area alone", t, area->offset.x, area->offset.y, area->extent.width, area->extent.height, g->width, g->height); if (hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD) { - if (n != 1 || t) - return G5P_REFUSE(cmd_buffer, "LOAD of colour attachment %u of %u (one target is built)", t, n); + if (!compact && (n != 1 || t)) + return G5P_REFUSE(cmd_buffer, "LOAD of colour attachment %u of %u (one target, or 5..8 compact targets, " + "are built)", t, n); load = true; } if (t < n && hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_CLEAR && @@ -841,6 +866,22 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd if (load && raw > 2 && !(raw >= 5 && raw <= 11)) return G5P_REFUSE(cmd_buffer, "integer boundary qualification supports CLEAR only"); const struct pvr_g5p_programs *load_program = raw && (raw <= 2 || (raw >= 5 && raw <= 11)) ? p->raw_load[raw - 1] : p; + if (compact && load) { + /* the compact LOAD of every target (lazy, as the integer variants) */ + const unsigned slot = v - 3; /* 0..3 BGRA8, 4..7 RGBA8 */ + struct pvr_g5p_programs *programs = &device->g5p->programs; + VkResult lr = VK_SUCCESS; + simple_mtx_lock(&device->g5p->clear_lock); + if (!programs->mrt_load[slot]) { + programs->mrt_load[slot] = vk_zalloc(&device->vk.alloc, sizeof(*programs), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); + lr = programs->mrt_load[slot] ? g5p_upload_load(device, programs->mrt_load[slot], 12 + slot) + : vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + } + load_program = programs->mrt_load[slot]; + simple_mtx_unlock(&device->g5p->clear_lock); + if (lr != VK_SUCCESS) + return lr; + } if (load && !load_program->load_usc) return G5P_REFUSE(cmd_buffer, "LOAD of a colour attachment%s: the load program: %s", partial ? " (a partial render area)" : "", load_program->load_refusal); @@ -878,7 +919,7 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd uint32_t pbe_words[G5P_MRT_PBE_STRIDE * G5P_MRT_MAX]; /* each target's source: output half 4t (8-bit RGBA, two dwords), 8t (raw32, four words) */ const unsigned pbe_dwords = g5p_mrt_pbe_block_halves(pbe_words, nt, (const uint64_t(*)[3])pbe, - raw ? 2 * raw_words : 4); + raw ? 2 * raw_words : compact ? 2 : 4); VkResult result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, clear, 16 * nt, &floats); if (result != VK_SUCCESS) return result; @@ -911,7 +952,8 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd /* PDS_BGRND0..3: the clear kick, the clear DMA, its data; sizes */ uint64_t bgnd[4] = { 0 }; if (load) { - result = g5p_load_background(cmd_buffer, load_state, g->width, g->height, bgnd, raw); + result = g5p_load_background(cmd_buffer, (const uint64_t(*)[4])load_state, g->width, g->height, bgnd, + load_program); if (result != VK_SUCCESS) return result; } else { diff --git a/src/imagination/vulkan/pvr_arch_g5p_blend.c b/src/imagination/vulkan/pvr_arch_g5p_blend.c index d82f5e969c3..51756df0f7a 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_blend.c +++ b/src/imagination/vulkan/pvr_arch_g5p_blend.c @@ -42,8 +42,21 @@ PVR_PER_ARCH(g5p_blend_fs_key)(const struct vk_graphics_pipeline_state *state, u BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_CB_WRITE_MASKS) || BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_CB_COLOR_WRITE_ENABLES)) return "dynamic blend enables, equations, write masks or colour-write enables"; - if (targets > 4) - return "more than four colour blend targets"; + if (targets > 8) + return "more than eight colour blend targets"; + /* 5..8: the compact UNORM8 output writes every channel of + * every target unblended; blending and masks there need its tile-read + * merge, which is not built */ + if (targets > 4) { + for (unsigned i = 0; i < targets; i++) { + const struct vk_color_blend_attachment_state *a = &cb->attachments[i]; + const unsigned mask = (cb->color_write_enables & BITFIELD_BIT(i)) ? a->write_mask : 0; + if (a->blend_enable || mask != 15) + return "five to eight colour targets with blending or a partial write mask (compact MRT is " + "full-write and unblended)"; + } + return NULL; + } if (targets > 1) { bool special = false; unsigned union_mask = 0; diff --git a/src/imagination/vulkan/pvr_arch_g5p_dgl.c b/src/imagination/vulkan/pvr_arch_g5p_dgl.c index e732e4135c9..10d9d90b154 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_dgl.c +++ b/src/imagination/vulkan/pvr_arch_g5p_dgl.c @@ -19,7 +19,7 @@ pvr_g5p_dgl_output_words(uint8_t *output, size_t capacity, const struct pvr_g5p_ { uint8_t code[10 * G5P_USC_MAX]; size_t size = 0; - if (!output || !sources || write_mask > 0xf || (words != 1 && words != 2 && words != 4 && words != 8) || (words != 4 && write_mask != 0xf)) + if (!output || !sources || write_mask > 0xf || words < 1 || words > 8 || words == 3 || (words != 4 && write_mask != 0xf)) return 0; if (write_mask != 0xf) { /* RGBA32 tile storage is four words. Preserve unmasked raw channels; diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 7c7dd55b093..e25a54a1e2e 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -507,6 +507,10 @@ g5p_check_state(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *c /* three colour targets compile as four: the fourth (the pass's null target) is written zeros */ if (colors == 3) key->color_targets = 4; + /* five to eight: one packed UNORM8 dword per target (pvr_arch_g5p_mrt_pipeline_check() admits RGBA8 or + * BGRA8, one format; pvr_arch_g5p_blend_fs_key() full-write and unblended) */ + if (colors > 4) + key->color_output = cf == VK_FORMAT_B8G8R8A8_UNORM ? PCO_G5P_COLOR_PACKED_BGRA8 : PCO_G5P_COLOR_PACKED_RGBA8; /* 32-bit integer targets: the raw32 output (four raw words, pvr_g5p_raw32_kind()); the PBE stores the * target's own channels from them */ const unsigned raw32 = colors ? pvr_g5p_raw32_kind(cf) : 0; diff --git a/src/imagination/vulkan/pvr_arch_g5p_mrt.c b/src/imagination/vulkan/pvr_arch_g5p_mrt.c index f51e5e60b73..bf6d34ca220 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_mrt.c +++ b/src/imagination/vulkan/pvr_arch_g5p_mrt.c @@ -132,10 +132,22 @@ PVR_PER_ARCH(g5p_mrt_pass_check)(struct pvr_cmd_buffer *cmd_buffer, va[t] = g5p_mrt_target_va(iview[t], &bytes[t]); } - /* The background programs clear or leave every target; a LOAD would need - * an N-target LOAD background (none exists) */ + /* 5..8 compact targets: the compact LOAD background loads every target, so a pass mixing CLEAR and LOAD + * would load the cleared ones */ + if (n > 4) { + bool has_clear = false, has_load = false; + for (unsigned i = 0; i < hw_render->color_init_count; i++) { + has_clear |= hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_CLEAR; + has_load |= hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD; + } + if (has_clear && has_load) + return G5P_REFUSE(cmd_buffer, "%u colour targets mixing CLEAR and LOAD (the compact background loads or " + "clears every target)", n); + } + /* The background programs clear or leave every target; a LOAD of 2..4 targets would need an N-target LOAD + * background (none exists; 5..8 have the compact one) */ for (unsigned i = 0; i < hw_render->color_init_count; i++) - if (hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD) + if (n <= 4 && hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD) return G5P_REFUSE(cmd_buffer, "LOAD of a colour attachment with %u colour targets (no N-target LOAD " "background)", n); diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index e0a7131023b..4cccf838d57 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -59,8 +59,9 @@ struct pvr_renderpass_hwsetup_render; struct pvr_sub_cmd_gfx; struct pvr_suballoc_bo; -/* Colour-target counts with programs: 1, 2 and 4, the counts with an Apple witness (others are refused) */ -#define PVR_G5P_MRT_VARIANTS 3 +/* Colour-target counts with programs: 1, 2 and 4 (Apple's packed-f16 pass), then 5..8 compact UNORM8 + * with a BGRA8 clear (variants 3..6) and an RGBA8 clear (7..10). 3 runs as 4. */ +#define PVR_G5P_MRT_VARIANTS 11 /* * The device-level programs every G5P render uses, uploaded once per @@ -108,11 +109,13 @@ struct pvr_g5p_programs { unsigned load_temps, load_common_dwords, load_uploads; struct { unsigned kind; /* PCO_G5P_UPLOAD_IMAGE, _SAMPLER or _IMAGE_SIZE */ - unsigned dwords, common; - } load_upload[3]; + unsigned dwords, common, binding; /* binding: the target an IMAGE upload reads (5..8 targets) */ + } load_upload[32]; char load_refusal[128]; /* Lazy integer variants: float device allocation order remains identical. */ struct pvr_g5p_programs *raw_load[11]; /* scalar R8 uses10/11, separate fromRGBA8 */ + /* Lazy compact LOAD programs: 5..8 targets, BGRA8 [0..3] and RGBA8 [4..7] */ + struct pvr_g5p_programs *mrt_load[8]; /* raw clear programs by total output words: 2, 4, 8 (two 32-bit integer targets) */ struct pvr_suballoc_bo *raw_clear_usc[4]; /*2/4/8 words, then scalar1 */ /* raw end-of-tile programs by target count: 1, 2 */ @@ -335,6 +338,9 @@ bool pvr_g5p_compile_load_rgb10a2ui(struct pco_g5p_program *out); bool pvr_g5p_compile_load_rgba8ui(struct pco_g5p_program *out); bool pvr_g5p_compile_load_rgba16ui(struct pco_g5p_program *out); bool pvr_g5p_compile_load_raw(struct pco_g5p_program *out, bool is_signed); +/* The compact LOAD background of 5..8 UNORM8 targets: texelFetch of every + * target (set 0, binding t), packed as the compact output. */ +bool pvr_g5p_compile_load_mrt(struct pco_g5p_program *out, unsigned targets, bool bgra); /* The clear quad's stages, built with nir_builder (pvr_g5p_nir.c): the * vertex stage passes its vec4 position at location 0 through, the * fragment stage writes the push constant's colour (bytes 0..15) to colour diff --git a/src/imagination/vulkan/pvr_g5p_formats.c b/src/imagination/vulkan/pvr_g5p_formats.c index 559ca7054c1..98426911246 100644 --- a/src/imagination/vulkan/pvr_g5p_formats.c +++ b/src/imagination/vulkan/pvr_g5p_formats.c @@ -43,6 +43,14 @@ pvr_g5p_format_tex(VkFormat format, uint32_t *key, bool *srgb) return true; } +/* The physical one-dword store of already packed bytes (5..8 compact + * targets): R32UI's PBE key, no format admission of its own. */ +uint32_t +pvr_g5p_packed_word_pbe_key(void) +{ + return pvr_g5p_format_find(VK_FORMAT_R32_UINT)->key; +} + bool pvr_g5p_format_pbe(VkFormat format, uint32_t *key, bool *srgb) { diff --git a/src/imagination/vulkan/pvr_g5p_formats.h b/src/imagination/vulkan/pvr_g5p_formats.h index 5fcaa3a389f..9beb01175f9 100644 --- a/src/imagination/vulkan/pvr_g5p_formats.h +++ b/src/imagination/vulkan/pvr_g5p_formats.h @@ -32,6 +32,8 @@ bool pvr_g5p_format_tex(VkFormat format, uint32_t *key, bool *srgb); /* The PBE key and sRGB bit of a colour target format, or false: the A9 * doesn't render it (the pipeline and the render job refuse it by name) */ bool pvr_g5p_format_pbe(VkFormat format, uint32_t *key, bool *srgb); +/* Physical one-dword stores for already packed bytes; no public format admission. */ +uint32_t pvr_g5p_packed_word_pbe_key(void); /* pvr_get_image_format_features2(): the features of a colour format on * the A9, in place of upstream's (flags) */ diff --git a/src/imagination/vulkan/pvr_g5p_mrt.h b/src/imagination/vulkan/pvr_g5p_mrt.h index 1e1b8120680..ce241bc6d59 100644 --- a/src/imagination/vulkan/pvr_g5p_mrt.h +++ b/src/imagination/vulkan/pvr_g5p_mrt.h @@ -26,8 +26,9 @@ struct pvr_cmd_buffer; struct pvr_device; struct pvr_renderpass_hwsetup_render; -/* the most colour targets a G5P pipeline or pass has (Apple's 4-target pass) */ -#define PVR_G5P_MRT_MAX_COLORS 4 +/* the most colour targets a G5P pipeline or pass has: Apple's 4-target pass, then 5..8 compact UNORM8 + * targets (one format, full write mask, no blending) */ +#define PVR_G5P_MRT_MAX_COLORS 8 /* The formats of Apple's 2- and 4-target pass */ static inline bool diff --git a/src/imagination/vulkan/pvr_g5p_nir.c b/src/imagination/vulkan/pvr_g5p_nir.c index af0c8de972d..68667822411 100644 --- a/src/imagination/vulkan/pvr_g5p_nir.c +++ b/src/imagination/vulkan/pvr_g5p_nir.c @@ -965,6 +965,42 @@ pvr_g5p_compile_load_raw(struct pco_g5p_program *out, bool is_signed) return g5p_compile_load_type(out, is_signed ? 2 : 1); } +bool +pvr_g5p_compile_load_mrt(struct pco_g5p_program *out, unsigned targets, bool bgra) +{ + static util_once_flag once = UTIL_ONCE_FLAG_INIT; + util_call_once(&once, g5p_init_options); + memset(out, 0, sizeof(*out)); + glsl_type_singleton_init_or_ref(); + + nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, &g5p_options, "g5p-load-mrt"); + nir_def *xy = nir_f2i32(&b, nir_trim_vector(&b, nir_load_frag_coord(&b), 2)); + for (unsigned t = 0; t < targets; t++) { + nir_variable *prev = nir_variable_create(b.shader, nir_var_uniform, + glsl_sampler_type(GLSL_SAMPLER_DIM_2D, false, false, GLSL_TYPE_FLOAT), + "prev"); + prev->data.descriptor_set = 0; + prev->data.binding = t; + nir_variable *color = nir_variable_create(b.shader, nir_var_shader_out, glsl_vec4_type(), "color"); + color->data.location = FRAG_RESULT_DATA0 + t; + nir_def *texel = nir_txf(&b, xy, .texture_deref = nir_build_deref_var(&b, prev), .lod = nir_imm_int(&b, 0), + .dim = GLSL_SAMPLER_DIM_2D, .dest_type = nir_type_float32); + nir_store_var(&b, color, texel, 0xf); + } + bool sample_mask = false; + nir_shader *nir = graphics_lower(g5p_nir_prepare(b.shader), targets, &sample_mask, false, NULL); + struct pco_g5p_fs_key key = PCO_G5P_FS_KEY_DEFAULT; + key.last_read_groups = true; + key.color_targets = targets; + key.color_output = bgra ? PCO_G5P_COLOR_PACKED_BGRA8 : PCO_G5P_COLOR_PACKED_RGBA8; + bool ok = nir && pco_g5p_compile_general_link(nir, true, NULL, &key, NULL, out); + if (!nir) + snprintf(out->refusal, sizeof(out->refusal), "the compact load program's NIR"); + ralloc_free(nir); + glsl_type_singleton_decref(); + return ok; +} + nir_shader * pvr_g5p_clear_nir(bool vertex) { diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index c0d6114d478..04bb1c47394 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -730,9 +730,9 @@ static bool pvr_physical_device_get_properties( /* Fragment Shader Limits */ .maxFragmentInputComponents = PVR_HAS_FEATURE(dev_info, apple_g9p) ? 120U : 64U, - .maxFragmentOutputAttachments = 4U, + .maxFragmentOutputAttachments = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8U : 4U, /* G5P: 5..8 compact UNORM8 targets */ .maxFragmentDualSrcAttachments = 0U, /* Requires dualSrcBlend */ - .maxFragmentCombinedOutputResources = 4U, + .maxFragmentCombinedOutputResources = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8U : 4U, /* Compute Shader Limits */ .maxComputeSharedMemorySize = 16U * 1024U, @@ -795,7 +795,7 @@ static bool pvr_physical_device_get_properties( .framebufferDepthSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, .framebufferStencilSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, .framebufferNoAttachmentsSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, - .maxColorAttachments = 4U, + .maxColorAttachments = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8U : 4U, /* G5P: 5..8 compact UNORM8 targets (pvr_g5p_mrt.h) */ .sampledImageColorSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, .sampledImageIntegerSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, .sampledImageDepthSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, From 4aacf1e82e8428312b6b4c943e0155fd1054f860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:08 +0100 Subject: [PATCH 033/109] pvr/g5p: 8192-texel images, framebuffers and viewports WebGPU core wants maxTextureDimension1D/2D 8192. - The kernel takes render targets of 1 to 8192 per side. - The texture and PBE states carry width - 1 and height - 1 in 14-bit fields. - The A9's device info already allows an 8192 render and a 16384 sampled extent; only the core limits said 4096. - Apple's feature-set tables give the A9 (Apple3) 16384 for 2D textures. maxImageDimension1D/2D/Cube, maxFramebufferWidth/Height and maxViewportDimensions go from 4096 to 8192 on the A9 (A10 unchanged), with viewportBoundsRange [-16384, 16383]. Tested on the A9 before the powervr interface: 8192x64 and 64x8192 renders sampled back, and an 8192x8192 R8 render. Assisted-by: LLM --- src/imagination/vulkan/pvr_physical_device.c | 21 +++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index 04bb1c47394..429dc7498fc 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -671,10 +671,12 @@ static bool pvr_physical_device_get_properties( .deviceType = VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU, /* deviceName and pipelineCacheUUID are filled below .*/ - .maxImageDimension1D = 4096U, - .maxImageDimension2D = 4096U, + /* G5P: 8192, the Orchard kernel's render-target bound (G5P_RU_MAX_DIM) and inside the A9's + * 14-bit width/height fields of the texture and PBE states (pvr_g5p_tex_encode.h, g5p_pbe.h) */ + .maxImageDimension1D = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, + .maxImageDimension2D = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, .maxImageDimension3D = pvr_get_texture_extent_max_z(dev_info), - .maxImageDimensionCube = 4096U, + .maxImageDimensionCube = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, .maxImageArrayLayers = rogue_get_render_size_max_z(dev_info), .maxTexelBufferElements = 64U * 1024U, .maxUniformBufferRange = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 64U * 1024U : 16U * 1024U, @@ -763,10 +765,11 @@ static bool pvr_physical_device_get_properties( .maxSamplerAnisotropy = 16.0f, /* Requires samplerAnisotropy */ .maxViewports = 1U, /* Requires multiViewport */ - .maxViewportDimensions[0] = 4096U, - .maxViewportDimensions[1] = 4096U, - .viewportBoundsRange[0] = -8192.0f, - .viewportBoundsRange[1] = 8191.0f, + .maxViewportDimensions[0] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, + .maxViewportDimensions[1] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, + /* at least [-2 * maxViewportDimensions, 2 * maxViewportDimensions - 1] */ + .viewportBoundsRange[0] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? -16384.0f : -8192.0f, + .viewportBoundsRange[1] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 16383.0f : 8191.0f, .viewportSubPixelBits = 0U, .minMemoryMapAlignment = pdevice->ws->page_size, @@ -786,8 +789,8 @@ static bool pvr_physical_device_get_properties( .maxInterpolationOffset = 0.5f, .subPixelInterpolationOffsetBits = 4U, - .maxFramebufferWidth = 4096U, - .maxFramebufferHeight = 4096U, + .maxFramebufferWidth = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, + .maxFramebufferHeight = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, .maxFramebufferLayers = 256U, /* Note: update nir_shader_compiler_options.max_samples when changing this. */ From a78ab703a89613167313e170de98aa2a666e0bab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:08 +0100 Subject: [PATCH 034/109] pvr/g5p: ASTC LDR, all 14 block sizes WebGPU's core gate wants BC, or ETC2 and ASTC LDR; the A9 had ETC2 only. Apple's A9 driver samples ASTC LDR in every size (texture format codes 0x60-0x6d, sRGB as gamma 3). - 28 format rows (UNORM and sRGB), 16-byte blocks: sampled, filtered, copy and blit sources. pvr_get_format() admits them on the A9 only; Rogue's table, which the A10 shares, has none. - Layout: a block grid that is not 4x4 is a power of two per side per level, as Apple's texture layout rounds ceil(width / bw) up per level and the Morton order addresses a 7-block side as 8. These images take their level offsets from the A9 layout; 4x4 grids keep theirs. - Copies are block copies through the CPU twiddle, as for ETC2. The CPU blit decodes a block of up to 12x12 texels with util's ASTC decoder. textureCompressionASTC_LDR is set on the A9. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_g5p_layout.c | 29 ++++++++--- src/imagination/vulkan/pvr_formats.c | 16 ++++++ src/imagination/vulkan/pvr_g5p_format_table.h | 52 +++++++++++++++---- src/imagination/vulkan/pvr_g5p_image.c | 23 +++++++- src/imagination/vulkan/pvr_physical_device.c | 3 +- 5 files changed, 106 insertions(+), 17 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_g5p_layout.c b/src/imagination/vulkan/pvr_arch_g5p_layout.c index 884b3f71281..d333074bc16 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_layout.c +++ b/src/imagination/vulkan/pvr_arch_g5p_layout.c @@ -31,6 +31,7 @@ #include "util/format/u_format.h" #include "util/log.h" #include "util/texcompress_etc.h" +#include "util/texcompress_astc.h" #include "util/u_math.h" #include "util/os_time.h" #include @@ -41,6 +42,9 @@ #include "pvr_g5p_ds.h" #include "pvr_g9p.h" #include "pvr_image.h" + +/* the most texels one compressed block holds: ASTC 12x12 (ETC2/EAC 16) */ +#define G5P_BLOCK_TEXELS 144 #include "pvr_job_transfer.h" @@ -288,9 +292,22 @@ g5p_etc2_decoded(enum pipe_format p) } /* A 4x4 block as 16 RGBA floats, row by row */ +/* The most texels of one block: ASTC 12x12 */ static void -g5p_decode_block(enum pipe_format format, const uint8_t *p, unsigned bpp, float block[16][4]) +g5p_decode_block(enum pipe_format format, const uint8_t *p, unsigned bpp, float block[G5P_BLOCK_TEXELS][4]) { + if (util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_ASTC) { + /* util's ASTC LDR decoder: RGBA8 (sRGB-encoded for the sRGB formats), + * one block of bw x bh texels, then the 8-bit format's unpack as for + * ETC2 below */ + const unsigned bw = util_format_get_blockwidth(format), bh = util_format_get_blockheight(format); + uint8_t texels[G5P_BLOCK_TEXELS * 4]; + + _mesa_unpack_astc_2d_ldr(texels, 4 * bw, p, bpp, bw, bh, format); + util_format_unpack_rgba(util_format_is_srgb(format) ? PIPE_FORMAT_R8G8B8A8_SRGB : PIPE_FORMAT_R8G8B8A8_UNORM, + block, texels, bw * bh); + return; + } const enum pipe_format dec = g5p_etc2_decoded(format); if (dec != PIPE_FORMAT_NONE) { @@ -311,7 +328,7 @@ g5p_decode_block(enum pipe_format format, const uint8_t *p, unsigned bpp, float * the last block decoded is kept */ static bool g5p_block_texel(const struct g5p_surface *s, unsigned x, unsigned y, float out[4], const uint8_t **last, - float block[16][4]) + float block[G5P_BLOCK_TEXELS][4]) { const unsigned bw = util_format_get_blockwidth(s->format), bh = util_format_get_blockheight(s->format); struct g5p_surface b = *s; @@ -325,7 +342,7 @@ g5p_block_texel(const struct g5p_surface *s, unsigned x, unsigned y, float out[4 const uint8_t *p = g5p_element(&b, x / bw, y / bh); - assert(bw == 4 && bh == 4); + assert(bw * bh <= G5P_BLOCK_TEXELS); if (!p) return false; if (p != *last) { @@ -356,7 +373,7 @@ g5p_normalize_axis(int32_t *offset, uint32_t *extent, bool *flip) * util_format_unpack_rgba() looks it up (a call_once) on every call. */ static bool g5p_texel(struct g5p_surface *s, const struct util_format_unpack_description *unpack, unsigned x, unsigned y, - float out[4], const uint8_t **last, float block[16][4]) + float out[4], const uint8_t **last, float block[G5P_BLOCK_TEXELS][4]) { if (s->blocks) return g5p_block_texel(s, x, y, out, last, block); @@ -377,7 +394,7 @@ g5p_texel(struct g5p_surface *s, const struct util_format_unpack_description *un static bool g5p_linear_sample(struct g5p_surface *src, const struct util_format_unpack_description *unpack, const struct pvr_rect_mapping *m, unsigned x, unsigned y, float out[4], - const uint8_t **last, float block[16][4]) + const uint8_t **last, float block[G5P_BLOCK_TEXELS][4]) { const double sw = m->src_rect.extent.width, sh = m->src_rect.extent.height; const double dw = m->dst_rect.extent.width, dh = m->dst_rect.extent.height; @@ -591,7 +608,7 @@ g5p_copy(const struct pvr_rect_mapping *mapping, struct g5p_surface *src, struct const unsigned dw = m->dst_rect.extent.width, dh = m->dst_rect.extent.height; const unsigned sw = m->src_rect.extent.width, sh = m->src_rect.extent.height; const uint8_t *last = NULL; - float block[16][4]; + float block[G5P_BLOCK_TEXELS][4]; if (!dw || !dh || !sw || !sh) return true; diff --git a/src/imagination/vulkan/pvr_formats.c b/src/imagination/vulkan/pvr_formats.c index 5ac7f41daec..03b44099529 100644 --- a/src/imagination/vulkan/pvr_formats.c +++ b/src/imagination/vulkan/pvr_formats.c @@ -32,6 +32,7 @@ #include "hwdef/rogue_hw_utils.h" #include "pvr_g5p_ds_hooks.h" # include "pvr_g5p_vertex.h" +#include "pvr_g5p_format_table.h" #include "pvr_common.h" #include "pvr_csb.h" #include "pvr_device.h" @@ -239,6 +240,21 @@ pvr_get_format(struct pvr_physical_device *pdevice, VkFormat vk_format) pdevice->formats.formats[format].bind != 0) { return &pdevice->formats.formats[format]; } + /* G5P: ASTC LDR exists only in the A9's own table (Rogue's table, which + * G9P shares, has none); the G5P hooks take its features and texture + * state from that row (pvr_g5p_format_table.h) */ + if (PVR_HAS_FEATURE(&pdevice->dev_info, apple_g5p) && + vk_format_description(vk_format)->layout == UTIL_FORMAT_LAYOUT_ASTC && + pvr_g5p_format_find(vk_format)) { + static const struct pvr_format g5p_astc = { + /* Rogue's texture-format codes are not used on G5P (its own key is) */ + .tex_format = ~0u, + .depth_tex_format = ~0u, + .stencil_tex_format = ~0u, + .bind = PVR_BIND_SAMPLER_VIEW, + }; + return &g5p_astc; + } mesa_logd("Format %s(%d) not supported\n", vk_Format_to_str(vk_format), diff --git a/src/imagination/vulkan/pvr_g5p_format_table.h b/src/imagination/vulkan/pvr_g5p_format_table.h index a4d4e57ffe4..91fd603396f 100644 --- a/src/imagination/vulkan/pvr_g5p_format_table.h +++ b/src/imagination/vulkan/pvr_g5p_format_table.h @@ -16,13 +16,13 @@ * 0x21 32, 0x31 32_32, 0x38 32_32_32_32, 0x26 10_10_10_2, 0x25 11_11_10, * 0x27 9_9_9_e5, 0x0b 5_6_5, 0x0d 1_5_5_5, 0x0c 4_4_4_4; block compressed * 0x58 ETC2 RGB8, 0x5a ETC2 RGB8A1, 0x59 ETC2 RGBA8 (EAC alpha), 0x5b EAC R11, - * 0x5c EAC RG11. Selectors: 0-3 a component (component 0 is the lowest + * 0x5c EAC RG11, 0x60..0x6d ASTC LDR 4x4..12x12. Selectors: 0-3 a component (component 0 is the lowest * field of a packed format), 4 one, 5 zero. * * What a row advertises is what has run on the A9 through this driver: every - * row's rungs of the formats hardware test (src/imagination/g5p/tests/formats) - * passed on the device, every pixel exact (2026-11-04: 118 rungs of the rows - * advertised then, and 39 rungs of the rows decoded but not yet run, the + * row's cases of the formats hardware test (src/imagination/g5p/tests/formats) + * passed on the device, every pixel exact (2026-11-04: 118 cases of the rows + * advertised then, and 39 cases of the rows decoded but not yet run, the * probes, promoted since). Before that: * - sampling: the texture state is format-independent (lane b's * pvr_arch_g5p_tex.c); each key below was sampled exactly on the device by @@ -36,7 +36,7 @@ * packs R8, RG8, R16F, RG16F and RG11B10F from the two-word program too: * exact on the device. Blending is the fragment program's, on the tile's * f16 values before the PBE, for every rendered row; - * - probes (for a device rung only): rows decoded or + * - probes (for a device case only): rows decoded or * derived but not yet run, not advertised by default. None at present. */ @@ -62,7 +62,7 @@ enum pvr_g5p_fmt_flags { PVR_G5P_FMT_FILTER = 1u << 1, /* linear filtering */ PVR_G5P_FMT_RENDER = 1u << 2, /* colour attachment: the two-word program + this PBE key */ PVR_G5P_FMT_BLEND = 1u << 3, /* blending into it (the fragment program's) */ - PVR_G5P_FMT_COMPRESSED = 1u << 4, /* 4x4 blocks, twiddled only */ + PVR_G5P_FMT_COMPRESSED = 1u << 4, /* blocks (4x4 ETC2/EAC, 4x4..12x12 ASTC), twiddled only */ PVR_G5P_FMT_INTEGER = 1u << 5, /* integer texel: texelFetch/unfiltered */ /* not advertised (none at present) */ PVR_G5P_FMT_PROBE_SAMPLED = 1u << 8, /* the sampled bits are a probe */ @@ -72,7 +72,7 @@ enum pvr_g5p_fmt_flags { /* Where a row's key and its advertised use come from */ enum pvr_g5p_fmt_evidence { - PVR_G5P_FMT_EV_G5P1B, /* device-proven on g5p-1b (tex-blend 22/22, formats 118 + 39 rungs) */ + PVR_G5P_FMT_EV_G5P1B, /* device-proven on g5p-1b (tex-blend 22/22, formats 118 + 39 cases) */ PVR_G5P_FMT_EV_ICD, /* captured key, device-proven by our ICD (gles3tex, textypes) */ PVR_G5P_FMT_EV_RULE, /* derived from the decoded code rule, not captured */ PVR_G5P_FMT_EV_CAPTURE, /* captured key, never run on the device */ @@ -81,7 +81,7 @@ enum pvr_g5p_fmt_evidence { struct pvr_g5p_format { VkFormat format; uint32_t key; - uint8_t cpp; /* bytes per texel, or per 4x4 block */ + uint8_t cpp; /* bytes per texel, or per block */ bool srgb; uint16_t flags; uint8_t evidence; @@ -163,7 +163,7 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { G5P_FMT(B4G4R4A4_UNORM_PACK16, 2, false, G5P_SF, G5P1B, 0x0c, 1, 2, 3, 0), /* integer (g3b-sample-b t5-t18, fmt-rgba8ui; textypes int-fetch and * int-sample on the device). RGBA32 UINT/SINT clear/draw/LOAD/masks - * passed the integer-r4 core/compat device rungs. Narrow integer render + * passed the integer-r4 core/compat device cases. Narrow integer render * targets remain absent or probe-only; integer blending is not exposed. */ G5P_FMT(R8_UINT, 1, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x100, G5P_SEL_R001), G5P_FMT(R8G8_UINT, 2, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x10a, G5P_SEL_RG01), @@ -199,6 +199,40 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { G5P_FMT(EAC_R11_SNORM_BLOCK, 8, false, G5P_CT, G5P1B, 0xdb, G5P_SEL_R001), G5P_FMT(EAC_R11G11_UNORM_BLOCK, 16, false, G5P_CT, G5P1B, 0x5c, G5P_SEL_RG01), G5P_FMT(EAC_R11G11_SNORM_BLOCK, 16, false, G5P_CT, G5P1B, 0xdc, G5P_SEL_RG01), + /* ASTC LDR, all 14 block sizes, 16 bytes per block: AGXMetalA9's texFormat + * rows (chooseTextureFormatOBJC 0x207dc25b4): codes 0x60..0x6d in Vulkan's + * size order, sRGB as gamma 3 (q1 bit 45); -[AGXA9FamilyDevice + * isASTCPixelFormatsSupported] returns 1 on the A9. + * Twiddled over power-of-two block grids + * (pvr_g5p_image.c); copies and blits decode with util's ASTC decoder. */ + G5P_FMT(ASTC_4x4_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x60, G5P_SEL_RGBA), + G5P_FMT(ASTC_4x4_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x60, G5P_SEL_RGBA), + G5P_FMT(ASTC_5x4_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x61, G5P_SEL_RGBA), + G5P_FMT(ASTC_5x4_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x61, G5P_SEL_RGBA), + G5P_FMT(ASTC_5x5_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x62, G5P_SEL_RGBA), + G5P_FMT(ASTC_5x5_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x62, G5P_SEL_RGBA), + G5P_FMT(ASTC_6x5_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x63, G5P_SEL_RGBA), + G5P_FMT(ASTC_6x5_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x63, G5P_SEL_RGBA), + G5P_FMT(ASTC_6x6_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x64, G5P_SEL_RGBA), + G5P_FMT(ASTC_6x6_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x64, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x5_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x65, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x5_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x65, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x6_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x66, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x6_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x66, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x8_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x67, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x8_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x67, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x5_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x68, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x5_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x68, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x6_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x69, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x6_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x69, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x8_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x6a, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x8_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x6a, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x10_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x6b, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x10_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x6b, G5P_SEL_RGBA), + G5P_FMT(ASTC_12x10_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x6c, G5P_SEL_RGBA), + G5P_FMT(ASTC_12x10_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x6c, G5P_SEL_RGBA), + G5P_FMT(ASTC_12x12_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x6d, G5P_SEL_RGBA), + G5P_FMT(ASTC_12x12_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x6d, G5P_SEL_RGBA), }; #undef G5P_FMT diff --git a/src/imagination/vulkan/pvr_g5p_image.c b/src/imagination/vulkan/pvr_g5p_image.c index 0fd3452a1c6..bfea4742ba2 100644 --- a/src/imagination/vulkan/pvr_g5p_image.c +++ b/src/imagination/vulkan/pvr_g5p_image.c @@ -89,10 +89,31 @@ pvr_g5p_image_layout(struct pvr_image *image, unsigned stride_align) const uint32_t chain = image->vk.mip_levels > 1 ? util_logbase2(MAX2(pw, ph)) + 1 : 1; VkDeviceSize size = 0, own = 0; + /* Blocks other than 4x4 (ASTC 5x4..12x12): each level's block grid is a + * power of two per side, pow2(ceil(level / block)), as AGXMetalA9's + * TextureGen2 lays them out (0x207dbfe4c: ceil(width / bw), then the + * next power of two, per plane level) and as the Morton order needs (a + * 7-block side is addressed as 8: g5p_twiddle2d()). Upstream's level + * offsets minify the block count instead, so these images take their + * offsets from here. 4x4 grids are already powers of two (or one block): + * ETC2/EAC and ASTC 4x4 keep upstream's offsets, which are the same. */ + const bool pow2_blocks = !is3d && vk_format_is_compressed(image->vk.format) && (bw != 4 || bh != 4); for (uint32_t l = 0; l < MAX2(chain, image->vk.mip_levels); l++) { - const uint32_t ew = DIV_ROUND_UP(u_minify(pw, l), bw), eh = DIV_ROUND_UP(u_minify(ph, l), bh); + uint32_t ew = DIV_ROUND_UP(u_minify(pw, l), bw), eh = DIV_ROUND_UP(u_minify(ph, l), bh); + if (pow2_blocks) { + ew = util_next_power_of_two(ew); + eh = util_next_power_of_two(eh); + } const VkDeviceSize level_size = (VkDeviceSize)cpp * ew * eh; + if (pow2_blocks && l < image->vk.mip_levels) { + struct pvr_mip_level *level = &plane->mip_levels[l]; + + level->pitch = cpp * ew; + level->height_pitch = eh; + level->size = level_size; + level->offset = own; + } if (is3d && l < image->vk.mip_levels) { /* a 3D image's levels are one slice's: upstream's 3DTWIDDLED levels hold every slice */ struct pvr_mip_level *level = &plane->mip_levels[l]; diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index 429dc7498fc..de78bbecc06 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -268,7 +268,8 @@ static void pvr_physical_device_get_supported_features( .multiViewport = false, .samplerAnisotropy = true, .textureCompressionETC2 = true, - .textureCompressionASTC_LDR = false, + /* G5P: all 28 ASTC LDR formats sampled, filtered and blit sources (pvr_g5p_format_table.h) */ + .textureCompressionASTC_LDR = PVR_HAS_FEATURE(dev_info, apple_g5p), .textureCompressionBC = false, .occlusionQueryPrecise = true, .pipelineStatisticsQuery = false, From 0b1dd95f9df755b5ef222dbf33ba250323e78419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:08 +0100 Subject: [PATCH 035/109] pvr/g5p: draw layer-0 array samples as 2D PPSSPP binds every texture and framebuffer through a 2D_ARRAY view and samples layer 0.0. The A9 keeps single-level 2D colour images linear, and it has no array state for a linear image, so such views had no texture state and the draws were refused. An array sample whose layer is a constant that clamps to 0 reads the view's base layer, so the compiler draws it as a 2D sample and marks its upload; the draw uploads the base layer's 2D state. An array sample at another layer is refused at its draw when the view has no array state. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 41 ++++++++++++++++++++- src/imagination/g5p/pco_g5p_general.h | 9 +++++ src/imagination/vulkan/pvr_arch_g5p_gfx.c | 6 ++- src/imagination/vulkan/pvr_arch_g5p_tex.c | 8 +++- src/imagination/vulkan/pvr_arch_g5p_vtex.c | 3 ++ src/imagination/vulkan/pvr_g5p_tex_encode.h | 31 ++++++++++++++++ 6 files changed, 94 insertions(+), 4 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 0b6f95a98aa..1302a5f1a0c 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -92,6 +92,7 @@ struct tex_bind { unsigned element, selement; /* array elements of the texture / sampler binding */ unsigned image, sampler; /* common dwords */ bool needs_sampler, needs_fetch_sampler; + unsigned image_flags; /* PCO_G5P_UPLOAD_IMAGE_AS_2D / _ARRAY: how its samples read the state */ }; struct range { @@ -102,6 +103,8 @@ struct range { }; enum { BLEND_NONE, BLEND_PREMULT, BLEND_STRAIGHT_OVER, BLEND_REPLACE, BLEND_GENERAL }; +/* nir_tex_instr.backend_flags: an array sample of layer 0 drawn as 2D (layer0_as_2d()) */ +#define G5P_TEX_LAYER0_AS_2D (1u << 30) /* The general blend epilogue (g5p_blend.c, hyprland-gpu.md §11.2) for * settings without a measured epilogue. Premultiplied, straight-over and @@ -2013,6 +2016,8 @@ static void tex_bindings(struct gctx *c, nir_tex_instr *t, struct tex_bind *b) sv->data.binding, element, selement}; b->needs_sampler = t->op != nir_texop_txf; b->needs_fetch_sampler = t->op == nir_texop_txf; + b->image_flags = (t->backend_flags & G5P_TEX_LAYER0_AS_2D) ? PCO_G5P_UPLOAD_IMAGE_AS_2D : + t->is_array ? PCO_G5P_UPLOAD_IMAGE_ARRAY : 0; } static struct tex_bind *tex_for(struct gctx *c, nir_tex_instr *t) @@ -2021,7 +2026,8 @@ static struct tex_bind *tex_for(struct gctx *c, nir_tex_instr *t) tex_bindings(c, t, &b); for (unsigned i = 0; i < c->ntex; i++) if (c->tex[i].set == b.set && c->tex[i].binding == b.binding && c->tex[i].sset == b.sset && - c->tex[i].sbinding == b.sbinding && c->tex[i].element == b.element && c->tex[i].selement == b.selement) + c->tex[i].sbinding == b.sbinding && c->tex[i].element == b.element && c->tex[i].selement == b.selement && + c->tex[i].image_flags == b.image_flags) { c->tex[i].needs_sampler |= b.needs_sampler; c->tex[i].needs_fetch_sampler |= b.needs_fetch_sampler; return &c->tex[i]; } @@ -2223,7 +2229,7 @@ static void plan_common(struct gctx *c) next = compact ? c->fetch_sampler_common + 2 * (qualified && fetch_sampler) : t->image + 10; if (t->image / 2 > (compact ? 120 : 28) || t->sampler / 2 > (compact ? 124 : 32)) refuse(c, "texture descriptors beyond the bounded qualified selectors"); - add_upload(c, PCO_G5P_UPLOAD_IMAGE, t->set, t->binding, t->element, 0, 8, t->image); + add_upload(c, PCO_G5P_UPLOAD_IMAGE, t->set, t->binding, t->element, t->image_flags, 8, t->image); if (!qualified || t->needs_sampler) add_upload(c, PCO_G5P_UPLOAD_SAMPLER, t->sset, t->sbinding, t->selement, 32, 2, t->sampler); } @@ -6145,6 +6151,36 @@ static bool hoist_branch_derivatives(nir_shader *nir) return nir_progress(progress, impl, nir_metadata_control_flow); } +/* An array sample at a constant layer that clamps to 0 reads the view's + * base layer: Vulkan's layer is clamp(RNE(z), 0, layers - 1). It is drawn + * as a 2D sample of that layer (the upload's PCO_G5P_UPLOAD_IMAGE_AS_2D), + * which also samples a single-layer array view of a linear image: the A9 has + * no array state for a linear image, and its array sample needs one even + * for one layer (pvr_arch_g5p_tex.c). PPSSPP binds every texture and + * framebuffer as a 2D_ARRAY view and samples layer 0.0. */ +static bool layer0_as_2d(nir_builder *b, nir_tex_instr *t, void *data) +{ + if (!t->is_array || t->sampler_dim != GLSL_SAMPLER_DIM_2D || t->coord_components != 3 || + (t->op != nir_texop_tex && t->op != nir_texop_txb && t->op != nir_texop_txl && + t->op != nir_texop_txd && t->op != nir_texop_txf && t->op != nir_texop_tg4)) + return false; + const int ci = nir_tex_instr_src_index(t, nir_tex_src_coord); + if (ci < 0) + return false; + nir_scalar z = nir_scalar_chase_movs(nir_get_scalar(t->src[ci].src.ssa, 2)); + if (!nir_scalar_is_const(z)) + return false; + if (t->op == nir_texop_txf ? nir_scalar_as_int(z) != 0 : !(nir_scalar_as_float(z) <= 0.5)) + return false; + b->cursor = nir_before_instr(&t->instr); + nir_def *c = t->src[ci].src.ssa; + nir_src_rewrite(&t->src[ci].src, nir_channels(b, c, 0x3)); + t->coord_components = 2; + t->is_array = false; + t->backend_flags |= G5P_TEX_LAYER0_AS_2D; + return true; +} + static bool lower(nir_shader *nir, bool preserve_cf) { bool progress; @@ -6168,6 +6204,7 @@ static bool lower(nir_shader *nir, bool preserve_cf) (nir->info.stage == MESA_SHADER_VERTEX && g5p_vtex_size_lod_probe()), }; NIR_PASS(_, nir, nir_lower_tex, &txp); + NIR_PASS(_, nir, nir_shader_tex_pass, layer0_as_2d, nir_metadata_control_flow, NULL); pvr_g5p_tbo_lower(nir); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_push_const, nir_address_format_32bit_offset); NIR_PASS(_, nir, nir_lower_flrp, 32, false); diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index 51992c327eb..9b5cba8604b 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -138,6 +138,15 @@ enum pco_g5p_upload_kind { PCO_G5P_UPLOAD_BLEND_CONSTANTS, /* 4 dwords: the dynamic blend constants (f32) */ PCO_G5P_UPLOAD_FETCH_SAMPLER, /* two generated samplerless-read policy words */ }; +/* PCO_G5P_UPLOAD_IMAGE's offset (the state is always descriptor +0): how the + * program samples it. AS_2D: an array sample at the constant layer 0, drawn + * as a 2D sample, so the driver uploads the view's base layer as a 2D state + * (a 2D_ARRAY state converted, pvr_g5p_tex_state_as_2d()); ARRAY: an array + * sample, which needs a 2D_ARRAY state (the A9's array sample reads its + * layer through the array descriptor even for one layer). */ +#define PCO_G5P_UPLOAD_IMAGE_AS_2D 1u +#define PCO_G5P_UPLOAD_IMAGE_ARRAY 2u + /* Varyings (docs/gpu/experiments/varyings.md). Apple's A9 compiler takes at * most 60 interpolated components ("Number of varying components(64) diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index e25a54a1e2e..25036bf2f6e 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -1530,12 +1530,16 @@ fs_uploads(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline *g, continue; } switch (u->kind) { - case PCO_G5P_UPLOAD_IMAGE: + case PCO_G5P_UPLOAD_IMAGE: { if (!pvr_g5p_tex_state_valid(desc)) return G5P_REFUSE(cmd_buffer, "set %u binding %u: a view without A9 texture state", u->set, u->binding); memcpy(out, desc, 4 * u->dwords); + const char *why = u->dwords == 8 ? pvr_g5p_tex_state_for(out, u->offset) : NULL; + if (why) + return G5P_REFUSE(cmd_buffer, "set %u binding %u: %s", u->set, u->binding, why); break; + } case PCO_G5P_UPLOAD_SAMPLER: /* a combined descriptor's sampler is at +32 (the upload's offset), * a standalone sampler's at +0 */ diff --git a/src/imagination/vulkan/pvr_arch_g5p_tex.c b/src/imagination/vulkan/pvr_arch_g5p_tex.c index 84d25aba15c..5ddec94a2f3 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_tex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_tex.c @@ -81,7 +81,13 @@ g5p_view_state(const struct pvr_image_view *iview, uint64_t q[3]) pvr_g5p_tex_twiddled(q, &v); return NULL; } - if (view_type != VK_IMAGE_VIEW_TYPE_2D || iview->vk.base_mip_level) + /* A single-layer array view (PPSSPP binds every texture so) gets the + * image's 2D state: the compiler draws its layer-0 samples as 2D + * (PCO_G5P_UPLOAD_IMAGE_AS_2D); an array sample at another layer is + * refused at its draw, which is skipped (pvr_g5p_tex_state_for()). */ + const bool one_layer_array = view_type == VK_IMAGE_VIEW_TYPE_2D_ARRAY && + image->vk.image_type == VK_IMAGE_TYPE_2D && iview->vk.layer_count == 1; + if ((view_type != VK_IMAGE_VIEW_TYPE_2D && !one_layer_array) || iview->vk.base_mip_level) return "array/3D/cube or mip view of a linear image (Apple samples those twiddled only)"; if (va % 16 || level->pitch % 16) return "linear texture off a 16-byte base or 16-byte rows"; diff --git a/src/imagination/vulkan/pvr_arch_g5p_vtex.c b/src/imagination/vulkan/pvr_arch_g5p_vtex.c index dffbe29c74f..7bcbd5ae0e7 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vtex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vtex.c @@ -122,6 +122,9 @@ vtex_descriptor(struct pvr_cmd_buffer *cmd, const struct pco_g5p_upload *u, uint if (!image || u->dwords != 8 || !pvr_g5p_tex_state_valid(desc)) return VTEX_REFUSE(cmd, "set %u binding %u has no A9 image state", u->set, u->binding); memcpy(out, desc, 32); + const char *why = pvr_g5p_tex_state_for(out, u->offset); + if (why) + return VTEX_REFUSE(cmd, "set %u binding %u: %s", u->set, u->binding, why); } else if (u->kind == PCO_G5P_UPLOAD_SAMPLER) { if (lb->type != VK_DESCRIPTOR_TYPE_SAMPLER && lb->type != VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) return VTEX_REFUSE(cmd, "set %u binding %u is not a sampler", u->set, u->binding); diff --git a/src/imagination/vulkan/pvr_g5p_tex_encode.h b/src/imagination/vulkan/pvr_g5p_tex_encode.h index 80d7f35e206..bc875761bec 100644 --- a/src/imagination/vulkan/pvr_g5p_tex_encode.h +++ b/src/imagination/vulkan/pvr_g5p_tex_encode.h @@ -326,6 +326,37 @@ pvr_g5p_tex_size(const uint32_t state[4], unsigned level, uint32_t out[4]) /* A descriptor's state as the draw path uploads it: a valid image state * never has q0 = 0 (its layout bits), a valid sampler word is never 0. */ +/* + * The texture state an upload's sample reads (PCO_G5P_UPLOAD_IMAGE's offset + * flags), in place over the 8 copied dwords; NULL, or the reason there is none. + * - AS_2D (an array sample at layer 0 drawn as 2D): a 2D_ARRAY state becomes + * its base layer's 2D state (layout 2, no layer count, no big-layer bit; + * the address is already the base layer's, as a 2D view of that layer + * gets); a 2D or linear state is used as it is; 3D/cube have no 2D form. + * - ARRAY: the state must be an array's (a single-layer array view of a + * linear image has only its 2D state: sampled at a non-constant layer). + */ +static inline const char * +pvr_g5p_tex_state_for(uint32_t dw[8], unsigned flags) +{ + const unsigned layout = dw[0] & 7; + if (flags & 1u /* PCO_G5P_UPLOAD_IMAGE_AS_2D */) { + if (layout == PVR_G5P_TEX_LAYOUT_2D_ARRAY) { + uint64_t q0 = dw[0] | (uint64_t)dw[1] << 32, q1 = dw[2] | (uint64_t)dw[3] << 32; + q0 = (q0 & ~UINT64_C(7)) | PVR_G5P_TEX_LAYOUT_2D; + q1 &= ~(UINT64_C(0x3fff) << PVR_G5P_TEX_DEPTH_SHIFT); + q1 &= ~(UINT64_C(1) << PVR_G5P_TEX_BIG_LAYER_BIT); + dw[0] = (uint32_t)q0, dw[1] = (uint32_t)(q0 >> 32), dw[2] = (uint32_t)q1, dw[3] = (uint32_t)(q1 >> 32); + } else if (layout != PVR_G5P_TEX_LAYOUT_2D) { + return "a layer-0 array sample of a 3D or cube view"; + } + } else if ((flags & 2u /* PCO_G5P_UPLOAD_IMAGE_ARRAY */) && layout != PVR_G5P_TEX_LAYOUT_2D_ARRAY) { + return "an array sample at a non-constant layer of a view without array state (a single-layer view of a " + "linear image: Apple samples arrays twiddled only)"; + } + return NULL; +} + static inline bool pvr_g5p_tex_state_valid(const uint32_t dwords[2]) { From 0e8e4fef0f60ef8862eb0c2a902a02c235137eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:08 +0100 Subject: [PATCH 036/109] pvr/g5p: write user clip distances to the A9 TA user planes As Apple's A9 compiler lays out the vertex, the clip distances are its last dwords, after the varyings and the point size, one per declared distance. As Apple's A9 driver builds its output select, TA_OUTPUT_SEL enables user planes 0..n-1 for n declared distances, with no cull plane. Per-plane GL enables are the program's (Zink's nir_lower_clip_disable), so a declared distance the program never stores is written as 0. Cull distances, transform feedback of gl_ClipDistance and more than eight distances are refused by name. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 46 ++++++++++++++++-- src/imagination/g5p/pco_g5p_general.h | 4 ++ src/imagination/vulkan/pvr_arch_g5p_gfx.c | 13 +++++ src/imagination/vulkan/pvr_g5p.h | 8 +++ src/imagination/vulkan/pvr_g5p_nir.c | 59 +++++++++++++++++++++++ 5 files changed, 125 insertions(+), 5 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 1302a5f1a0c..4ed3adc5823 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -236,6 +236,7 @@ struct gctx { unsigned nvout; const struct pco_g5p_vs_link *link; unsigned psize_dword; /* a point pipeline's point size: the last output dword */ + unsigned clip_dword; /* the first user clip distance (Apple's A9 vertex: they follow the point size) */ bool eager_vout; /* VS temps retry: each output written as soon as its value is final */ bool compact_vs; /* last retry: compact movable bank1 values between instructions */ bool vs_bank0; /* VS pressure retry: temps in bank 0, outputs in bank 1 (vs-two-bank-allocator.md) */ @@ -1409,6 +1410,19 @@ static bool smooth_centre(nir_intrinsic_instr *in) interp == PCO_G5P_INTERP_SMOOTH && !centroid; } +/* A vertex output store's location, its constant slot offset included (a + * compact gl_ClipDistance[4..7] store is CLIP_DIST0 at offset 1); other + * offsets are refused by collect_vs_outputs(). */ +static unsigned +out_location(nir_intrinsic_instr *in) +{ + unsigned loc = nir_intrinsic_io_semantics(in).location; + nir_src *off = nir_get_io_offset_src(in); + if ((loc == VARYING_SLOT_CLIP_DIST0 || loc == VARYING_SLOT_CLIP_DIST1) && off && nir_src_is_const(*off)) + loc += nir_src_as_uint(*off); + return loc; +} + /* First output dword of a vertex output location: 0 for the position, -1 * when the location is not written (its fragment program does not read it), * -2 for a location without an output slot. */ @@ -1418,6 +1432,8 @@ static int vs_base(struct gctx *c, unsigned loc) return 0; if (loc == VARYING_SLOT_PSIZ && c->link && c->link->point_size) return c->psize_dword; + if ((loc == VARYING_SLOT_CLIP_DIST0 || loc == VARYING_SLOT_CLIP_DIST1) && c->p->clip_count) + return c->clip_dword + 4 * (loc - VARYING_SLOT_CLIP_DIST0); if (loc < VARYING_SLOT_VAR0 || loc > VARYING_SLOT_VAR31) return -2; unsigned d = c->p->vout_dword[loc - VARYING_SLOT_VAR0]; @@ -1478,6 +1494,21 @@ static void plan_vs_layout(struct gctx *c) * alpha for the size) */ if (c->link && c->link->point_size) c->psize_dword = c->p->outputs++; + /* User clip distances: the vertex's last dwords, after the point size, + * one per declared distance (Apple's applegpu_g5p layout, LD_MD +164; the + * TA's user planes 0..n-1, TA_OUTPUT_SEL plane0..7). Disabled GL planes + * are the program's business (Zink's nir_lower_clip_disable, Apple's + * clipDistanceEnableMask in the program key). */ + const unsigned clips = c->nir->info.clip_distance_array_size; + if (c->nir->info.cull_distance_array_size) + refuse(c, "cull distances (A9's TA user planes are clip planes; Apple's A9 driver sets no cull plane)"); + if (clips && !(c->link && c->link->clip_distance)) + refuse(c, "user clip distances"); + if (clips > 8) + refuse(c, "%u clip distances (the TA has 8 user planes)", clips); + c->clip_dword = c->p->outputs; + c->p->clip_count = clips; + c->p->outputs += clips; /* The A9 TA vertex holds 128 dwords, the position included * (varyings-capacity-1: 124 varying components exact, 128 wrong as a * whole with no GPU event); refuse past it by name, never draw it. */ @@ -1490,7 +1521,7 @@ static void plan_vs_layout(struct gctx *c) if (ins->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *in = nir_instr_as_intrinsic(ins); - if (in->intrinsic != nir_intrinsic_store_output || vs_base(c, nir_intrinsic_io_semantics(in).location) < 0) + if (in->intrinsic != nir_intrinsic_store_output || vs_base(c, out_location(in)) < 0) continue; last = in; last_comp = util_last_bit(nir_intrinsic_write_mask(in)) - 1; @@ -1773,7 +1804,7 @@ static void liveness(struct gctx *c) continue; if (in->instr.block->cf_node.parent->type != nir_cf_node_function) refuse(c, "store_output inside control flow"); - unsigned loc = nir_intrinsic_io_semantics(in).location; + unsigned loc = c->vs ? out_location(in) : nir_intrinsic_io_semantics(in).location; unsigned comp = nir_intrinsic_component(in); u_foreach_bit(k, nir_intrinsic_write_mask(in)) { if (c->vs) { @@ -5732,13 +5763,18 @@ static void collect_vs_outputs(struct gctx *c) nir_intrinsic_instr *in = nir_instr_as_intrinsic(ins); if (in->intrinsic != nir_intrinsic_store_output) continue; - unsigned loc = nir_intrinsic_io_semantics(in).location; + const unsigned sem_loc = nir_intrinsic_io_semantics(in).location; + const unsigned loc = out_location(in); int base = vs_base(c, loc); if (base == -2) - refuse(c, "store_output to %s (only position and generic varyings have output slots)", + refuse(c, "store_output to %s (only position, generic varyings and clip distances have output slots)", gl_varying_slot_name_for_stage(loc, MESA_SHADER_VERTEX)); - if (!nir_src_is_const(in->src[1]) || nir_src_as_uint(in->src[1])) + if (!nir_src_is_const(in->src[1]) || (loc == sem_loc && nir_src_as_uint(in->src[1]))) refuse(c, "indirect store_output"); + if (base >= 0 && (loc == VARYING_SLOT_CLIP_DIST0 || loc == VARYING_SLOT_CLIP_DIST1) && + 4 * (loc - VARYING_SLOT_CLIP_DIST0) + nir_intrinsic_component(in) + + util_last_bit(nir_intrinsic_write_mask(in)) > c->p->clip_count) + refuse(c, "a clip distance past the %u declared", c->p->clip_count); if (base == -1) continue; /* not read by the fragment program */ u_foreach_bit(k, nir_intrinsic_write_mask(in)) { diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index 9b5cba8604b..e204356bea3 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -187,6 +187,9 @@ struct pco_g5p_vs_link { /* a point pipeline: gl_PointSize is the last output dword, after the * varyings (TA_OUTPUT_SEL psprite_size_pres; Apple's g3b-points) */ bool point_size; + /* user clip distances admitted (the enabled user-plane path): their + * dwords follow the point size, the vertex's last (Apple's applegpu_g5p) */ + bool clip_distance; }; /* Per-draw transform feedback block (PCO_G5P_UPLOAD_XFB): the VA each @@ -249,6 +252,7 @@ struct pco_g5p_program { /* VS of a non-point pipeline: it wrote gl_PointSize, which was dropped * (set by pvr_g5p_nir.c after compilation; polygon mode POINT refuses it) */ bool psize_dropped; + uint8_t clip_count; /* VS: user clip distances, the last output dwords (TA user planes 0..n-1) */ char refusal[512]; }; diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 25036bf2f6e..bb530374482 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -625,6 +625,7 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo link.point_size = s->ia && s->ia->primitive_topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST; g->vs_point_size = link.point_size; g->polygon_mode = s->rs ? s->rs->polygon_mode : VK_POLYGON_MODE_FILL; + link.clip_distance = pvr_g5p_clip_distance_probe(); for (unsigned i = 0; i < fs->num_varyings; i++) link.interp[fs->vary[i].location] = fs->vary[i].interp; /* the transform feedback variant (pvr_g5p_xfb.h): the capturing vertex program */ @@ -753,6 +754,7 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo g->vs_written_temps = vs->written_temps; g->fs_temps = fs->temps; g->vs_outputs = vs->outputs; + g->vs_clip_count = vs->clip_count; g->vs_id_mask = vs->id_mask; memcpy(g->vs_id_reg, vs->id_reg, sizeof(g->vs_id_reg)); g->fs_discard = (fs->evidence & PCO_G5P_EV_DISCARD) != 0; @@ -1889,6 +1891,17 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g os.rhw_pres = true; /* points: the size follows the position (g3b-points) */ os.psprite_size_pres = g->vs_point_size; + /* user planes 0..n-1 from the declared clip distances, the vertex's + * last dwords (AGXMetalA9 VertexProgramVariant 0x207d28594: + * ~(-1 << count) into TA_OUTPUT_SEL; no cull plane) */ + os.plane0 = g->vs_clip_count > 0; + os.plane1 = g->vs_clip_count > 1; + os.plane2 = g->vs_clip_count > 2; + os.plane3 = g->vs_clip_count > 3; + os.plane4 = g->vs_clip_count > 4; + os.plane5 = g->vs_clip_count > 5; + os.plane6 = g->vs_clip_count > 6; + os.plane7 = g->vs_clip_count > 7; } PPP_BLOCK(w, 2); /* 6. ISP: the colour-only form (no depth/stencil attachment); Apple's diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index 4cccf838d57..0f152b6d73a 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -46,6 +46,13 @@ static inline VkResult pvr_refusal_to_api(VkResult result) #include "pvr_common.h" #include "pvr_limits.h" + +/* User clip distances are written to the A9 TA user planes. */ +static inline bool +pvr_g5p_clip_distance_probe(void) +{ + return true; +} #include "pvr_macros.h" #include "util/simple_mtx.h" #include "vk_meta.h" @@ -276,6 +283,7 @@ struct pvr_g5p_pipeline { * ISP object type, as AGXMetalA9 encodeISPPrimitiveType picks it from * its triangle fill mode (desktop-gl-polygon-mode.md) */ VkPolygonMode polygon_mode; + unsigned vs_clip_count; /* user clip distances: TA user planes 0..n-1 (Apple's VertexProgramVariant) */ /* the fragment program's tile read (pco_g5p_program::blend_mode: 0 none) * and colour write mask, for the ISP pass type (pvr_g5p_blend.h) */ unsigned fs_blend_mode; diff --git a/src/imagination/vulkan/pvr_g5p_nir.c b/src/imagination/vulkan/pvr_g5p_nir.c index 68667822411..75f71129e36 100644 --- a/src/imagination/vulkan/pvr_g5p_nir.c +++ b/src/imagination/vulkan/pvr_g5p_nir.c @@ -21,6 +21,7 @@ #include "compiler/spirv/spirv_info.h" #include "util/u_call_once.h" #include "util/u_math.h" +#include "vk_nir.h" #include "vk_shader_module.h" #include "vk_util.h" @@ -391,6 +392,14 @@ g5p_nir_prepare(nir_shader *nir) * as its location (glslang's gl_PointSize became slot 1, COL0) */ nir_split_var_copies(nir); nir_split_per_member_structs(nir); + /* gl_ClipDistance/gl_CullDistance sizes, before IO lowering, after the + * dead outputs are gone (vk_nir.c: glslang declares an unwritten + * gl_ClipDistance[1] and gl_CullDistance[1] in every gl_PerVertex) */ + if (stage == MESA_SHADER_VERTEX) { + const nir_remove_dead_variables_options dead = { .can_remove_var = nir_vk_is_not_xfb_output }; + nir_remove_dead_variables(nir, nir_var_shader_out, &dead); + nir_gather_clip_cull_distance_sizes_from_vars(nir); + } nir_lower_vars_to_ssa(nir); nir_opt_deref(nir); nir_lower_system_values(nir); @@ -538,6 +547,50 @@ pad_color_slots(nir_shader *nir, unsigned color_targets) nir_index_ssa_defs(impl); } +/* The clip distance components a vertex program stores (a bit per + * distance; compact CLIP_DIST0/1, constant offsets) */ +static bool +clip_written(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + if (in->intrinsic != nir_intrinsic_store_output) + return false; + const unsigned loc = nir_intrinsic_io_semantics(in).location; + if (loc != VARYING_SLOT_CLIP_DIST0 && loc != VARYING_SLOT_CLIP_DIST1) + return false; + nir_src *off = nir_get_io_offset_src(in); + const unsigned first = 4 * (loc - VARYING_SLOT_CLIP_DIST0 + (nir_src_is_const(*off) ? nir_src_as_uint(*off) : 0)) + + nir_intrinsic_component(in); + *(unsigned *)data |= nir_intrinsic_write_mask(in) << first; + return false; +} + +/* A declared clip distance the program never stores is 0 (on the plane, + * not clipped): the TA reads every declared one (plane0..n-1), and an + * unwritten output dword would clip by a stale register. GL's disabled + * planes are already stored as 0 (Zink's nir_lower_clip_disable). */ +static void +fill_clip_distances(nir_shader *nir) +{ + const unsigned n = nir->info.clip_distance_array_size; + unsigned written = 0; + if (!n) + return; + nir_shader_intrinsics_pass(nir, clip_written, nir_metadata_all, &written); + const unsigned missing = BITFIELD_MASK(n) & ~written; + if (!missing) + return; + nir_function_impl *impl = nir_shader_get_entrypoint(nir); + nir_builder b = nir_builder_at(nir_after_impl(impl)); + u_foreach_bit(i, missing) { + const gl_varying_slot loc = i < 4 ? VARYING_SLOT_CLIP_DIST0 : VARYING_SLOT_CLIP_DIST1; + nir_store_output(&b, nir_imm_float(&b, 0.0f), nir_imm_int(&b, 0), .base = 0, .component = i % 4, + .write_mask = 1, .src_type = nir_type_float32, + .io_semantics = (nir_io_semantics){ .location = loc, .num_slots = 1 }); + nir->info.outputs_written |= BITFIELD64_BIT(loc); + } + nir_index_ssa_defs(impl); +} + /* G5P renders layer 0 of single-layer views only (no layered rendering is * exposed): a vertex shader's gl_Layer is dropped and a fragment shader's * gl_Layer reads 0. vk_meta's rectangle and blit shaders use both. */ @@ -622,6 +675,8 @@ graphics_lower(nir_shader *nir, unsigned color_targets, bool *sample_mask, bool nir_opt_dce(nir); nir_index_ssa_defs(nir_shader_get_entrypoint(nir)); } + if (nir && stage == MESA_SHADER_VERTEX) + fill_clip_distances(nir); bool writes_psize = false; if (nir && stage == MESA_SHADER_VERTEX && point_size) nir_shader_intrinsics_pass(nir, find_point_size, nir_metadata_all, &writes_psize); @@ -736,6 +791,10 @@ xfb_lower(nir_shader *nir, struct g5p_xfb_layout *layout, char *refusal, size_t snprintf(refusal, refusal_size, "transform feedback of 16-bit data"); return false; } + if (o->location == VARYING_SLOT_CLIP_DIST0 || o->location == VARYING_SLOT_CLIP_DIST1) { + snprintf(refusal, refusal_size, "transform feedback of gl_ClipDistance (capture not qualified)"); + return false; + } const unsigned r = layout->nranges++; layout->ranges[r].buffer = o->buffer; layout->ranges[r].offset = o->offset; From dcd83e1aeb87b39a113ee44e74fb21be1fa3c9a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:08 +0100 Subject: [PATCH 037/109] pvr/g5p: do not advertise shaderCullDistance The A9 TA's user planes clip; Apple's A9 driver sets no cull plane. A cull distance drops whole primitives whose vertices are all outside, which a clip plane does not do. The A9 refuses vertex programs that write gl_CullDistance, yet advertised shaderCullDistance and 8 maxCullDistances, so PPSSPP wrote them and lost those draws. Report the feature off and maxCullDistances 0 on the A9. Assisted-by: LLM --- src/imagination/vulkan/pvr_physical_device.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index de78bbecc06..4cd9218f126 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -286,7 +286,10 @@ static void pvr_physical_device_get_supported_features( .shaderStorageBufferArrayDynamicIndexing = true, .shaderStorageImageArrayDynamicIndexing = true, .shaderClipDistance = true, - .shaderCullDistance = true, + /* G5P: the A9 TA's user planes clip; Apple's A9 driver sets no cull plane, and a cull + * distance is no clip distance (it drops whole primitives): not supported, so applications + * (PPSSPP) do not write gl_CullDistance */ + .shaderCullDistance = !PVR_HAS_FEATURE(dev_info, apple_g5p), .shaderFloat64 = false, .shaderInt64 = false, .shaderInt16 = false, @@ -810,7 +813,7 @@ static bool pvr_physical_device_get_properties( .timestampPeriod = 0.0f, .maxClipDistances = PVR_MAX_USER_PLANES, - .maxCullDistances = PVR_MAX_USER_PLANES, + .maxCullDistances = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 0U : PVR_MAX_USER_PLANES, /* no shaderCullDistance */ .maxCombinedClipAndCullDistances = PVR_MAX_USER_PLANES, .discreteQueuePriorities = 2U, From 8bd125cd6caf2cb5f4f581f5c346ee3d01023e6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:09 +0100 Subject: [PATCH 038/109] pvr/g5p: honour a render pass's dependencies on commands outside it pvr's render pass kept only dependencies between its own subpasses; a dependency with srcSubpass or dstSubpass VK_SUBPASS_EXTERNAL was dropped. A copy recorded after a render pass whose 0 -> EXTERNAL dependency orders it (colour output -> transfer, the usual render-then-read in one command buffer) read the attachment before the render job wrote it: on the A9 the copy read zeros. Explicit vkCmdPipelineBarrier()s were honoured. The pass merges its external dependencies (VkMemoryBarrier2 in pNext, or the 32-bit masks) into one barrier at begin and one at end. On the A9 they are recorded through CmdPipelineBarrier2 before the pass begins and after it ends. Other devices keep today's behaviour; applying this to them changes their command streams and is left to their maintainers. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_cmd_buffer.c | 16 ++++++++++++++ src/imagination/vulkan/pvr_arch_pass.c | 22 ++++++++++++++++++++ src/imagination/vulkan/pvr_pass.h | 7 +++++++ 3 files changed, 45 insertions(+) diff --git a/src/imagination/vulkan/pvr_arch_cmd_buffer.c b/src/imagination/vulkan/pvr_arch_cmd_buffer.c index 7efc5ccfff3..01f00920c45 100644 --- a/src/imagination/vulkan/pvr_arch_cmd_buffer.c +++ b/src/imagination/vulkan/pvr_arch_cmd_buffer.c @@ -4340,6 +4340,13 @@ void PVR_PER_ARCH(CmdBeginRenderPass2)( assert(!state->render_pass_info.pass); assert(cmd_buffer->vk.level == VK_COMMAND_BUFFER_LEVEL_PRIMARY); + /* G5P: the pass's EXTERNAL -> subpass dependencies, before it begins (pvr_render_pass::ext_begin) */ + if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g5p) && pass->ext_begin) { + const VkDependencyInfo ext = { .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, + .pMemoryBarriers = &pass->ext_begin_barrier }; + PVR_PER_ARCH(CmdPipelineBarrier2)(commandBuffer, &ext); + } + /* FIXME: Create a separate function for everything using pass->subpasses, * look at cmd_buffer_begin_subpass() for example. */ state->render_pass_info.pass = pass; @@ -10181,6 +10188,7 @@ void PVR_PER_ARCH(CmdEndRenderPass2)(VkCommandBuffer commandBuffer, assert(state->render_pass_info.pass); assert(state->render_pass_info.framebuffer); + const struct pvr_render_pass *ended_pass = state->render_pass_info.pass; result = pvr_arch_cmd_buffer_end_sub_cmd(cmd_buffer); if (result != VK_SUCCESS) @@ -10199,6 +10207,14 @@ void PVR_PER_ARCH(CmdEndRenderPass2)(VkCommandBuffer commandBuffer, state->render_pass_info.attachments = attachments; state->render_pass_info.clear_values = clear_values; + /* G5P: the pass's subpass -> EXTERNAL dependencies, after it ended: a transfer or + * draw after the pass that the pass declares it must precede (xapi-present V4, arr0: + * a copy of the pass's colour read the memory before the render wrote it) */ + if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g5p) && ended_pass->ext_end) { + const VkDependencyInfo ext = { .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, + .pMemoryBarriers = &ended_pass->ext_end_barrier }; + PVR_PER_ARCH(CmdPipelineBarrier2)(commandBuffer, &ext); + } } static VkResult diff --git a/src/imagination/vulkan/pvr_arch_pass.c b/src/imagination/vulkan/pvr_arch_pass.c index 8234da8b1c9..2f17922d4e5 100644 --- a/src/imagination/vulkan/pvr_arch_pass.c +++ b/src/imagination/vulkan/pvr_arch_pass.c @@ -871,6 +871,28 @@ PVR_PER_ARCH(CreateRenderPass2)(VkDevice _device, pass->max_sample_count = attachment->sample_count; } + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p)) { + /* Dependencies on commands outside the pass, merged (pvr_render_pass::ext_*) */ + pass->ext_begin_barrier = (VkMemoryBarrier2){ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2 }; + pass->ext_end_barrier = (VkMemoryBarrier2){ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2 }; + for (uint32_t i = 0; i < pCreateInfo->dependencyCount; i++) { + const VkSubpassDependency2 *dep = &pCreateInfo->pDependencies[i]; + if ((dep->srcSubpass == VK_SUBPASS_EXTERNAL) == (dep->dstSubpass == VK_SUBPASS_EXTERNAL)) + continue; + /* synchronization2: a VkMemoryBarrier2 in pNext replaces the 32-bit masks */ + const VkMemoryBarrier2 *mb2 = vk_find_struct_const(dep->pNext, MEMORY_BARRIER_2); + VkMemoryBarrier2 *b = dep->srcSubpass == VK_SUBPASS_EXTERNAL ? &pass->ext_begin_barrier : &pass->ext_end_barrier; + b->srcStageMask |= mb2 ? mb2->srcStageMask : dep->srcStageMask; + b->dstStageMask |= mb2 ? mb2->dstStageMask : dep->dstStageMask; + b->srcAccessMask |= mb2 ? mb2->srcAccessMask : dep->srcAccessMask; + b->dstAccessMask |= mb2 ? mb2->dstAccessMask : dep->dstAccessMask; + if (dep->srcSubpass == VK_SUBPASS_EXTERNAL) + pass->ext_begin = true; + else + pass->ext_end = true; + } + } + /* Count how many dependencies each subpass has. */ for (uint32_t i = 0; i < pCreateInfo->dependencyCount; i++) { const VkSubpassDependency2 *dep = &pCreateInfo->pDependencies[i]; diff --git a/src/imagination/vulkan/pvr_pass.h b/src/imagination/vulkan/pvr_pass.h index 1b52787e9f3..c290ab9519a 100644 --- a/src/imagination/vulkan/pvr_pass.h +++ b/src/imagination/vulkan/pvr_pass.h @@ -101,6 +101,13 @@ struct pvr_render_subpass { struct pvr_render_pass { struct vk_object_base base; + /* The pass's dependencies on commands outside it (srcSubpass or + * dstSubpass VK_SUBPASS_EXTERNAL), merged: G5P records them as pipeline + * barriers when the pass begins and ends (a copy after a render pass that + * declares 0 -> EXTERNAL must see its colour). */ + bool ext_begin, ext_end; + VkMemoryBarrier2 ext_begin_barrier, ext_end_barrier; + /* Saved information from pCreateInfo. */ uint32_t attachment_count; From 5c8fba2ee2fe0c1f84cb2f8747acb653c6246c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:09 +0100 Subject: [PATCH 039/109] pvr/g5p: launch vertex programs that read no attribute and no vertex index A WGSL vertex program that pulls from a storage buffer by instance only, or uses only constants, was refused ("no fetch-less PDS form"). Such a program still runs once per vertex, so it is launched with the vertex-index PDS form that a gl_VertexIndex-only program already uses. The index goes to an input register the program never reads. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index bb530374482..80ff6d0014b 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -775,8 +775,16 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo g->fetch_count = vertex.fetch_count; memcpy(g->fetch_list, vertex.fetch_list, sizeof(g->fetch_list)); - if (!g->fetch_count && !(g->vs_id_mask & 1)) - return G5P_REFUSE(device, "a vertex shader without attributes or vertex index (no fetch-less PDS form)"); + /* A vertex program that reads no attribute and no vertex index (WGSL vertex pulling from a + * storage buffer by instance only, or constants: nfsu2) still runs once per vertex: it is + * launched with the vertex-index PDS form that a gl_VertexIndex-only program uses (proven: + * a fullscreen vertex shader), the index written to an input register the program never + * reads (r0, or the one after the instance index). Apple's vertex task always receives the + * vertex id the same way. */ + if (!g->fetch_count && !(g->vs_id_mask & 1)) { + g->vs_id_reg[0] = (g->vs_id_mask & 2) ? g->vs_id_reg[1] + 1 : 0; + g->vs_id_mask |= 1; + } /* The unified store holds every input dword: Apple's sizes fit 4 units * per 8 input dwords; above 8 temps or 8 input dwords it grows with the @@ -1501,6 +1509,17 @@ fs_uploads(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline *g, result = pvr_arch_g5p_upload_buffer(cmd_buffer, u->set, u->binding, u->element, &addr, &size); if (result != VK_SUCCESS) return result; + if (u->kind == PCO_G5P_UPLOAD_SSBO_ADDR && u->offset == PCO_G5P_SSBO_SIZE_OFFSET) { + /* get_ssbo_size: the bound size in bytes */ + const uint32_t sz[2] = { (uint32_t)MIN2(size, UINT32_MAX), 0 }; + if (u->dwords > 2) + return G5P_REFUSE(cmd_buffer, "fragment storage buffer size: %u dwords", u->dwords); + result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, sz, sizeof(sz), &bo); + if (result != VK_SUCCESS) + return result; + src[i] = bo->dev_addr.addr; + continue; + } if (u->dwords != 2 || (u->kind == PCO_G5P_UPLOAD_UBO_ADDR && u->offset >= size)) return G5P_REFUSE(cmd_buffer, "fragment buffer address: %u dwords at offset %u of %" PRIu64 " bytes", u->dwords, u->offset, size); From aab4d14e78c4475ea5e6ec970b731a205c0319ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:09 +0100 Subject: [PATCH 040/109] pco/g5p: arrayLength, saturating add/sub, bank-0 operands of vertex buffer loads - get_ssbo_size (WGSL arrayLength): the binding's bound size, a common dword written by the CPU and uploaded by the fragment, vertex and vertex-texture paths. - usub_sat, uadd_sat and iadd_sat are lowered in NIR (compare and select). - A bank-0 index of a vertex register-indexed UBO load is copied into bank 1 with the mixed-bank move, as a storage buffer's offset already is, and a vertex storage-buffer load writes a bank-1 run when the program allocates in bank 0. A WebGPU game's bounds-clamped vertex-pulling shaders (32 of them) now compile; earlier programs are unchanged. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 30 ++++++++++++++++++-- src/imagination/g5p/pco_g5p_general.h | 3 ++ src/imagination/vulkan/pvr_arch_g5p_upload.c | 11 +++++++ src/imagination/vulkan/pvr_arch_g5p_vtex.c | 6 ++++ src/imagination/vulkan/pvr_g5p_nir.c | 4 +++ 5 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 4ed3adc5823..83c357368b5 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -401,6 +401,8 @@ static void vs_bank1_only(struct gctx *c, const char *what, int r0, int r1, int refuse(c, "%s with a register in vertex bank 0 (no witnessed mixed-bank form)", what); } +static unsigned vs_bank1_index(struct gctx *c, unsigned reg); + /* ---- registers (dwords) ---- */ /* A vertex bank-0 register: the first free one, else -1. */ static int alloc_bank0(struct gctx *c) @@ -1185,6 +1187,9 @@ static void scan_cf(struct gctx *c, struct exec_list *list, struct unsupported * case nir_intrinsic_load_ubo: ok = true; break; + case nir_intrinsic_get_ssbo_size: + ok = in->def.bit_size == 32 && in->def.num_components == 1; + break; case nir_intrinsic_store_ssbo: ok = in->src[0].ssa->bit_size==32 && in->num_components==1 && nir_intrinsic_write_mask(in)==1; break; @@ -2156,6 +2161,13 @@ static void plan_common(struct gctx *c) c->uptr[c->nuptr].element = element; c->uptr[c->nuptr++].base = base; } + } else if (in->intrinsic == nir_intrinsic_get_ssbo_size) { + /* the bound size, a CPU-written dword (PCO_G5P_SSBO_SIZE_OFFSET) */ + if (!block_binding_element(in->src[0].ssa, &set, &binding, &element)) + refuse(c, "get_ssbo_size: block index is not a constant descriptor binding"); + struct range *r = range_for(c, PCO_G5P_UPLOAD_SSBO_ADDR, set, binding, element); + r->lo = PCO_G5P_SSBO_SIZE_OFFSET; + r->hi = PCO_G5P_SSBO_SIZE_OFFSET + 4; } else if (in->intrinsic == nir_intrinsic_load_ubo) { if (!block_binding_element(in->src[0].ssa, &set, &binding, &element)) refuse(c, "load_ubo: block index is not a constant descriptor binding"); @@ -4285,8 +4297,10 @@ static void emit_dyn_ubo(struct gctx *c, nir_intrinsic_instr *in) c->uptr[i].element == element && c->uptr[i].base == base) common = c->uptr[i].common; unsigned index = reg_of(c, ix, "register-indexed UBO load index"); + /* a bank-0 index is copied into bank 1 first (the witnessed mixed-bank move), as a storage + * buffer's offset is (vs_bank1_index()): nfsu2's vertex pulling indexes its uniforms so */ if (c->vs) - vs_bank1_only(c, "vertex register-indexed UBO load", index, -1, -1, -1); + index = vs_bank1_index(c, index); bool kill = dies(c, ix); /* Registers are read at issue: like Apple's v_cexpr (index r0 -> pair * r0:r1 -> data r0..r3), the pair may take a dying index's register and @@ -4512,7 +4526,9 @@ static void emit_address_ssbo_load(struct gctx *c, nir_intrinsic_instr *in, unsi c->max_written = MAX2(c->max_written, pair + 2); put(c, raw, g5p_usc_address_add_shift(raw, 2 * pair, base, H(c, index), 0, kill, c->vs), "storage-buffer address"); - const unsigned dst = alloc_run(c, n, "load_ssbo"); + /* a vertex load writes bank 1 (the witnessed form), also when the program allocates in + * bank 0: its results are ordinary bank-1 registers, as a vertex sample's are */ + const unsigned dst = c->vs ? alloc_bank1_run(c, n, "load_ssbo") : alloc_run(c, n, "load_ssbo"); if (c->vs) { vs_bank1_only(c, "vertex storage-buffer load", dst, -1, -1, -1); put(c, raw, g5p_usc_vertex_address_load(raw, 2 * dst, bytes, 2 * pair), "vertex storage-buffer load"); @@ -4616,6 +4632,16 @@ static void emit_intrinsic(struct gctx *c, nir_intrinsic_instr *in) } break; } + case nir_intrinsic_get_ssbo_size: { + unsigned set, binding, element; + if (!block_binding_element(in->src[0].ssa, &set, &binding, &element)) + refuse(c, "get_ssbo_size: block index is not a constant descriptor binding"); + struct range *r = range_for(c, PCO_G5P_UPLOAD_SSBO_ADDR, set, binding, element); + struct val *v = V(c, &in->def, 0); + v->state = VAL_SHARED; + v->imm = r->common; + break; + } case nir_intrinsic_load_ssbo: { unsigned set, binding; block_binding(in->src[0].ssa, &set, &binding); diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index e204356bea3..ba998af668f 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -144,6 +144,9 @@ enum pco_g5p_upload_kind { * (a 2D_ARRAY state converted, pvr_g5p_tex_state_as_2d()); ARRAY: an array * sample, which needs a 2D_ARRAY state (the A9's array sample reads its * layer through the array descriptor even for one layer). */ +/* PCO_G5P_UPLOAD_SSBO_ADDR at this offset: not the buffer's address but its + * bound size in bytes (one dword; get_ssbo_size, WGSL arrayLength) */ +#define PCO_G5P_SSBO_SIZE_OFFSET 0xfffcu #define PCO_G5P_UPLOAD_IMAGE_AS_2D 1u #define PCO_G5P_UPLOAD_IMAGE_ARRAY 2u diff --git a/src/imagination/vulkan/pvr_arch_g5p_upload.c b/src/imagination/vulkan/pvr_arch_g5p_upload.c index 6b27795936d..3cf3c9dbd70 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_upload.c +++ b/src/imagination/vulkan/pvr_arch_g5p_upload.c @@ -249,6 +249,17 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc result = g5p_up_buffer(cmd_buffer, u, &addr, &size); if (result != VK_SUCCESS) return result; + if (u->kind == PCO_G5P_UPLOAD_SSBO_ADDR && u->offset == PCO_G5P_SSBO_SIZE_OFFSET) { + /* get_ssbo_size: the bound size in bytes */ + const uint32_t sz[2] = { (uint32_t)MIN2(size, UINT32_MAX), 0 }; + if (u->dwords > 2) + return G5P_REFUSE(cmd_buffer, "vertex storage buffer size: %u dwords", u->dwords); + result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, sz, sizeof(sz), &bo); + if (result != VK_SUCCESS) + return result; + src[i] = bo->dev_addr.addr; + break; + } if (u->dwords != 2 || (u->kind == PCO_G5P_UPLOAD_UBO_ADDR && u->offset >= size)) return G5P_REFUSE(cmd_buffer, "vertex %s: %u dwords at offset %u of %" PRIu64 " bytes", g5p_up_kind_str(u->kind), u->dwords, u->offset, size); diff --git a/src/imagination/vulkan/pvr_arch_g5p_vtex.c b/src/imagination/vulkan/pvr_arch_g5p_vtex.c index 7bcbd5ae0e7..2cf6c436306 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vtex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vtex.c @@ -177,6 +177,12 @@ PVR_PER_ARCH(g5p_vtex_draw)(struct pvr_cmd_buffer *cmd, const struct pvr_g5p_sta if (u->offset > size || bytes > size - u->offset) return VTEX_REFUSE(cmd, "UBO interval exceeds the bound range"); direct[i] = addr + u->offset; + } else if (u->kind == PCO_G5P_UPLOAD_SSBO_ADDR && u->offset == PCO_G5P_SSBO_SIZE_OFFSET) { + /* get_ssbo_size: the bound size in bytes */ + const uint32_t sz = (uint32_t)MIN2(size, UINT32_MAX); + if (u->dwords != 1) + return VTEX_REFUSE(cmd, "invalid storage buffer size upload"); + memcpy(dst, &sz, sizeof(sz)); } else { if (u->dwords != 2 || (u->kind == PCO_G5P_UPLOAD_UBO_ADDR && u->offset >= size)) return VTEX_REFUSE(cmd, "invalid buffer pointer upload"); diff --git a/src/imagination/vulkan/pvr_g5p_nir.c b/src/imagination/vulkan/pvr_g5p_nir.c index 75f71129e36..2ee5501d8ce 100644 --- a/src/imagination/vulkan/pvr_g5p_nir.c +++ b/src/imagination/vulkan/pvr_g5p_nir.c @@ -63,6 +63,10 @@ g5p_init_options(void) * shifts, masks and selects. Keep PCO's BFE/BFI capabilities disabled. */ g5p_options.lower_bitfield_extract = true; g5p_options.lower_bitfield_insert = true; + /* saturating integer add/subtract (naga's bounds clamps: usub_sat(arrayLength, 1)): compare and select */ + g5p_options.lower_usub_sat = true; + g5p_options.lower_uadd_sat = true; + g5p_options.lower_iadd_sat = true; g5p_vertex_options = g5p_options; g5p_vertex_options.max_unroll_iterations = 16; /* The isolated Apple iv_reg_pop/reverse witnesses qualify native VS From 2a0bb4dfbc06f2ff3d3d2beed89a94571e578706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:09 +0100 Subject: [PATCH 041/109] pvr/g5p: sample 4-sample colour images (sampler2DMS) A 4-sample colour image with SAMPLED usage is admitted (4-byte texels, 2D, one layer, level 0). Its samples are stored 16 bytes per pixel: sample s of pixel (x, y) is texel (2x + (s >> 1), 2y + (s & 1)) of a twiddled 2W x 2H 2D texture. The view takes that 2D state, and the compiler rewrites txf_ms to txf there, txs to the halved 2D size, and texture_samples to 4. sampledImageColorSampleCounts reports 1 | 4. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 38 +++++++++++++++++++++++ src/imagination/vulkan/pvr_arch_g5p_tex.c | 21 +++++++++++++ src/imagination/vulkan/pvr_g5p_image.c | 9 ++++-- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 83c357368b5..05f8ecfb7c6 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -6243,6 +6243,43 @@ static bool layer0_as_2d(nir_builder *b, nir_tex_instr *t, void *data) return true; } +/* A 4-sample texture (texture_multisampled_2d, sampler2DMS) is sampled as the 2W x 2H 2D + * texture of its samples (pvr_arch_g5p_tex.c): sample s of texel p is texel + * 2p + (s >> 1, s & 1); its size is half that texture's; it has 4 samples. */ +static bool ms_as_2d(nir_builder *b, nir_tex_instr *t, void *data) +{ + if (t->sampler_dim != GLSL_SAMPLER_DIM_MS || t->is_array) + return false; + b->cursor = nir_after_instr(&t->instr); + if (t->op == nir_texop_texture_samples) { + nir_def_replace(&t->def, nir_imm_int(b, 4)); + return true; + } + if (t->op == nir_texop_txs) { + t->sampler_dim = GLSL_SAMPLER_DIM_2D; + nir_def *half = nir_ushr_imm(b, &t->def, 1); + nir_def_rewrite_uses_after(&t->def, half); + return true; + } + if (t->op != nir_texop_txf_ms) + return false; + const int ci = nir_tex_instr_src_index(t, nir_tex_src_coord); + const int si = nir_tex_instr_src_index(t, nir_tex_src_ms_index); + if (ci < 0 || si < 0) + return false; + b->cursor = nir_before_instr(&t->instr); + nir_def *c = t->src[ci].src.ssa, *s = t->src[si].src.ssa; + nir_def *x = nir_iadd(b, nir_ishl_imm(b, nir_channel(b, c, 0), 1), nir_ushr_imm(b, s, 1)); + nir_def *y = nir_iadd(b, nir_ishl_imm(b, nir_channel(b, c, 1), 1), nir_iand_imm(b, s, 1)); + nir_tex_instr_remove_src(t, si); + nir_src_rewrite(&t->src[nir_tex_instr_src_index(t, nir_tex_src_coord)].src, nir_vec2(b, x, y)); + if (nir_tex_instr_src_index(t, nir_tex_src_lod) < 0) + nir_tex_instr_add_src(t, nir_tex_src_lod, nir_imm_int(b, 0)); + t->op = nir_texop_txf; + t->sampler_dim = GLSL_SAMPLER_DIM_2D; + return true; +} + static bool lower(nir_shader *nir, bool preserve_cf) { bool progress; @@ -6266,6 +6303,7 @@ static bool lower(nir_shader *nir, bool preserve_cf) (nir->info.stage == MESA_SHADER_VERTEX && g5p_vtex_size_lod_probe()), }; NIR_PASS(_, nir, nir_lower_tex, &txp); + NIR_PASS(_, nir, nir_shader_tex_pass, ms_as_2d, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_shader_tex_pass, layer0_as_2d, nir_metadata_control_flow, NULL); pvr_g5p_tbo_lower(nir); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_push_const, nir_address_format_32bit_offset); diff --git a/src/imagination/vulkan/pvr_arch_g5p_tex.c b/src/imagination/vulkan/pvr_arch_g5p_tex.c index 5ddec94a2f3..cca160c64e3 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_tex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_tex.c @@ -96,6 +96,27 @@ g5p_view_state(const struct pvr_image_view *iview, uint64_t q[3]) return NULL; } + /* A 4-sample colour image: each pixel's four samples are 16 consecutive bytes over the + * twiddled power-of-two surface (pvr_g5p_msaa_layout(), the PBE's multisample store), which is + * the 2W x 2H twiddled 2D texture whose texel (2x + (s >> 1), 2y + (s & 1)) is sample s of + * pixel (x, y) (Morton order, y in bit 0). The compiler reads sample s there (txf_ms lowered + * to txf, layer0/txs halved: pco_g5p_general.c ms_as_2d()). */ + if (image->vk.samples == VK_SAMPLE_COUNT_4_BIT) { + if (image->vk.image_type != VK_IMAGE_TYPE_2D || iview->vk.base_mip_level || iview->vk.layer_count != 1 || + (view_type != VK_IMAGE_VIEW_TYPE_2D && view_type != VK_IMAGE_VIEW_TYPE_2D_ARRAY) || + vk_format_get_blocksize(iview->vk.format) != 4) + return "a 4-sample view other than one layer of a 2D 4-byte colour image"; + const struct pvr_g5p_tex_twiddled_view v = { + .key = key, .layout = PVR_G5P_TEX_LAYOUT_2D, + .width = 2 * image->vk.extent.width, .height = 2 * image->vk.extent.height, .depth = 1, + .level_count = 1, .srgb = f->srgb, + .va = image->dev_addr.addr + plane->offset + iview->vk.base_array_layer * plane->layer_size, + }; + if (v.va % 64) + return "a 4-sample view off a 64-byte base"; + pvr_g5p_tex_twiddled(q, &v); + return NULL; + } if (image->vk.image_type == VK_IMAGE_TYPE_3D && iview->vk.view_type != VK_IMAGE_VIEW_TYPE_3D) return "2D view of a 3D image (a slice of the A9's 3D bricks is no 2D texture)"; diff --git a/src/imagination/vulkan/pvr_g5p_image.c b/src/imagination/vulkan/pvr_g5p_image.c index bfea4742ba2..b207212d4ea 100644 --- a/src/imagination/vulkan/pvr_g5p_image.c +++ b/src/imagination/vulkan/pvr_g5p_image.c @@ -214,7 +214,9 @@ pvr_g5p_msaa_limits(struct pvr_physical_device *pdevice, struct vk_properties *p p->framebufferStencilSampleCounts = fb; p->framebufferNoAttachmentsSampleCounts = fb; p->framebufferIntegerColorSampleCounts = VK_SAMPLE_COUNT_1_BIT; - p->sampledImageColorSampleCounts = VK_SAMPLE_COUNT_1_BIT; + /* 4-sample colour images are sampled (texelFetch of a 2D multisample texture, + * pvr_arch_g5p_tex.c's 2Wx2H view state and the compiler's txf_ms lowering) */ + p->sampledImageColorSampleCounts = fb; p->sampledImageIntegerSampleCounts = VK_SAMPLE_COUNT_1_BIT; p->sampledImageDepthSampleCounts = VK_SAMPLE_COUNT_1_BIT; p->sampledImageStencilSampleCounts = VK_SAMPLE_COUNT_1_BIT; @@ -237,9 +239,12 @@ pvr_g5p_msaa_image_samples(struct pvr_physical_device *pdevice, const VkPhysical const bool format_ok = pvr_g5p_ds_format(info->format) || (vk_format_is_color(info->format) && !vk_format_is_compressed(info->format) && !vk_format_is_int(info->format) && vk_format_get_blocksize(info->format) == 4); + /* a 4-sample colour attachment may also be sampled (its view is the 2Wx2H twiddled texture of + * its samples, pvr_arch_g5p_tex.c); depth/stencil, storage and input attachments stay 1x */ + const VkImageUsageFlags colour_read = vk_format_is_color(info->format) ? VK_IMAGE_USAGE_SAMPLED_BIT : 0; const bool four = tiling == VK_IMAGE_TILING_OPTIMAL && info->type == VK_IMAGE_TYPE_2D && !(info->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) && - (info->usage & attach) && !(info->usage & read) && format_ok; + (info->usage & attach) && !(info->usage & read & ~colour_read) && format_ok; props->sampleCounts &= four ? VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT : VK_SAMPLE_COUNT_1_BIT; } From b8610a626400e00c9c42b57ef6ca14895d8739e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 03:43:10 +0100 Subject: [PATCH 042/109] pvr: don't pack Rogue texture words that an Apple descriptor replaces Skip Rogue words only at the image-view sample packs overwritten by G5P or G9P and the G5P cube storage pack repacked at descriptor write. Preserve metadata and keep all other callers on the original csbgen path, including the stride assertion. Neither Apple image-view writer reads these words before replacing them. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_image.c | 4 ++++ src/imagination/vulkan/pvr_arch_tex_state.c | 5 +++++ src/imagination/vulkan/pvr_tex_state.h | 1 + 3 files changed, 10 insertions(+) diff --git a/src/imagination/vulkan/pvr_arch_image.c b/src/imagination/vulkan/pvr_arch_image.c index 5f92166cea5..fb4be87c5a8 100644 --- a/src/imagination/vulkan/pvr_arch_image.c +++ b/src/imagination/vulkan/pvr_arch_image.c @@ -203,6 +203,8 @@ VkResult PVR_PER_ARCH(CreateImageView)(VkDevice _device, util_format_compose_swizzles(format_swizzle, input_swizzle, info.swizzle); + info.skip_rogue_words = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p) || + PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g9p); result = pvr_arch_pack_tex_state(device, &info, &iview->image_state[info.tex_state_type]); @@ -220,6 +222,7 @@ VkResult PVR_PER_ARCH(CreateImageView)(VkDevice _device, */ if (info.is_cube && image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT) { info.tex_state_type = PVR_TEXTURE_STATE_STORAGE; + info.skip_rogue_words = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p); result = pvr_arch_pack_tex_state(device, @@ -230,6 +233,7 @@ VkResult PVR_PER_ARCH(CreateImageView)(VkDevice _device, } if (image->vk.usage & VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT) { + info.skip_rogue_words = false; /* Attachment state is created as if the mipmaps are not supported, so the * baselevel is set to zero and num_mip_levels is set to 1. Which gives an * impression that this is the only level in the image. This also requires diff --git a/src/imagination/vulkan/pvr_arch_tex_state.c b/src/imagination/vulkan/pvr_arch_tex_state.c index eaf373656d3..662aa274d90 100644 --- a/src/imagination/vulkan/pvr_arch_tex_state.c +++ b/src/imagination/vulkan/pvr_arch_tex_state.c @@ -121,6 +121,10 @@ VkResult pvr_arch_pack_tex_state(struct pvr_device *device, enum pvr_memlayout mem_layout; VkImageViewType iview_type; PVR_G5P_HOOK(texel_state, device, info, state) + if (info->skip_rogue_words) { + memset(state->words, 0, sizeof(state->words)); + goto metadata; + } if (info->type == VK_IMAGE_VIEW_TYPE_1D && info->mem_layout == PVR_MEMLAYOUT_LINEAR) { /* Change the memory layout to twiddled as there isn't a TEXSTATE_TEXTYPE @@ -315,6 +319,7 @@ VkResult pvr_arch_pack_tex_state(struct pvr_device *device, } } +metadata: state->meta[PCO_IMAGE_META_LAYER_SIZE] = info->layer_size; state->meta[PCO_IMAGE_META_BUFFER_ELEMS] = info->buffer_elems; state->meta[PCO_IMAGE_META_Z_SLICE] = info->z_slice; diff --git a/src/imagination/vulkan/pvr_tex_state.h b/src/imagination/vulkan/pvr_tex_state.h index fe48892f234..7764ef79824 100644 --- a/src/imagination/vulkan/pvr_tex_state.h +++ b/src/imagination/vulkan/pvr_tex_state.h @@ -48,6 +48,7 @@ /** Parameters for #pvr_pack_tex_state(). */ struct pvr_texture_state_info { + bool skip_rogue_words; VkFormat format; enum pvr_memlayout mem_layout; uint32_t flags; From 004018ae61147d52163841d81c4f82a0d239b3d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:09 +0100 Subject: [PATCH 043/109] pco/g5p: GLSL's packing built-ins packUnorm/packSnorm 2x16 and 4x8 and their unpacks lower in NIR, and packHalf2x16/unpackHalf2x16 go through the same integer-only pass (A9 programs hold no 16-bit values), as G9P's 26b4dcbc83d ("pvr/g9p: packing built-ins") does. All ten were refused before; they now compile in fragment and vertex programs. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 58 +++++++++++++++++++++++++++ src/imagination/vulkan/pvr_g5p_nir.c | 10 +++++ 2 files changed, 68 insertions(+) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 05f8ecfb7c6..bc1dc05b4a8 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -6280,9 +6280,67 @@ static bool ms_as_2d(nir_builder *b, nir_tex_instr *t, void *data) return true; } +/* A half float's bits (in a 32-bit value) as a 32-bit float, in integer code (G9P's 26b4dcbc83d ("pvr/g9p: packing built-ins")) */ +static nir_def * +half_to_f32(nir_builder *b, nir_def *h) +{ + nir_def *magnitude = nir_iand_imm(b, h, 0x7fff); + nir_def *shifted = nir_ishl_imm(b, magnitude, 13); + /* exponent 31: inf and NaN; exponent 0: zero and subnormals */ + nir_def *bits = nir_bcsel(b, nir_uge_imm(b, magnitude, 0x7c00), nir_ior_imm(b, shifted, 0x7f800000), + nir_iadd_imm(b, shifted, 0x38000000)); + + bits = nir_bcsel(b, nir_ult_imm(b, magnitude, 0x400), + nir_fmul_imm(b, nir_u2f32(b, magnitude), 1.0f / (1 << 24)), bits); + return nir_ior(b, bits, nir_ishl_imm(b, nir_iand_imm(b, h, 0x8000), 16)); +} + +/* A 32-bit float as a half float's bits, rounded to nearest, in integer code */ +static nir_def * +f32_to_half(nir_builder *b, nir_def *f) +{ + nir_def *sign = nir_iand_imm(b, nir_ushr_imm(b, f, 16), 0x8000); + nir_def *abs = nir_iand_imm(b, f, 0x7fffffff); + nir_def *normal = nir_ushr_imm(b, nir_iadd_imm(b, abs, 0x1000 - 0x38000000), 13); + nir_def *shift = nir_isub(b, nir_imm_int(b, 126), nir_ushr_imm(b, abs, 23)); + nir_def *mantissa = nir_ior_imm(b, nir_iand_imm(b, abs, 0x7fffff), 0x800000); + nir_def *rounded = nir_iadd(b, mantissa, nir_ishl(b, nir_imm_int(b, 1), nir_iadd_imm(b, shift, -1))); + nir_def *subnormal = nir_ushr(b, rounded, shift); + nir_def *v = nir_bcsel(b, nir_uge_imm(b, abs, 0x38800000), normal, + nir_bcsel(b, nir_uge_imm(b, abs, 0x33000000), subnormal, nir_imm_int(b, 0))); + + v = nir_bcsel(b, nir_uge_imm(b, abs, 0x47800000), nir_imm_int(b, 0x7c00), v); + v = nir_bcsel(b, nir_ult(b, nir_imm_int(b, 0x7f800000), abs), nir_imm_int(b, 0x7e00), v); + return nir_ior(b, sign, nir_iand_imm(b, v, 0x7fff)); +} + +/* packHalf2x16(), unpackHalf2x16(): no 16-bit values in G5P programs (g9p 26b4dcbc83d) */ +static bool +lower_half_packing(nir_builder *b, nir_instr *instr, void *data) +{ + if (instr->type != nir_instr_type_alu) + return false; + nir_alu_instr *alu = nir_instr_as_alu(instr); + + if (alu->op != nir_op_pack_half_2x16 && alu->op != nir_op_unpack_half_2x16) + return false; + b->cursor = nir_before_instr(instr); + nir_def *src = nir_ssa_for_alu_src(b, alu, 0); + + if (alu->op == nir_op_pack_half_2x16) { + nir_def_replace(&alu->def, nir_ior(b, f32_to_half(b, nir_channel(b, src, 0)), + nir_ishl_imm(b, f32_to_half(b, nir_channel(b, src, 1)), 16))); + } else { + nir_def_replace(&alu->def, nir_vec2(b, half_to_f32(b, nir_iand_imm(b, src, 0xffff)), + half_to_f32(b, nir_ushr_imm(b, src, 16)))); + } + return true; +} + static bool lower(nir_shader *nir, bool preserve_cf) { bool progress; + NIR_PASS(_, nir, nir_shader_instructions_pass, lower_half_packing, nir_metadata_control_flow, NULL); /* Projective sampling (GLSL textureProj/texture2DProj, Mesa's fixed-function * fragment program; Zink passes 2D projectors on as OpImageSampleProj*): * Apple's GPU has no projected sample, Metal divides in the program, so diff --git a/src/imagination/vulkan/pvr_g5p_nir.c b/src/imagination/vulkan/pvr_g5p_nir.c index 2ee5501d8ce..f92c5d451ea 100644 --- a/src/imagination/vulkan/pvr_g5p_nir.c +++ b/src/imagination/vulkan/pvr_g5p_nir.c @@ -63,6 +63,16 @@ g5p_init_options(void) * shifts, masks and selects. Keep PCO's BFE/BFI capabilities disabled. */ g5p_options.lower_bitfield_extract = true; g5p_options.lower_bitfield_insert = true; + /* GLSL's norm packings (packUnorm2x16() ...) lower to the measured float clamps, rounds, conversions and + * shifts (G9P's 26b4dcbc83d ("pvr/g9p: packing built-ins"); the half packings: pco_g5p_general.c lower_half_packing()) */ + g5p_options.lower_pack_snorm_2x16 = true; + g5p_options.lower_unpack_snorm_2x16 = true; + g5p_options.lower_pack_unorm_2x16 = true; + g5p_options.lower_unpack_unorm_2x16 = true; + g5p_options.lower_pack_snorm_4x8 = true; + g5p_options.lower_unpack_snorm_4x8 = true; + g5p_options.lower_pack_unorm_4x8 = true; + g5p_options.lower_unpack_unorm_4x8 = true; /* saturating integer add/subtract (naga's bounds clamps: usub_sat(arrayLength, 1)): compare and select */ g5p_options.lower_usub_sat = true; g5p_options.lower_uadd_sat = true; From 535a77fe1fb3a0ea360270b14affe16b15d638d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:10 +0100 Subject: [PATCH 044/109] pvr/g5p: flat varyings from the draw's provoking vertex; flat fans The A9 took every flat varying from vertex 0, although pvr advertises provokingVertexLast, which Zink uses for GL's last-vertex convention, and it refused flat varyings in triangle fans. A pipeline with flat varyings now also builds its coefficient program with FLAT_VERTEX1 and FLAT_VERTEX2, plus a fragment PPP block for each. The draw selects 2 for the last provoking vertex, 1 for a fan and 0 otherwise, in that block and in every VDM_STATE0 (Rogue's flatshade_control), as G9P's c74fd92b9fc and 5a7204430d1 do. Pipelines without flat varyings keep their words. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 48 ++++++++++++++++---- src/imagination/vulkan/pvr_arch_g5p_vertex.c | 2 + src/imagination/vulkan/pvr_g5p.h | 13 ++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 80ff6d0014b..92e86e5dc1b 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -336,7 +336,7 @@ static_assert(4 * PCO_G5P_MAX_FS_VARYINGS == 112, "G5P varying components"); static unsigned varying_program(const struct pco_g5p_program *fs, const struct pco_g5p_program *vs, uint32_t *data, - uint32_t *code, unsigned *code_dwords) + uint32_t *code, unsigned *code_dwords, unsigned pv) { pdsc_init(); pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-varying"); @@ -350,8 +350,10 @@ varying_program(const struct pco_g5p_program *fs, const struct pco_g5p_program * const bool flat = v->interp == PCO_G5P_INTERP_FLAT; item[items++] = (struct pdsc_douti_src0){ .dest = v->set, - /* PPP_CTRL flatshade_vtx is VERTEX_0 */ - .shademodel = flat ? PDSC_DOUTI_SHADEMODEL_FLAT_VERTEX0 : PDSC_DOUTI_SHADEMODEL_GOURAUD, + /* flat: the triangle's vertex pv (0, or 1/2 for a fan / the last provoking vertex: + * PVR_G5P_FLATSHADE_CONTROL(), G9P's c74fd92b9fc ("pvr/g9p: flat varyings") + 5a7204430d1) */ + .shademodel = flat ? (enum pdsc_douti_shademodel)(PDSC_DOUTI_SHADEMODEL_FLAT_VERTEX0 + pv) + : PDSC_DOUTI_SHADEMODEL_GOURAUD, .size = PDSC_DOUTI_SIZE_4D, .perspective = v->interp == PCO_G5P_INTERP_SMOOTH, .f32_offset = off, @@ -549,6 +551,10 @@ g5p_pipeline_free(struct pvr_device *device, struct pvr_g5p_pipeline *g) pvr_arch_g5p_vs_uploads_destroy(device, g->vs_uploads); pvr_bo_suballoc_free(g->static_vary); pvr_bo_suballoc_free(g->static_frag); + for (unsigned i = 0; i < 2; i++) { + pvr_bo_suballoc_free(g->varying_pv[i].pvr_bo); + pvr_bo_suballoc_free(g->static_frag_pv[i]); + } g5p_pipeline_free(device, g->xfb); g5p_pipeline_free(device, g->punch); vk_free(&device->vk.alloc, g); @@ -844,10 +850,17 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo unsigned varying_data_dwords = 0; if (fs->num_varyings || fs->w_set) { unsigned code_dwords; - varying_data_dwords = varying_program(fs, vs, data, code, &code_dwords); + varying_data_dwords = varying_program(fs, vs, data, code, &code_dwords, 0); result = pvr_gpu_upload_pds(device, data, varying_data_dwords, 16, code, code_dwords, 16, 64, &g->varying); if (result != VK_SUCCESS) return result; + for (unsigned pv = 1; g->fs_flat && pv <= 2; pv++) { + varying_program(fs, vs, data, code, &code_dwords, pv); + result = pvr_gpu_upload_pds(device, data, varying_data_dwords, 16, code, code_dwords, 16, 64, + &g->varying_pv[pv - 1]); + if (result != VK_SUCCESS) + return result; + } if (g->varying.code_offset != g->varying.data_offset + 4 * varying_data_dwords) return vk_errorf(device, VK_ERROR_INITIALIZATION_FAILED, "G5P: the coefficient code is at data + %#x, not + %#x", g->varying.code_offset - g->varying.data_offset, 4 * varying_data_dwords); @@ -925,6 +938,17 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo result = pvr_gpu_upload(device, device->heaps.general_heap, w, 24, 16, &g->static_frag); if (result != VK_SUCCESS) return result; + /* the same pointers naming the flat programs of vertices 1 and 2 */ + for (unsigned pv = 1; g->fs_flat && (fs->num_varyings || fs->w_set) && pv <= 2; pv++) { + uint32_t *v = g->static_frag_pv_words[pv - 1]; + memcpy(v, w, sizeof(g->static_frag_words)); + pvr_csb_pack (&v[5], TA_STATE_PDS_VARYINGBASE, vb) { + vb.addr = PVR_DEV_ADDR(g->varying_pv[pv - 1].data_offset); + } + result = pvr_gpu_upload(device, device->heaps.general_heap, v, 24, 16, &g->static_frag_pv[pv - 1]); + if (result != VK_SUCCESS) + return result; + } } return VK_SUCCESS; @@ -1816,9 +1840,8 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g return result; } - /* flat varyings take vertex 0; a fan's first vertex is the hardware's vertex 1 */ - if (dyn->ia.primitive_topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN && g->fs_flat) - return G5P_REFUSE(cmd_buffer, "flat varyings with a triangle fan"); + /* flat varyings: a fan's triangle takes them from the vertex VDM_STATE0's flatshade_control names + * (PVR_G5P_FLATSHADE_CONTROL(): 1 for fans; the refusal "flat varyings with a triangle fan" before) */ /* a depth/stencil attachment: its tests and bias (pvr_arch_g5p_ds.c) */ /* rasterizer discard: no depth/stencil aspects (Rogue's rule), the colour-only ISP form */ const VkImageAspectFlags ds_aspects = discard ? 0 : pvr_arch_g5p_ds_aspects(cmd_buffer); @@ -2125,6 +2148,7 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g pvr_csb_pack (&w[0], VDMCTRL_VDM_STATE0, v0) { v0.vs_other_present = true; v0.cam_size = cam; + v0.flatshade_control = PVR_G5P_FLATSHADE_CONTROL(dyn); } pvr_csb_pack (&w[1], VDMCTRL_VDM_STATE3, v3) { v3.vs_pds_code_base_addr = PVR_DEV_ADDR(fetch_code->code_offset); @@ -2162,6 +2186,7 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g pvr_csb_pack (&w[0], VDMCTRL_VDM_STATE0, v0) { v0.vs_data_addr_present = true; v0.cam_size = cam; + v0.flatshade_control = PVR_G5P_FLATSHADE_CONTROL(dyn); } pvr_csb_pack (&w[1], VDMCTRL_VDM_STATE2, v2) { v2.vs_pds_data_base_addr = PVR_DEV_ADDR(fetch_data.data_offset); @@ -2170,7 +2195,14 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g } /* 4. the fragment pointers and sizes */ - emit_ppp(csb, "ppp fragment", g->static_frag->dev_addr.addr, g->static_frag_words, 6); + { + /* flat varyings from the draw's provoking vertex (PVR_G5P_FLATSHADE_CONTROL()) */ + const unsigned pv = g->static_frag_pv[0] ? PVR_G5P_FLATSHADE_CONTROL(dyn) : 0; + if (pv) + emit_ppp(csb, "ppp fragment", g->static_frag_pv[pv - 1]->dev_addr.addr, g->static_frag_pv_words[pv - 1], 6); + else + emit_ppp(csb, "ppp fragment", g->static_frag->dev_addr.addr, g->static_frag_words, 6); + } if (g->fs_num_uploads) { /* then, as in the ICD's draws, the per-draw loader data pointer */ emit_ppp(csb, "ppp uniform data", uniform_va, uniform_block, 2); diff --git a/src/imagination/vulkan/pvr_arch_g5p_vertex.c b/src/imagination/vulkan/pvr_arch_g5p_vertex.c index 4d5eef84018..a727d43c6e7 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vertex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vertex.c @@ -81,6 +81,7 @@ PVR_PER_ARCH(g5p_vertex_index_list)(struct pvr_cmd_buffer *cmd_buffer, const str v0.cut_index_present = true; v0.cut_index_enable = true; v0.cam_size = vector <= 32 ? 95 : vector <= 48 ? 63 : vector <= 64 ? 31 : 15; + v0.flatshade_control = PVR_G5P_FLATSHADE_CONTROL(&cmd_buffer->vk.dynamic_graphics_state); } pvr_csb_pack (&w[(*n)++], VDMCTRL_VDM_STATE1, v1) { v1.cut_index = isize == 1 ? 0xff : isize == 2 ? 0xffff : 0xffffffff; @@ -188,6 +189,7 @@ PVR_PER_ARCH(g5p_vertex_indirect)(struct pvr_cmd_buffer *cmd_buffer, const struc state.cut_index_present = true; state.cut_index_enable = true; state.cam_size = vector <= 32 ? 95 : vector <= 48 ? 63 : vector <= 64 ? 31 : 15; + state.flatshade_control = PVR_G5P_FLATSHADE_CONTROL(&cmd_buffer->vk.dynamic_graphics_state); } pvr_csb_emit (csb, VDMCTRL_VDM_STATE1, state) { state.cut_index = isize == 1 ? 0xff : isize == 2 ? 0xffff : 0xffffffff; diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index 0f152b6d73a..e5942f73c10 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -252,6 +252,11 @@ struct pvr_g5p_pipeline { * pointers and sizes (6 words) */ struct pvr_suballoc_bo *static_vary, *static_frag; uint32_t static_vary_words[4], static_frag_words[6]; + /* flat varyings from the provoking vertex 1 and 2 (pvr_arch_g5p_gfx.c, PVR_G5P_FLATSHADE_CONTROL()): the + * coefficient program with FLAT_VERTEX1/2 and the fragment pointers naming it; [0] is varying/static_frag */ + struct pvr_pds_upload varying_pv[2]; + struct pvr_suballoc_bo *static_frag_pv[2]; + uint32_t static_frag_pv_words[2][6]; /* the vertex program: DOUTU temps, output dwords, unified-store size, * system values the fetch writes (bit 0 vertex index, 1 instance index) * and their registers */ @@ -462,4 +467,12 @@ uint32_t PVR_PER_ARCH(g5p_compute_stream)(const struct pvr_device *device, uint6 #endif /* PVR_PER_ARCH */ +/* VDM_STATE0 flatshade_control (Rogue's field, bits 1-2): the vertex of a strip's or fan's triangle whose flat + * varyings the triangle takes. 2 for the last provoking vertex, 1 for fans, 0 otherwise (G9P's 5a7204430d1 ("pvr/g9p: the flat colour's vertex in VDM_STATE0"): without it odd strip triangles took another vertex's colour, dEQP + * rasterization.flatshading.triangle_strip). @dyn: a vk_dynamic_graphics_state. */ +#define PVR_G5P_FLATSHADE_CONTROL(dyn) \ + ((dyn)->rs.provoking_vertex == VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT ? 2u \ + : (dyn)->ia.primitive_topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN ? 1u \ + : 0u) + #endif /* PVR_G5P_H */ From 809d31ee939ef5b3611dc19d2fad0af3209b7d9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:10 +0100 Subject: [PATCH 045/109] pco/g5p: explicit gradients in vertex programs as an explicit level textureGrad() in a vertex program lowers in NIR (nir_lower_tex lower_txd) to the vertex textureLod() with GL's isotropic level, as G9P's 2e5d5bc3e2f does, instead of being refused. Fragment gradients keep their native form, so fragment programs are unchanged. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index bc1dc05b4a8..028e53d3dfb 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -6359,6 +6359,9 @@ static bool lower(nir_shader *nir, bool preserve_cf) * This adds no TPU instruction or shadow-sampling recipe. */ .lower_txs_lod = nir->info.stage == MESA_SHADER_FRAGMENT || (nir->info.stage == MESA_SHADER_VERTEX && g5p_vtex_size_lod_probe()), + /* Explicit gradients in a vertex program as an explicit level, GL's isotropic formula (G9P's 2e5d5bc3e2f ("pvr/g9p: explicit gradients as explicit levels")): the qualified vertex txl instead of the unqualified gradient recipe. Fragment + * gradients keep their device-qualified recipe (R24). */ + .lower_txd = nir->info.stage == MESA_SHADER_VERTEX && !g5p_vtex_gradient_probe(), }; NIR_PASS(_, nir, nir_lower_tex, &txp); NIR_PASS(_, nir, nir_shader_tex_pass, ms_as_2d, nir_metadata_control_flow, NULL); From c47f2f0e3b8a98beb112266201f366053a30d31b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:10 +0100 Subject: [PATCH 046/109] pvr/g5p: no VK_EXT_non_seamless_cube_map on the A9 The A9 advertised the extension (shared pvr code) but samples every cube seamlessly and ignores the sampler's non-seamless bit. Not advertising it leaves GL's non-seamless cubes to Zink. The A10 keeps the extension (G9P's 82abaf9be98). Assisted-by: LLM --- src/imagination/vulkan/pvr_physical_device.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index 4cd9218f126..132fc8e830d 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -1305,6 +1305,11 @@ VkResult pvr_physical_device_init(struct pvr_physical_device *pdevice, supported_properties.transformFeedbackDraw = false; supported_properties.transformFeedbackQueries = true; supported_extensions.EXT_color_write_enable = false; + /* G5P samples every cube seamlessly and ignores VkSamplerCreateFlags' non-seamless bit; not advertising + * the extension leaves GL's non-seamless cubes to Zink (G9P's 82abaf9be98 ("pvr/g9p: non-seamless cube maps") builds them natively + * on the A10) */ + supported_extensions.EXT_non_seamless_cube_map = false; + supported_features.nonSeamlessCubeMap = false; supported_features.dualSrcBlend = true; supported_properties.maxFragmentDualSrcAttachments = 1; supported_features.colorWriteEnable = false; PVR_G5P_HOOK(release9_features, supported_features) From 590277b87e6108f87aaee3337c3eb4ea7c44ba74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:10 +0100 Subject: [PATCH 047/109] pco/g5p: textureSize() at a non-constant level in vertex programs A vertex textureSize(t, lod) whose level is not constant was refused. It now lowers, as fragment programs already do, to the level-0 size minified on the integer ALU (max(size >> lod, 1)), as G9P's 03ea1941019 does. 2D, 2D array, 3D, cube, integer and shadow forms compile. Assisted-by: LLM --- src/imagination/g5p/g5p_vtex.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imagination/g5p/g5p_vtex.h b/src/imagination/g5p/g5p_vtex.h index 31cdda806d3..25e686015f8 100644 --- a/src/imagination/g5p/g5p_vtex.h +++ b/src/imagination/g5p/g5p_vtex.h @@ -15,6 +15,6 @@ static inline bool g5p_vtex_fetch_offset_probe(void) { return true; } static inline bool g5p_vtex_gradient_probe(void) { return false; } static inline bool g5p_vtex_integer_probe(void) { return false; } static inline bool g5p_vtex_xyz_offset_probe(void) { return false; } -static inline bool g5p_vtex_size_lod_probe(void) { return false; } +static inline bool g5p_vtex_size_lod_probe(void) { return true; } static inline bool g5p_vtex_shadow_probe(void) { return false; } #endif From b6f8c0e4f86fc28b263a63216ab5e85704933b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:10 +0100 Subject: [PATCH 048/109] pvr/g5p: 8- and 16-bit integer vertex attributes; unbound inputs read (0, 0, 0, 1) glVertexAttribIPointer with GL_BYTE to GL_UNSIGNED_SHORT reaches the driver as R8G8B8A8_UINT, R16G16_SINT and the like. The A9 had no conversion for them, so they were not advertised and Zink converted them on the CPU (u_vbuf) per draw. They now fetch as the raw dwords that hold them, and the vertex program cuts each component out in NIR (a shift and mask, or a sign-extending shift pair). Every 8/16-bit pure-integer format with a dword-multiple size is advertised for vertex buffers. A location the pipeline gives no attribute reads (0, 0, 0, 1), GL's generic-attribute default, as G9P's 4a8c71666d9 does. Assisted-by: LLM --- src/imagination/vulkan/pvr_g5p_vertex.h | 1 + src/imagination/vulkan/pvr_g5p_vertex_nir.c | 57 +++++++++++++++++++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/imagination/vulkan/pvr_g5p_vertex.h b/src/imagination/vulkan/pvr_g5p_vertex.h index e65e9e8fbd6..7d771511e5f 100644 --- a/src/imagination/vulkan/pvr_g5p_vertex.h +++ b/src/imagination/vulkan/pvr_g5p_vertex.h @@ -71,6 +71,7 @@ struct pvr_g5p_vertex_format { uint8_t components, bytes; int8_t convert; bool packed_1010102; /* one raw dword; components unpacked in NIR */ + bool raw_int; /* 8/16-bit pure integers: the dwords fetched raw, each component shifted out in NIR */ uint8_t swizzle[4]; }; diff --git a/src/imagination/vulkan/pvr_g5p_vertex_nir.c b/src/imagination/vulkan/pvr_g5p_vertex_nir.c index 5facbc525b5..b0ce338b870 100644 --- a/src/imagination/vulkan/pvr_g5p_vertex_nir.c +++ b/src/imagination/vulkan/pvr_g5p_vertex_nir.c @@ -118,9 +118,16 @@ pvr_g5p_vertex_format_info(VkFormat format, struct pvr_g5p_vertex_format *f) f->convert = PCO_G5P_VSF_F16; return true; } - /* to float: normalized or scaled; 8- and 16-bit integer inputs are not - * converted yet */ - if (!(is_signed || is_unsigned) || ch->pure_integer) + /* 8- and 16-bit pure integers (glVertexAttribIPointer with GL_BYTE ... GL_UNSIGNED_SHORT): the dwords + * that hold them fetched raw, each component shifted out (and sign-extended) in NIR, no conversion + * (G9P's 4a8c71666d9 ("pvr/g9p: Qt Quick on Zink: packed vertex formats, texelFetch") for the A10) */ + if ((is_signed || is_unsigned) && ch->pure_integer) { + f->convert = -1; + f->raw_int = true; + return true; + } + /* to float: normalized or scaled */ + if (!(is_signed || is_unsigned)) return false; f->convert = (bits == 16 ? PCO_G5P_VSF_U16 : PCO_G5P_VSF_U8) + (ch->normalized ? 2 : 0) + is_signed; return true; @@ -295,6 +302,20 @@ rewrite_load(nir_builder *b, nir_intrinsic_instr *in, void *data) c[k] = comp != 3 ? nir_imm_int(b, 0) : is_float ? nir_imm_float(b, 1.0f) : nir_imm_int(b, 1); continue; } + if (f.raw_int) { + /* the component's bytes in the attribute's fetched dwords */ + const unsigned at = lead + f.bytes * f.swizzle[comp], reg = rw->base[loc] + at / 4; + const unsigned shift = 8 * (at % 4), bits = 8 * f.bytes; + nir_def *word = nir_load_input(b, 1, 32, nir_imm_int(b, 0), .base = reg / 4, + .component = reg % 4, .dest_type = nir_type_uint32, + .io_semantics = (nir_io_semantics){ + .location = VERT_ATTRIB_GENERIC0 + reg / 4, .num_slots = 1 }); + const bool is_signed = vk_format_description(rw->l->vi->attributes[loc].format)->channel[0].type == + UTIL_FORMAT_TYPE_SIGNED; + c[k] = is_signed ? nir_ishr_imm(b, nir_ishl_imm(b, word, 32 - shift - bits), 32 - bits) + : nir_iand_imm(b, nir_ushr_imm(b, word, shift), BITFIELD_MASK(bits)); + continue; + } if (f.packed_1010102) { unsigned reg = rw->base[loc]; nir_def *word = nir_load_input(b, 1, 32, nir_imm_int(b, 0), .base = reg / 4, @@ -332,11 +353,37 @@ rewrite_load(nir_builder *b, nir_intrinsic_instr *in, void *data) return true; } +/* A load of a location the pipeline gives no attribute: (0, 0, 0, 1) (G9P's 4a8c71666d9 ("pvr/g9p: Qt Quick on Zink: packed vertex formats, texelFetch"); the + * refusal "reads location %u, which has no vertex attribute" before) */ +static bool +unbound_input(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + const uint32_t valid = *(const uint32_t *)data; + if (in->intrinsic != nir_intrinsic_load_input) + return false; + const int loc = generic_location(in); + if (loc < 0 || (valid & BITFIELD_BIT(loc)) || in->def.bit_size != 32) + return false; + const bool is_float = nir_alu_type_get_base_type(nir_intrinsic_dest_type(in)) == nir_type_float; + b->cursor = nir_before_instr(&in->instr); + nir_def *c[4]; + for (unsigned k = 0; k < in->def.num_components; k++) + c[k] = nir_intrinsic_component(in) + k != 3 ? nir_imm_int(b, 0) : is_float ? nir_imm_float(b, 1.0f) + : nir_imm_int(b, 1); + nir_def_replace(&in->def, nir_vec(b, c, in->def.num_components)); + return true; +} + bool pvr_g5p_vertex_lower(nir_shader *vs, struct pvr_g5p_vertex_layout *l) { const struct vk_vertex_input_state *vi = l->vi; + { + uint32_t valid = vi ? vi->attributes_valid : 0; + nir_shader_intrinsics_pass(vs, unbound_input, nir_metadata_control_flow, &valid); + } + /* the locations the program reads */ l->read = 0; nir_foreach_function_impl (impl, vs) { @@ -378,14 +425,14 @@ pvr_g5p_vertex_lower(nir_shader *vs, struct pvr_g5p_vertex_layout *l) /* the fetch is dword-granular: a 32-bit attribute starts on a dword, * a converted one is read from the dwords that hold its bytes */ const unsigned lead = at->offset % 4; - if (f.convert < 0 && lead) + if (f.convert < 0 && !f.raw_int && lead) return REFUSE(l, "vertex attribute %u at offset %u (the fetch is dword-granular)", loc, at->offset); if (lead % f.bytes) return REFUSE(l, "vertex attribute %u: %u-byte components at offset %u", loc, f.bytes, at->offset); if (!l->dynamic_strides && vi->bindings[at->binding].stride % 4) return REFUSE(l, "vertex binding %u: stride %u (the fetch is dword-granular)", at->binding, vi->bindings[at->binding].stride); - converted |= f.convert >= 0 || f.packed_1010102; + converted |= f.convert >= 0 || f.packed_1010102 || f.raw_int; a[n++] = (struct attrib){ loc, at->binding, DIV_ROUND_UP(lead + f.bytes * f.components, 4), at->offset - lead, lead, f }; } From 6ecfb5664bc0915dfc0042c0c3b1c49fedacaea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:11 +0100 Subject: [PATCH 049/109] pvr/g5p: vertex bindings off dwords draw from one aligned copy per sub-command The vertex fetch reads dwords, so a binding whose address or stride is not a multiple of 4 (GL allows a stride of 6 for an RG8 attribute, or an offset of 3) was refused at the draw. Such a binding is now copied at submission into a dword-aligned buffer, with the stride rounded up to 4 (at least 16), and the fetch reads the copy. A 32-bit component that straddles dwords inside the element reads both dwords and funnel-shifts them. The copy is G9P's (52e91eea418) with its wait rule (fd62672ceef): the A9 queues it on the sub-command's g9p_realigns list, which the queue drains before the render, after the geometry sync when the source may be GPU-written. All draws of a primary command buffer's sub-command from the same source, size and stride share one copy; a secondary command buffer copies at recording. Bindings over 64 MiB are refused. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 106 ++++++++++++++++++- src/imagination/vulkan/pvr_arch_g5p_vertex.c | 6 +- src/imagination/vulkan/pvr_arch_g9p_gfx.c | 8 +- src/imagination/vulkan/pvr_g5p_vertex_nir.c | 25 +++-- src/imagination/vulkan/pvr_g9p.h | 12 +++ 5 files changed, 137 insertions(+), 20 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 92e86e5dc1b..66f95758c51 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -57,6 +57,8 @@ #include "pvr_g5p_upload.h" #include "pvr_g5p_xfb.h" #include "pvr_g5p_shader_dump.h" +#include "pvr_g9p.h" +#include "winsys/apple_g5p/pvr_g5p_winsys.h" #include "pvr_g5p_ds.h" #include "pvr_g5p_msaa.h" #include "pvr_g5p_vertex.h" @@ -1682,6 +1684,94 @@ g5p_record_ds_usage(struct pvr_sub_cmd_gfx *sub, const struct vk_dynamic_graphic } } +/* ---- unaligned vertex bindings (G9P's 52e91eea418 ("pvr/g9p: copy unaligned vertex bindings at submission, linear blits"), its pvr_arch_g9p_gfx.c realign) ---- + * The DDMAD fetch reads dwords from base + index * stride. A binding whose address or stride is not a multiple + * of 4 draws from a copy whose elements start on dwords: new stride = align(max(stride, 16), 4), room for the + * widest attribute past a small stride (GL's byte vec4 of stride 2). The bytes inside an element keep their + * offsets; an attribute at an unaligned offset is shifted out in the vertex program (pvr_g5p_vertex_nir.c). + * The copy runs when the job is submitted (pvr_arch_queue.c drains sub_cmd->gfx.g9p_realigns): a transfer + * recorded before the draw (they run on the CPU at submission) may be what fills the source; a source the GPU + * can write waits for the jobs before it. */ +static uint32_t +g5p_realigned_stride(uint32_t stride) +{ + return stride ? align(MAX2(stride, 16), 4) : 0; +} + +static uint8_t * +g5p_cpu_addr(struct pvr_device *device, uint64_t addr, uint64_t *available) +{ + uint8_t *p = NULL; + *available = 0; + simple_mtx_lock(&device->g9p_vmas.lock); + util_dynarray_foreach (&device->g9p_vmas.vmas, struct pvr_winsys_vma *, v) { + struct pvr_winsys_vma *vma = *v; + if (addr < vma->dev_addr.addr || addr >= vma->dev_addr.addr + vma->mapped_size) + continue; + if (!vma->bo->map && device->ws->ops->buffer_map(vma->bo, NULL) != VK_SUCCESS) + break; + const uint64_t offset = addr - vma->dev_addr.addr; + p = (uint8_t *)vma->bo->map + vma->bo_offset + offset; + *available = MIN2(vma->mapped_size - offset, vma->bo->size - vma->bo_offset - offset); + break; + } + simple_mtx_unlock(&device->g9p_vmas.lock); + return p; +} + +static VkResult +g5p_realign_binding(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_vertex_binding *vb, uint32_t stride, + uint64_t *addr) +{ + uint64_t available; + const uint8_t *src = g5p_cpu_addr(cmd_buffer->device, vb->buffer->dev_addr.addr + vb->offset, &available); + if (!src) + return G5P_REFUSE(cmd_buffer, "an unaligned vertex binding without a CPU mapping to copy it from"); + const uint32_t new_stride = g5p_realigned_stride(stride); + uint64_t size = vb->size == VK_WHOLE_SIZE ? vb->buffer->vk.size - vb->offset : vb->size; + size = MIN2(size, available); + const uint64_t count = stride ? size / stride + 1 : 1; + /* the sub-command's draws from the same binding share one copy (all are copied at submission) */ + if (cmd_buffer->vk.level != VK_COMMAND_BUFFER_LEVEL_SECONDARY) { + util_dynarray_foreach (&cmd_buffer->state.current_sub_cmd->gfx.g9p_realigns, struct g9p_realign, e) { + if (e->src == src && e->size == size && e->stride == stride && e->dst_addr) { + *addr = e->dst_addr; + return VK_SUCCESS; + } + } + } + const uint64_t copy_size = (uint64_t)MAX2(new_stride, 16) * count; + if (copy_size > 64u << 20) + return G5P_REFUSE(cmd_buffer, "an unaligned vertex binding of %" PRIu64 " bytes to copy", copy_size); + uint8_t *zero = calloc(1, copy_size); + if (!zero) + return vk_error(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY); + struct pvr_suballoc_bo *bo; + VkResult result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, zero, copy_size, &bo); + free(zero); + if (result != VK_SUCCESS) + return result; + *addr = bo->dev_addr.addr; + const struct g9p_realign r = { + .src = src, .dst = pvr_bo_suballoc_get_map_addr(bo), .size = size, .count = count, .stride = stride, + .new_stride = new_stride, .dst_addr = bo->dev_addr.addr, + .gpu_written = vb->buffer->g9p_gpu_written || + (vb->buffer->vk.usage & (VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | + VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT | + VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT)), + }; + /* a secondary's render joins its primary's job: copied now */ + if (cmd_buffer->vk.level == VK_COMMAND_BUFFER_LEVEL_SECONDARY) { + for (uint64_t i = 0; i < r.count; i++) + if (i * r.stride < r.size) + memcpy(r.dst + i * r.new_stride, r.src + i * r.stride, + MIN2(MAX2(r.new_stride, 16), r.size - i * r.stride)); + } else { + util_dynarray_append_typed(&cmd_buffer->state.current_sub_cmd->gfx.g9p_realigns, struct g9p_realign, r); + } + return VK_SUCCESS; +} + VkResult PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_draw *draw) { @@ -1891,15 +1981,27 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g const unsigned last_id = g->vs_id_mask ? util_last_bit(g->vs_id_mask) - 1 : 0; u_foreach_bit (i, g->vs_id_mask) fdata[fetch_ids(g->fetch_count) + k++] = doutw_control(g->vs_id_reg[i], i == 1, i == last_id); + uint64_t realigned[PVR_MAX_VERTEX_INPUT_BINDINGS] = { 0 }; for (unsigned f = 0; f < g->fetch_count; f++) { const struct pvr_g5p_fetch *fe = &g->fetch_list[f]; const struct pvr_vertex_binding *vb = &cs->vertex_bindings[fe->binding]; if (!vb->buffer) return G5P_REFUSE(cmd_buffer, "no vertex buffer bound at binding %u", fe->binding); - const uint32_t stride = dyn->vi_binding_strides[fe->binding]; + uint32_t stride = dyn->vi_binding_strides[fe->binding]; + uint64_t binding_base = vb->buffer->dev_addr.addr + vb->offset; + /* an address or stride off dwords: the copy's (g5p_realign_binding()) */ + if ((binding_base | stride) & 3) { + if (!realigned[fe->binding]) { + result = g5p_realign_binding(cmd_buffer, vb, stride, &realigned[fe->binding]); + if (result != VK_SUCCESS) + return result; + } + binding_base = realigned[fe->binding]; + stride = g5p_realigned_stride(stride); + } /* divisor 0: every instance reads the first instance's element */ const bool divisor0 = fe->rate == PVR_G5P_FETCH_INSTANCE_DIVISOR0; - const uint64_t base = vb->buffer->dev_addr.addr + vb->offset + fe->offset + + const uint64_t base = binding_base + fe->offset + (divisor0 ? (uint64_t)draw->first_instance * stride : 0); const uint64_t ctl = ddmad_src3(fe->dwords, fe->ao, f == g->fetch_count - 1); fdata[8 * f] = (uint32_t)base; diff --git a/src/imagination/vulkan/pvr_arch_g5p_vertex.c b/src/imagination/vulkan/pvr_arch_g5p_vertex.c index a727d43c6e7..3806bd02911 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vertex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vertex.c @@ -41,15 +41,13 @@ PVR_PER_ARCH(g5p_vertex_draw_check)(struct pvr_cmd_buffer *cmd_buffer, const str if (t != VK_INDEX_TYPE_UINT16 && t != VK_INDEX_TYPE_UINT32 && t != VK_INDEX_TYPE_UINT8_KHR) return G5P_REFUSE(cmd_buffer, "index type %d", t); } - /* the DDMAD reads dwords: every fetch's base and stride are dword-aligned */ + /* the DDMAD reads dwords: a binding whose address or stride is off dwords draws from a dword-aligned copy + * (pvr_arch_g5p_gfx.c g5p_realign_binding()) */ for (unsigned f = 0; f < g->fetch_count; f++) { const struct pvr_g5p_fetch *fe = &g->fetch_list[f]; const struct pvr_vertex_binding *vb = &cs->vertex_bindings[fe->binding]; if (!vb->buffer) return G5P_REFUSE(cmd_buffer, "no vertex buffer bound at binding %u", fe->binding); - if ((vb->offset + fe->offset) % 4 || dyn->vi_binding_strides[fe->binding] % 4) - return G5P_REFUSE(cmd_buffer, "vertex binding %u at offset %" PRIu64 ", stride %u (the fetch is dword-granular)", - fe->binding, (uint64_t)(vb->offset + fe->offset), dyn->vi_binding_strides[fe->binding]); } return VK_SUCCESS; } diff --git a/src/imagination/vulkan/pvr_arch_g9p_gfx.c b/src/imagination/vulkan/pvr_arch_g9p_gfx.c index b59bd652c09..14a6b9a5310 100644 --- a/src/imagination/vulkan/pvr_arch_g9p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g9p_gfx.c @@ -1640,13 +1640,7 @@ binding_needs_copy(const struct pvr_vertex_binding *vb, uint32_t stride) * The copy reads the buffer when the job is submitted: a transfer recorded * before the draw (they run on the CPU at submission) may be what fills it. */ -struct g9p_realign { - const uint8_t *src; - uint8_t *dst; - uint64_t size, count; - uint32_t stride, new_stride; - bool gpu_written; /* the source may be written by an earlier GPU job */ -}; +/* struct g9p_realign: pvr_g9p.h (G5P's draws queue them too) */ static void realign_copy(const struct g9p_realign *r) diff --git a/src/imagination/vulkan/pvr_g5p_vertex_nir.c b/src/imagination/vulkan/pvr_g5p_vertex_nir.c index b0ce338b870..e2d160013ce 100644 --- a/src/imagination/vulkan/pvr_g5p_vertex_nir.c +++ b/src/imagination/vulkan/pvr_g5p_vertex_nir.c @@ -344,6 +344,19 @@ rewrite_load(nir_builder *b, nir_intrinsic_instr *in, void *data) /* the component's byte in the fetched dwords; a converted one names * its format and byte in the load's base */ const unsigned at = lead + f.bytes * f.swizzle[comp], reg = rw->base[loc] + at / 4; + /* a 32-bit component off dwords (an attribute at an unaligned offset): the two dwords it straddles, + * funnel-shifted (the binding itself is dword-aligned, pvr_arch_g5p_gfx.c g5p_realign_binding()) */ + if (f.convert < 0 && at % 4) { + nir_def *w[2]; + for (unsigned i = 0; i < 2; i++) + w[i] = nir_load_input(b, 1, 32, nir_imm_int(b, 0), .base = (reg + i) / 4, .component = (reg + i) % 4, + .dest_type = nir_type_uint32, + .io_semantics = (nir_io_semantics){ .location = VERT_ATTRIB_GENERIC0 + (reg + i) / 4, + .num_slots = 1 }); + const unsigned sh = 8 * (at % 4); + c[k] = nir_ior(b, nir_ushr_imm(b, w[0], sh), nir_ishl_imm(b, w[1], 32 - sh)); + continue; + } const unsigned base = f.convert < 0 ? reg / 4 : PCO_G5P_VS_CONVERT_BASE + 4 * f.convert + at % 4; c[k] = nir_load_input(b, 1, 32, nir_imm_int(b, 0), .base = base, .component = reg % 4, .dest_type = type, .io_semantics = (nir_io_semantics){ .location = VERT_ATTRIB_GENERIC0 + reg / 4, @@ -425,14 +438,12 @@ pvr_g5p_vertex_lower(nir_shader *vs, struct pvr_g5p_vertex_layout *l) /* the fetch is dword-granular: a 32-bit attribute starts on a dword, * a converted one is read from the dwords that hold its bytes */ const unsigned lead = at->offset % 4; - if (f.convert < 0 && !f.raw_int && lead) - return REFUSE(l, "vertex attribute %u at offset %u (the fetch is dword-granular)", loc, at->offset); - if (lead % f.bytes) + /* a 32-bit attribute at an offset off dwords: read from the two dwords it straddles (rewrite_load()) */ + const bool straddle = f.convert < 0 && !f.raw_int && !f.packed_1010102 && lead; + if (!straddle && lead % f.bytes) return REFUSE(l, "vertex attribute %u: %u-byte components at offset %u", loc, f.bytes, at->offset); - if (!l->dynamic_strides && vi->bindings[at->binding].stride % 4) - return REFUSE(l, "vertex binding %u: stride %u (the fetch is dword-granular)", at->binding, - vi->bindings[at->binding].stride); - converted |= f.convert >= 0 || f.packed_1010102 || f.raw_int; + /* a stride off dwords: the draw's copy (pvr_arch_g5p_gfx.c g5p_realign_binding()) */ + converted |= f.convert >= 0 || f.packed_1010102 || f.raw_int || straddle; a[n++] = (struct attrib){ loc, at->binding, DIV_ROUND_UP(lead + f.bytes * f.components, 4), at->offset - lead, lead, f }; } diff --git a/src/imagination/vulkan/pvr_g9p.h b/src/imagination/vulkan/pvr_g9p.h index baeac2dd543..9179788dad9 100644 --- a/src/imagination/vulkan/pvr_g9p.h +++ b/src/imagination/vulkan/pvr_g9p.h @@ -7,6 +7,7 @@ #ifndef PVR_G9P_H #define PVR_G9P_H +#include #include #include @@ -23,6 +24,17 @@ struct pvr_graphics_pipeline; struct pvr_render_job; struct pvr_renderpass_hwsetup_render; struct pvr_sub_cmd_compute; +/* A vertex binding copied to a dword-aligned stride when its job is submitted (pvr_arch_g9p_gfx.c; G5P's draws + * queue them too, pvr_arch_g5p_gfx.c): src's count elements of stride bytes to dst at new_stride */ +struct g9p_realign { + const uint8_t *src; + uint8_t *dst; + uint64_t size, count; + uint32_t stride, new_stride; + bool gpu_written; /* the source may be written by an earlier GPU job */ + uint64_t dst_addr; /* the copy's device address (G5P reuses a copy across the sub-command's draws) */ +}; + struct pvr_sub_cmd_gfx; struct vk_graphics_pipeline_state; From 4fd2d1bd10557d8d37742fb8cf333f17647cafdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:41 +0100 Subject: [PATCH 050/109] pvr/g5p: log unbound vertex inputs and substitute (0, 0, 0, 1) Substitution is the fixed behaviour for unbound vertex inputs. Assisted-by: LLM --- src/imagination/vulkan/pvr_g5p_vertex_nir.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/imagination/vulkan/pvr_g5p_vertex_nir.c b/src/imagination/vulkan/pvr_g5p_vertex_nir.c index e2d160013ce..217b4103b6e 100644 --- a/src/imagination/vulkan/pvr_g5p_vertex_nir.c +++ b/src/imagination/vulkan/pvr_g5p_vertex_nir.c @@ -26,7 +26,9 @@ #include "compiler/nir/nir.h" #include "compiler/nir/nir_builder.h" #include "compiler/shader_enums.h" +#include "util/log.h" #include "util/macros.h" +#include "util/u_debug.h" #include "util/format/u_format.h" #include "util/u_math.h" #include "vk_format.h" @@ -366,17 +368,22 @@ rewrite_load(nir_builder *b, nir_intrinsic_instr *in, void *data) return true; } -/* A load of a location the pipeline gives no attribute: (0, 0, 0, 1) (G9P's 4a8c71666d9 ("pvr/g9p: Qt Quick on Zink: packed vertex formats, texelFetch"); the - * refusal "reads location %u, which has no vertex attribute" before) */ +/* A load of a location the pipeline gives no attribute: (0, 0, 0, 1), GL's generic-attribute default that Zink's GL + * relies on (G9P's 4a8c71666d9 ("pvr/g9p: Qt Quick on Zink: packed vertex formats, texelFetch")). Diagnostic: each pipeline logs the locations it substitutes. */ +struct unbound_state { + uint32_t valid, substituted; +}; + static bool unbound_input(nir_builder *b, nir_intrinsic_instr *in, void *data) { - const uint32_t valid = *(const uint32_t *)data; + struct unbound_state *u = data; if (in->intrinsic != nir_intrinsic_load_input) return false; const int loc = generic_location(in); - if (loc < 0 || (valid & BITFIELD_BIT(loc)) || in->def.bit_size != 32) + if (loc < 0 || (u->valid & BITFIELD_BIT(loc)) || in->def.bit_size != 32) return false; + u->substituted |= BITFIELD_BIT(loc); const bool is_float = nir_alu_type_get_base_type(nir_intrinsic_dest_type(in)) == nir_type_float; b->cursor = nir_before_instr(&in->instr); nir_def *c[4]; @@ -393,8 +400,10 @@ pvr_g5p_vertex_lower(nir_shader *vs, struct pvr_g5p_vertex_layout *l) const struct vk_vertex_input_state *vi = l->vi; { - uint32_t valid = vi ? vi->attributes_valid : 0; - nir_shader_intrinsics_pass(vs, unbound_input, nir_metadata_control_flow, &valid); + struct unbound_state u = { .valid = vi ? vi->attributes_valid : 0 }; + nir_shader_intrinsics_pass(vs, unbound_input, nir_metadata_control_flow, &u); + u_foreach_bit (loc, u.substituted) + mesa_logw("G5P vertex input %u has no attribute: reading (0,0,0,1)", loc); } /* the locations the program reads */ From ed6485f649f626f66e2d6b592222b3baab98f8b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:41 +0100 Subject: [PATCH 051/109] pvr/g5p: vertex formats of 1, 2, 3 and 6 bytes, through the aligned copy R8, RG8, RGB8, R16 and RGB16 (normalized, scaled, integer, f16) were not advertised for vertex buffers: the fetch reads whole dwords, and a tightly packed array of them has a stride off dwords, so Zink converted them on the CPU per draw. They are advertised now, and such a binding draws from the aligned copy. Each fetch records how far its whole-dword reads go past the end of its binding's last attribute. When that could leave the buffer's GPU mapping (the VMA's device address plus its mapped size), the binding draws from the copy too, and every copy ends in that many zero bytes. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 36 ++++++++++++++++----- src/imagination/vulkan/pvr_formats.c | 14 ++++---- src/imagination/vulkan/pvr_g5p.h | 3 ++ src/imagination/vulkan/pvr_g5p_vertex_nir.c | 18 +++++++++++ src/imagination/vulkan/pvr_g9p.h | 1 + 5 files changed, 58 insertions(+), 14 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 66f95758c51..fa4eba270a5 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -58,7 +58,6 @@ #include "pvr_g5p_xfb.h" #include "pvr_g5p_shader_dump.h" #include "pvr_g9p.h" -#include "winsys/apple_g5p/pvr_g5p_winsys.h" #include "pvr_g5p_ds.h" #include "pvr_g5p_msaa.h" #include "pvr_g5p_vertex.h" @@ -1721,7 +1720,7 @@ g5p_cpu_addr(struct pvr_device *device, uint64_t addr, uint64_t *available) static VkResult g5p_realign_binding(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_vertex_binding *vb, uint32_t stride, - uint64_t *addr) + uint32_t extra, uint64_t *addr) { uint64_t available; const uint8_t *src = g5p_cpu_addr(cmd_buffer->device, vb->buffer->dev_addr.addr + vb->offset, &available); @@ -1734,13 +1733,14 @@ g5p_realign_binding(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_vertex_b /* the sub-command's draws from the same binding share one copy (all are copied at submission) */ if (cmd_buffer->vk.level != VK_COMMAND_BUFFER_LEVEL_SECONDARY) { util_dynarray_foreach (&cmd_buffer->state.current_sub_cmd->gfx.g9p_realigns, struct g9p_realign, e) { - if (e->src == src && e->size == size && e->stride == stride && e->dst_addr) { + if (e->src == src && e->size == size && e->stride == stride && e->extra >= extra && e->dst_addr) { *addr = e->dst_addr; return VK_SUCCESS; } } } - const uint64_t copy_size = (uint64_t)MAX2(new_stride, 16) * count; + /* the elements, then zeros for the fetches' reads past the last one (extra: its fetch extent) */ + const uint64_t copy_size = (uint64_t)MAX2(new_stride, 16) * count + extra; if (copy_size > 64u << 20) return G5P_REFUSE(cmd_buffer, "an unaligned vertex binding of %" PRIu64 " bytes to copy", copy_size); uint8_t *zero = calloc(1, copy_size); @@ -1754,7 +1754,7 @@ g5p_realign_binding(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_vertex_b *addr = bo->dev_addr.addr; const struct g9p_realign r = { .src = src, .dst = pvr_bo_suballoc_get_map_addr(bo), .size = size, .count = count, .stride = stride, - .new_stride = new_stride, .dst_addr = bo->dev_addr.addr, + .new_stride = new_stride, .dst_addr = bo->dev_addr.addr, .extra = extra, .gpu_written = vb->buffer->g9p_gpu_written || (vb->buffer->vk.usage & (VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT | @@ -1982,17 +1982,37 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g u_foreach_bit (i, g->vs_id_mask) fdata[fetch_ids(g->fetch_count) + k++] = doutw_control(g->vs_id_reg[i], i == 1, i == last_id); uint64_t realigned[PVR_MAX_VERTEX_INPUT_BINDINGS] = { 0 }; + /* A binding draws from a dword-aligned, zero-padded copy (g5p_realign_binding()) when its address or stride + * is off dwords, or when a fetch's whole-dword overread past the bound range could leave the buffer's GPU + * mapping (a sub-dword attribute at the very end of its BO) */ + bool copy[PVR_MAX_VERTEX_INPUT_BINDINGS] = { false }; + uint32_t extent[PVR_MAX_VERTEX_INPUT_BINDINGS] = { 0 }; for (unsigned f = 0; f < g->fetch_count; f++) { const struct pvr_g5p_fetch *fe = &g->fetch_list[f]; const struct pvr_vertex_binding *vb = &cs->vertex_bindings[fe->binding]; if (!vb->buffer) return G5P_REFUSE(cmd_buffer, "no vertex buffer bound at binding %u", fe->binding); + const uint32_t stride = dyn->vi_binding_strides[fe->binding]; + const uint64_t binding_base = vb->buffer->dev_addr.addr + vb->offset; + extent[fe->binding] = MAX2(extent[fe->binding], fe->offset + 4 * fe->dwords); + if ((binding_base | stride) & 3) { + copy[fe->binding] = true; + } else if (fe->overread) { + const uint64_t range = vb->size == VK_WHOLE_SIZE ? vb->buffer->vk.size - vb->offset : vb->size; + const struct pvr_winsys_vma *vma = vb->buffer->vma; + const uint64_t gpu_end = vma && vma->bo ? vma->dev_addr.addr + vma->mapped_size : 0; + if (gpu_end && binding_base + range + fe->overread > gpu_end) + copy[fe->binding] = true; + } + } + for (unsigned f = 0; f < g->fetch_count; f++) { + const struct pvr_g5p_fetch *fe = &g->fetch_list[f]; + const struct pvr_vertex_binding *vb = &cs->vertex_bindings[fe->binding]; uint32_t stride = dyn->vi_binding_strides[fe->binding]; uint64_t binding_base = vb->buffer->dev_addr.addr + vb->offset; - /* an address or stride off dwords: the copy's (g5p_realign_binding()) */ - if ((binding_base | stride) & 3) { + if (copy[fe->binding]) { if (!realigned[fe->binding]) { - result = g5p_realign_binding(cmd_buffer, vb, stride, &realigned[fe->binding]); + result = g5p_realign_binding(cmd_buffer, vb, stride, extent[fe->binding], &realigned[fe->binding]); if (result != VK_SUCCESS) return result; } diff --git a/src/imagination/vulkan/pvr_formats.c b/src/imagination/vulkan/pvr_formats.c index 03b44099529..60f5517ff85 100644 --- a/src/imagination/vulkan/pvr_formats.c +++ b/src/imagination/vulkan/pvr_formats.c @@ -453,15 +453,17 @@ pvr_get_format_swizzle_for_tpu(const struct util_format_description *desc) } /* G5P fetches the vertex formats of pvr_g5p_vertex_format_info() (32-bit - * float and integer, 8/16-bit normalized and scaled, 16-bit float) with - * dword strides. Of those, it advertises the ones a whole number of dwords - * in size, whose tightly packed strides are dword multiples too (Zink passes - * GL's strides through), and no other, so that Zink converts the rest. */ + * float and integer, 8/16-bit normalized, scaled and pure integer, 16-bit + * float) as whole dwords and advertises all of them: a binding whose address + * or stride is off dwords (a tightly packed R8 or RGB16 array) draws from a + * dword-aligned copy, as does one whose last element sits at the end of the + * buffer's GPU mapping (pvr_arch_g5p_gfx.c). */ static bool pvr_g5p_vertex_buffer_format(VkFormat vk_format) { - return pvr_g5p_vertex_format_info(vk_format, &(struct pvr_g5p_vertex_format){ 0 }) && - vk_format_get_blocksize(vk_format) % 4 == 0; + /* Any size: the fetch reads whole dwords, and a read that could pass the end of the buffer's GPU mapping + * draws from a padded copy instead (pvr_arch_g5p_gfx.c, pvr_g5p_fetch.overread) */ + return pvr_g5p_vertex_format_info(vk_format, &(struct pvr_g5p_vertex_format){ 0 }); } static VkFormatFeatureFlags2 diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index e5942f73c10..c0e1f952565 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -211,6 +211,9 @@ enum pvr_g5p_fetch_rate { struct pvr_g5p_fetch { uint8_t binding, dwords, ao, rate; uint32_t offset, divisor; + /* bytes the fetch reads past the end of its binding's last attribute in the element (the fetch reads whole + * dwords, the ICD layout 16 bytes a location): the draw keeps that inside the buffer's GPU mapping */ + uint16_t overread; }; /* One common-store upload of the fragment program (the backend's diff --git a/src/imagination/vulkan/pvr_g5p_vertex_nir.c b/src/imagination/vulkan/pvr_g5p_vertex_nir.c index 217b4103b6e..c9aeeffb838 100644 --- a/src/imagination/vulkan/pvr_g5p_vertex_nir.c +++ b/src/imagination/vulkan/pvr_g5p_vertex_nir.c @@ -276,6 +276,22 @@ packed_fetches(struct pvr_g5p_vertex_layout *l, struct attrib *a, unsigned n, un return true; } +/* Each fetch's overread: how far it reads past the end of the last attribute of its binding's element. Vulkan + * keeps every fetched vertex's attributes inside the bound range, so the draw's reads end at most that many bytes + * past it (pvr_arch_g5p_gfx.c keeps them inside the buffer's GPU mapping, or draws from a padded copy). */ +static void +fetch_overreads(struct pvr_g5p_vertex_layout *l, const struct attrib *a, unsigned n) +{ + uint32_t end[MESA_VK_MAX_VERTEX_BINDINGS] = { 0 }; + for (unsigned i = 0; i < n; i++) + end[a[i].binding] = MAX2(end[a[i].binding], a[i].offset + a[i].lead + a[i].f.bytes * a[i].f.components); + for (unsigned f = 0; f < l->fetch_count; f++) { + struct pvr_g5p_fetch *fe = &l->fetch_list[f]; + const uint32_t fetch_end = fe->offset + 4 * fe->dwords; + fe->overread = fetch_end > end[fe->binding] ? MIN2(fetch_end - end[fe->binding], UINT16_MAX) : 0; + } +} + struct rewrite { const struct pvr_g5p_vertex_layout *l; const unsigned *base; @@ -462,12 +478,14 @@ pvr_g5p_vertex_lower(nir_shader *vs, struct pvr_g5p_vertex_layout *l) l->packed = converted || l->read >= BITFIELD_BIT(ICD_LOCATIONS); if (!l->packed) { icd_fetches(l, a, n); + fetch_overreads(l, a, n); return fetch_divisors(l); } unsigned base[PCO_G5P_MAX_ATTRIBS] = { 0 }; if (!packed_fetches(l, a, n, base) || !fetch_divisors(l)) return false; + fetch_overreads(l, a, n); const struct rewrite rw = { l, base }; nir_shader_intrinsics_pass(vs, rewrite_load, nir_metadata_control_flow, (void *)&rw); nir_opt_constant_folding(vs); diff --git a/src/imagination/vulkan/pvr_g9p.h b/src/imagination/vulkan/pvr_g9p.h index 9179788dad9..4aeae805fa9 100644 --- a/src/imagination/vulkan/pvr_g9p.h +++ b/src/imagination/vulkan/pvr_g9p.h @@ -33,6 +33,7 @@ struct g9p_realign { uint32_t stride, new_stride; bool gpu_written; /* the source may be written by an earlier GPU job */ uint64_t dst_addr; /* the copy's device address (G5P reuses a copy across the sub-command's draws) */ + uint32_t extra; /* G5P: zero bytes after the copy's last element, for its fetches' overread */ }; struct pvr_sub_cmd_gfx; From 4192e4636e1734d31f13059e12b2369bbe17414d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:42 +0100 Subject: [PATCH 052/109] pvr/g5p: retain narrow integer formats as unadvertised probes Keep narrow integer render targets unavailable until they are qualified. Assisted-by: LLM --- src/imagination/vulkan/pvr_g5p_format_table.h | 2 +- src/imagination/vulkan/pvr_g5p_formats.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/imagination/vulkan/pvr_g5p_format_table.h b/src/imagination/vulkan/pvr_g5p_format_table.h index 91fd603396f..f5851a2eec9 100644 --- a/src/imagination/vulkan/pvr_g5p_format_table.h +++ b/src/imagination/vulkan/pvr_g5p_format_table.h @@ -64,7 +64,7 @@ enum pvr_g5p_fmt_flags { PVR_G5P_FMT_BLEND = 1u << 3, /* blending into it (the fragment program's) */ PVR_G5P_FMT_COMPRESSED = 1u << 4, /* blocks (4x4 ETC2/EAC, 4x4..12x12 ASTC), twiddled only */ PVR_G5P_FMT_INTEGER = 1u << 5, /* integer texel: texelFetch/unfiltered */ - /* not advertised (none at present) */ + /* narrow integer render rows remain unadvertised probes */ PVR_G5P_FMT_PROBE_SAMPLED = 1u << 8, /* the sampled bits are a probe */ PVR_G5P_FMT_PROBE_RENDER = 1u << 9, /* the render (and blend) bits are a probe */ PVR_G5P_FMT_PROBE_BLEND = 1u << 10, /* the blend bit alone is a probe */ diff --git a/src/imagination/vulkan/pvr_g5p_formats.c b/src/imagination/vulkan/pvr_g5p_formats.c index 98426911246..1962ddef2c7 100644 --- a/src/imagination/vulkan/pvr_g5p_formats.c +++ b/src/imagination/vulkan/pvr_g5p_formats.c @@ -18,6 +18,7 @@ #include #include "util/log.h" +#include "util/u_debug.h" #include "vk_enum_to_str.h" #include "vk_format.h" @@ -31,6 +32,7 @@ pvr_g5p_formats_probe(void) return false; } + bool pvr_g5p_format_tex(VkFormat format, uint32_t *key, bool *srgb) { From d5ed9b6735593b1d3abbc86ba56be60025c851a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:42 +0100 Subject: [PATCH 053/109] pvr/g5p: 16-bit packed textures from Apple's A9 format table PPSSPP samples A1R5G5B5 (with B4G4R4A4 and R5G6B5) once a Vulkan driver offers it. The A9 had only the two forms Rogue's table knows. Apple's A9 texture-format table (hardware code; sampling swizzle in 3-bit selectors) and its Metal pixel-format map give: B5G6R5Unorm -> RGB565 0x0b [c2 c1 c0 1] A1BGR5Unorm -> RGB5A1 0x0d [c3 c2 c1 c0] ABGR4Unorm -> RGBA4 0x0c [c3 c2 c1 c0] BGR5A1Unorm -> A1RGB5 0x0e [c2 c1 c0 c3] BGRA4 0x0c [c2 c1 c0 c3] New sampled and filtered rows: A1R5G5B5 (0x0e), R5G5B5A1 and B5G5R5A1 (0x0d), B5G6R5 (0x0b), R4G4B4A4, A4R4G4B4 and A4B4G4R4 (0x0c), each with Apple's swizzle or a reordering of it. pvr_get_format() gives the 2-byte rows that Rogue's table lacks the A9 stub entry. Tested on the A9 before the powervr interface: every format, optimal and linear, exact, with no kernel error. Assisted-by: LLM --- src/imagination/vulkan/pvr_formats.c | 10 ++++++---- src/imagination/vulkan/pvr_g5p_format_table.h | 18 ++++++++++++++++++ src/imagination/vulkan/pvr_g5p_formats.c | 19 ++++++++++++++++--- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/imagination/vulkan/pvr_formats.c b/src/imagination/vulkan/pvr_formats.c index 60f5517ff85..9c92664d401 100644 --- a/src/imagination/vulkan/pvr_formats.c +++ b/src/imagination/vulkan/pvr_formats.c @@ -240,11 +240,13 @@ pvr_get_format(struct pvr_physical_device *pdevice, VkFormat vk_format) pdevice->formats.formats[format].bind != 0) { return &pdevice->formats.formats[format]; } - /* G5P: ASTC LDR exists only in the A9's own table (Rogue's table, which - * G9P shares, has none); the G5P hooks take its features and texture - * state from that row (pvr_g5p_format_table.h) */ + /* G5P: ASTC LDR and the 16-bit packed forms Rogue lacks (A1R5G5B5, + * B5G6R5, R4G4B4A4, ...) exist only in the A9's own table (Rogue's table, + * which G9P shares, has none); the G5P hooks take their features and + * texture state from that row (pvr_g5p_format_table.h) */ if (PVR_HAS_FEATURE(&pdevice->dev_info, apple_g5p) && - vk_format_description(vk_format)->layout == UTIL_FORMAT_LAYOUT_ASTC && + (vk_format_description(vk_format)->layout == UTIL_FORMAT_LAYOUT_ASTC || + vk_format_get_blocksize(vk_format) == 2) && pvr_g5p_format_find(vk_format)) { static const struct pvr_format g5p_astc = { /* Rogue's texture-format codes are not used on G5P (its own key is) */ diff --git a/src/imagination/vulkan/pvr_g5p_format_table.h b/src/imagination/vulkan/pvr_g5p_format_table.h index f5851a2eec9..8f7bab5c983 100644 --- a/src/imagination/vulkan/pvr_g5p_format_table.h +++ b/src/imagination/vulkan/pvr_g5p_format_table.h @@ -76,6 +76,7 @@ enum pvr_g5p_fmt_evidence { PVR_G5P_FMT_EV_ICD, /* captured key, device-proven by our ICD (gles3tex, textypes) */ PVR_G5P_FMT_EV_RULE, /* derived from the decoded code rule, not captured */ PVR_G5P_FMT_EV_CAPTURE, /* captured key, never run on the device */ + PVR_G5P_FMT_EV_AGX, /* AGXMetalA9's own texFormat row (code and swizzle words), never run on the device */ }; struct pvr_g5p_format { @@ -161,6 +162,23 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { * with alpha low). */ G5P_FMT(R5G6B5_UNORM_PACK16, 2, false, G5P_SF, G5P1B, 0x0b, 2, 1, 0, G5P_ONE), G5P_FMT(B4G4R4A4_UNORM_PACK16, 2, false, G5P_SF, G5P1B, 0x0c, 1, 2, 3, 0), + /* The other 16-bit packed forms (PPSSPP's A1R5G5B5 among them), sampled and filtered, from AGXMetalA9's + * texFormat rows (word 0 the code, word 16 the sampling swizzle, + * 3-bit selectors from bit 6: 2 + component, 0 zero, 1 one) and chooseTextureFormatOBJC's MTLPixelFormat map: + * B5G6R5Unorm (40) -> RGB565 0x0b [c2 c1 c0 1] A1BGR5Unorm (41) -> RGB5A1 0x0d [c3 c2 c1 c0] + * ABGR4Unorm (42) -> RGBA4 0x0c [c3 c2 c1 c0] BGR5A1Unorm (43) -> A1RGB5 0x0e [c2 c1 c0 c3] + * (no MTLPixelFormat) BGRA4 0x0c [c2 c1 c0 c3] + * Component 0 is the lowest field: the 565 and 4444 rows above, which assume it, are exact on the device. So + * 0x0d is 1_5_5_5 (alpha bit 0) and 0x0e 5_5_5_1 (alpha bit 15); the Vulkan forms below are those rows' own + * swizzles or a reordering of them. Advertised by default since the tex16 cases (packed-texture format checks) were exact + * on the device (2026-11-14); these formats are always enabled. */ + G5P_FMT(A1R5G5B5_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0e, 2, 1, 0, 3), + G5P_FMT(R5G5B5A1_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0d, 3, 2, 1, 0), + G5P_FMT(B5G5R5A1_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0d, 1, 2, 3, 0), + G5P_FMT(B5G6R5_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0b, 0, 1, 2, G5P_ONE), + G5P_FMT(R4G4B4A4_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0c, 3, 2, 1, 0), + G5P_FMT(A4R4G4B4_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0c, 2, 1, 0, 3), + G5P_FMT(A4B4G4R4_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0c, 0, 1, 2, 3), /* integer (g3b-sample-b t5-t18, fmt-rgba8ui; textypes int-fetch and * int-sample on the device). RGBA32 UINT/SINT clear/draw/LOAD/masks * passed the integer-r4 core/compat device cases. Narrow integer render diff --git a/src/imagination/vulkan/pvr_g5p_formats.c b/src/imagination/vulkan/pvr_g5p_formats.c index 1962ddef2c7..9421cfda431 100644 --- a/src/imagination/vulkan/pvr_g5p_formats.c +++ b/src/imagination/vulkan/pvr_g5p_formats.c @@ -32,13 +32,26 @@ pvr_g5p_formats_probe(void) return false; } +/* Qualified 16-bit packed texture formats are always enabled. */ +static bool +tex16_row(VkFormat format) +{ + const struct pvr_g5p_format *f = pvr_g5p_format_find(format); + return f && f->evidence == PVR_G5P_FMT_EV_AGX && f->cpp == 2; +} + +static bool +row_probe(VkFormat format) +{ + return pvr_g5p_formats_probe() || tex16_row(format); +} bool pvr_g5p_format_tex(VkFormat format, uint32_t *key, bool *srgb) { const struct pvr_g5p_format *f = pvr_g5p_format_find(format); - if (!f || !(pvr_g5p_format_flags(f, pvr_g5p_formats_probe()) & PVR_G5P_FMT_SAMPLED)) + if (!f || !(pvr_g5p_format_flags(f, row_probe(format)) & PVR_G5P_FMT_SAMPLED)) return false; *key = f->key; *srgb = f->srgb; @@ -56,7 +69,7 @@ pvr_g5p_packed_word_pbe_key(void) bool pvr_g5p_format_pbe(VkFormat format, uint32_t *key, bool *srgb) { - return pvr_g5p_format_pbe_key(format, pvr_g5p_formats_probe(), key, srgb); + return pvr_g5p_format_pbe_key(format, row_probe(format), key, srgb); } VkFormatFeatureFlags2 @@ -66,7 +79,7 @@ pvr_g5p_color_format_features(VkFormat format, VkImageTiling tiling, VkFormatFea /* the A9's own answer; upstream's flags only say the format exists */ (void)flags; - return pvr_g5p_format_features(f, tiling == VK_IMAGE_TILING_LINEAR, pvr_g5p_formats_probe()); + return pvr_g5p_format_features(f, tiling == VK_IMAGE_TILING_LINEAR, row_probe(format)); } VkResult From 593db1cb57a23c63cc3ba3bf995902165b9b22aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 02:47:55 +0100 Subject: [PATCH 054/109] pvr/g5p: fetch 10:10:10:2 vertex formats The vertex fetch lowering already unpacks a 10:10:10:2 attribute from one 32-bit fetch with scalar NIR operations, but the format table never selected it, so these formats were not advertised for vertex buffers. Report every 4-channel 32-bit 10/10/10/2 format as one dword with packed_1010102 set. No hardware packed conversion is assumed. On the iPad this path fixed all 42 dEQP-GLES3 vertex_arrays.single_attribute *2_10_10_10 cases (34 signed, 8 unsigned; wrong results before) and kept the 24 neighbouring cases passing, with no other case lost. GLES3 reaches only the two GL types; the other Vulkan formats (A2R10G10B10 order, scaled and pure integer) take the same code but have not run on the device yet. Assisted-by: LLM --- src/imagination/vulkan/pvr_g5p_vertex_nir.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/imagination/vulkan/pvr_g5p_vertex_nir.c b/src/imagination/vulkan/pvr_g5p_vertex_nir.c index c9aeeffb838..14994e55402 100644 --- a/src/imagination/vulkan/pvr_g5p_vertex_nir.c +++ b/src/imagination/vulkan/pvr_g5p_vertex_nir.c @@ -83,8 +83,25 @@ pvr_g5p_vertex_format_info(VkFormat format, struct pvr_g5p_vertex_format *f) if (!d || d->layout != UTIL_FORMAT_LAYOUT_PLAIN || d->block.width != 1 || d->block.height != 1 || d->nr_channels < 1 || d->nr_channels > 4) return false; - /* Packed 10:10:10:2 formats (a 32-bit fetch followed by scalar NIR - * operations, f->packed_1010102) have not run on the device: not taken. */ + /* Packed 10:10:10:2 formats: a 32-bit fetch followed by ordinary scalar + * NIR operations (f->packed_1010102); no hardware packed conversion is + * assumed. Tested on the A9 before the powervr interface: all 42 + * dEQP-GLES3 vertex_arrays.single_attribute *2_10_10_10 cases fixed + * (int and uint, wrong result before), 24 controls kept, none lost. */ + if (d->nr_channels == 4 && d->block.bits == 32 && + d->channel[0].size == 10 && d->channel[1].size == 10 && + d->channel[2].size == 10 && d->channel[3].size == 2) { + f->components = 4; + f->bytes = 1; /* four logical components occupy one dword */ + f->convert = -1; + f->packed_1010102 = true; + for (unsigned c = 0; c < 4; c++) { + if (d->swizzle[c] > PIPE_SWIZZLE_W) + return false; + f->swizzle[c] = d->swizzle[c]; + } + return true; + } const struct util_format_channel_description *ch = &d->channel[0]; const unsigned bits = ch->size; if ((bits != 8 && bits != 16 && bits != 32) || d->block.bits != bits * d->nr_channels) From 8445a71a4b61d8718fdf5c695dcefc55865f3a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:42 +0100 Subject: [PATCH 055/109] pco/g5p: preserve the qualified kill order for tile-reading programs Retain the proven production byte order while the masked and general blend paths place their deferred kill after the tile read. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 69 +++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 028e53d3dfb..e32dbe5c1ed 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -183,6 +183,9 @@ struct gctx { nir_def *discards[16]; /* demote_if/terminate_if conditions, emitted before the output */ unsigned ndiscards; bool discard_always; + /* the kill held for the destination-reading epilogue (fs_output(): Apple's order, the tile read + * issued before the kill): -2 none, -1 unconditional, else the condition register */ + int deferred_kill; int fragcoord; /* dword of the raw X (Y at +1), -1 when unused */ bool remat; /* register-pressure retry: constants/uniforms re-read per use */ bool count_pass; /* round 19: first codegen pass only records register reads */ @@ -5577,30 +5580,73 @@ static void fs_prologue(struct gctx *c) /* Discards run after the body (no derivative work follows them): a test * of the 0.0/1.0 condition against 0.0 feeding `04 61`, or `04 68`. */ -static void fs_discards(struct gctx *c) +static void fs_discards(struct gctx *c, bool defer) { tex_flush(c); uint8_t raw[G5P_USC_MAX]; c->p->discard_tokens = c->ndiscards + c->discard_always; + c->deferred_kill = -2; for (unsigned i = 0; i < c->ndiscards; i++) { struct cond k = cond_of(c, c->discards[i], 0); unsigned b = k.zero_b ? reg_of(c, k.a, "discard condition") : select_into(c, k, NULL, K_ONE, NULL, K_ZERO); - unsigned t = alloc_reg(c, -1); + /* held past the output moves (halves 0..7: the colour, then the epilogue's tile read at + * half 4): a register from dword 4 up */ + int t = -1; + /* above the colour halves (dwords 0..1), the tile read (2..3) and dual-source outputs (halves 8..11) */ + for (unsigned r = c->has_secondary ? 6 : 4; defer && t < 0 && r < 32; r++) + if (!((c->busy | c->reserved | c->tex_coord_pending | c->tex_pending) & BITFIELD64_BIT(r))) + t = alloc_reg(c, r); + if (t < 0) + t = alloc_reg(c, -1); + if (defer && t < (c->has_secondary ? 6 : 4)) + refuse(c, "no free register above the outputs for the held discard condition"); unsigned z = zero_reg(c); unsigned gb = k.zero_b ? vgrp(c, k.a) : 1, gz = kgrp(c, K_ZERO); put(c, raw, g5p_usc_discard_test(raw, G5P_FEQ, 2 * t, S(2 * b, gb), S(2 * z, gz), 0), "discard test"); - put(c, raw, g5p_usc_discard(raw, 2 * t), "discard"); - c->busy &= ~BITFIELD64_BIT(t); + if (defer) { + c->deferred_kill = t; /* stays busy until fs_kill() */ + } else { + put(c, raw, g5p_usc_discard(raw, 2 * t), "discard"); + c->busy &= ~BITFIELD64_BIT(t); + } free_scratch(c); c->p->evidence |= PCO_G5P_EV_DISCARD; } if (c->discard_always) { - put(c, raw, g5p_usc_discard(raw, -1), "discard"); + if (defer) + c->deferred_kill = -1; + else + put(c, raw, g5p_usc_discard(raw, -1), "discard"); c->p->evidence |= PCO_G5P_EV_DISCARD; } } +/* A held kill (fs_discards(defer)), in Apple's place: after the epilogue's tile read is issued, + * before its wait. A9 AGXMetalA9 compiles every destination-reading discarding program that way + * (three Metal probes of discarding programs with a blended or RGB-masked colour output: the tile read, + * then the kill, the wait for the tile read later), PUNCH_THROUGH and FAST_PUNCH_THROUGH alike; ours issued + * the kill first and read the tile after it. @epi: the epilogue's bytes; with no leading tile read + * the kill goes first, the old order. Used by masked and general-blend programs. */ +static void fs_put_epilogue(struct gctx *c, const uint8_t *epi, unsigned n, const char *what) +{ + uint8_t raw[G5P_USC_MAX]; + if (c->deferred_kill == -2) { + put(c, epi, n, what); + return; + } + const unsigned read = n >= 4 && epi[0] == 0x1c && epi[3] == 0x28 ? 4 + : n >= 6 && epi[0] == 0x1c && epi[3] == 0x29 ? 6 : 0; + if (read) + put(c, epi, read, "tile read (before the kill: Apple's order)"); + put(c, raw, g5p_usc_discard(raw, c->deferred_kill < 0 ? -1 : 2 * c->deferred_kill), "discard"); + if (c->deferred_kill >= 0) + c->busy &= ~BITFIELD64_BIT(c->deferred_kill); + c->deferred_kill = -2; + if (n > read) + put(c, epi + read, n - read, what); +} + static void fs_output(struct gctx *c) { tex_flush(c); @@ -5614,7 +5660,12 @@ static void fs_output(struct gctx *c) put(c, raw, g5p_usc_depth_feedback(raw, 2 * reg), "fragment depth feedback"); c->p->depth_feedback = true; } - fs_discards(c); + /* a discarding program whose epilogue reads the tile: the kill waits for that read (fs_put_epilogue()) */ + const bool defer = kill_after_tile_read() && c->ndiscards + c->discard_always == 1 && + c->key->color_output == PCO_G5P_COLOR_F16 && !compact_output(c->key) && + !c->key->mrt_blend_state && any_color(c) && + (c->blend == 1 || c->blend == 2 || c->blend == BLEND_GENERAL); + fs_discards(c, defer); c->now = LAST_END; if (compact_output(c->key)) { /* target t's packed dword: output word t (c->out[4t], lower_compact_store()) */ @@ -5739,7 +5790,7 @@ static void fs_output(struct gctx *c) unsigned n = g5p_blend_epilogue(buf, sizeof(buf)); if (!n) refuse(c, "blend epilogue encoder rejected its fields (%s)", g5p_usc_error ? g5p_usc_error : "?"); - put(c, buf, n, "premultiplied blend epilogue"); + fs_put_epilogue(c, buf, n, "premultiplied blend epilogue"); c->max_dword = MAX2(c->max_dword, 7); /* destination halves 4..7, 1-alpha in half 12 */ c->p->evidence |= PCO_G5P_EV_BLEND; } else if (c->blend == BLEND_GENERAL) { @@ -5771,11 +5822,13 @@ static void fs_output(struct gctx *c) if (err) refuse(c, "general blend epilogue: %s", err); if (n) { - put(c, buf, n, "general blend epilogue"); + fs_put_epilogue(c, buf, n, "general blend epilogue"); c->max_dword = MAX2(c->max_dword, c->binfo.max_half / 2); } c->p->evidence |= PCO_G5P_EV_BLEND; } + if (c->deferred_kill != -2) /* no epilogue took the held kill (a general epilogue of no bytes) */ + fs_put_epilogue(c, NULL, 0, "no epilogue"); /* One emitpix carries the total word count over all targets (2/4/8). */ put(c, raw, g5p_usc_emit_mrt(raw, c->mrt.words, 0), "emit"); put(c, raw, g5p_usc_special(raw, true), "end"); From 3c434ba6881be19dc9ba6a3c7c6a1a4752ebbecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:42 +0100 Subject: [PATCH 056/109] pvr/g5p: Apple's masked form for an RGB write mask, by default Apple's A9 driver compiles a program under colour write mask RGB with blending off (BGRA8, one sample) as kill mode 1/0 with no destination term in its colour math: FAST_PUNCH_THROUGH when it discards, else OPAQUE, ISPCTL upass 7, and the program reads the tile's alpha word before its kill. Ours forced every partial mask to the general (tile-reading) epilogue run TRANSLUCENT, and refused it with a discard. A one-target 1x RGBA8/BGRA8 pipeline with write mask RGB and no blending now takes Apple's form (ISPCTL upass 7; pass type FAST_PUNCH_THROUGH when it discards). Tested on the A9 before the powervr interface: three passes of 48 overlapping depth-tested draws over a D32F_S8 target (opaque, masked discard, masked opaque, premultiplied), 300 s each mode, every frame exact against a CPU model, no kernel error. Apple's pass type for a discarding masked draw is not captured yet. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 4 ++-- src/imagination/g5p/pco_g5p_general.h | 6 ++++++ src/imagination/vulkan/pvr_arch_g5p_gfx.c | 24 ++++++++++++++++++++--- src/imagination/vulkan/pvr_g5p.h | 1 + 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index e32dbe5c1ed..61a53c1fdb9 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -5661,7 +5661,7 @@ static void fs_output(struct gctx *c) c->p->depth_feedback = true; } /* a discarding program whose epilogue reads the tile: the kill waits for that read (fs_put_epilogue()) */ - const bool defer = kill_after_tile_read() && c->ndiscards + c->discard_always == 1 && + const bool defer = (c->key->mask_upass) && c->ndiscards + c->discard_always == 1 && c->key->color_output == PCO_G5P_COLOR_F16 && !compact_output(c->key) && !c->key->mrt_blend_state && any_color(c) && (c->blend == 1 || c->blend == 2 || c->blend == BLEND_GENERAL); @@ -5811,7 +5811,7 @@ static void fs_output(struct gctx *c) * iPad in Aquarium with that form under depth writes (not bisected * yet), so it is off: such a program is refused at pipeline * creation, the r39-6 rule. */ - if (c->ndiscards || c->discard_always) + if ((c->ndiscards || c->discard_always) && c->key->mask_upass != 1) refuse(c, "discard in a program with a general blend epilogue " "(general-blend punch-through is off)"); uint8_t buf[512]; diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index ba998af668f..d66261b06e0 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -120,7 +120,13 @@ struct pco_g5p_fs_key { bool packed_sint; /* saturate signed channels, then preserve low bits */ bool packed_scalar8; /* R8: one scalar source; never pack unused G/B/A */ uint8_t packed_uint_bits; /* 8/10/16: packed output, scalar8 is one word; full mask only */ + /* Apple's masked form (enabled): no blending, write mask RGB, one 4-byte UNORM8 target, one + * sample. AGXMetalA9 runs it OPAQUE (FAST_PUNCH_THROUGH with a discard) with ISPCTL upass 7, the program + * keeping the destination alpha from a tile read issued before its kill. 1: the program may discard; + * The qualified masked form is selected at pipeline creation. */ + uint8_t mask_upass; }; +#define PCO_G5P_HAS_MASK_UPASS 1 #define PCO_G5P_FS_KEY_DEFAULT ((struct pco_g5p_fs_key){.write_mask = 0xf}) enum pco_g5p_upload_kind { diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index fa4eba270a5..3d92421a36b 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -533,6 +533,17 @@ g5p_check_state(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *c * discarding program still runs for its feedback) */ if (!colors) key->write_mask = 0; + /* Apple's masked form (discarding and non-discarding shaders under write mask RGB, + * no blending, BGRA8: kill mode 1 / 0, A none, B 7 -> FAST_PUNCH_THROUGH / OPAQUE, output mask 7; the program + * reads the tile's alpha word, then kills, and moves it to its alpha half). Enabled by default: on the + * device, masked RGB, overlap and mixed-rendering tests pass (300 s each, every frame exact, + * 0 kernel errors); without it a masked discard is skipped and a game scene + * stays black. */ + const unsigned mask_mode = 1; + key->mask_upass = mask_mode && colors == 1 && !key->blend && + key->write_mask == 0x7 && key->color_output == PCO_G5P_COLOR_F16 && + (cf == VK_FORMAT_B8G8R8A8_UNORM || cf == VK_FORMAT_R8G8B8A8_UNORM) && + (!s->ms || s->ms->rasterization_samples == VK_SAMPLE_COUNT_1_BIT) ? mask_mode : 0; return VK_SUCCESS; } @@ -582,6 +593,7 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo if (result != VK_SUCCESS) return result; key.punch = g->punch_variant; + g->fs_mask_upass = key.mask_upass; g->samples = s->ms ? MAX2(s->ms->rasterization_samples, VK_SAMPLE_COUNT_1_BIT) : VK_SAMPLE_COUNT_1_BIT; g->bresenham_line_probe = g5p_native_bresenham_line(s); @@ -972,7 +984,7 @@ g5p_pipeline_build_punch(struct pvr_device *device, * in Aquarium with it, cause not yet bisected. Without the variant, * g5p_pipeline_check_draw_variants() refuses the pipeline at creation * when a depth/stencil state could need it, the non-fatal form r39-3 had. */ - if (g->fs_blend_mode == 4 /* general */) { + if (g->fs_blend_mode == 4 /* general */ && !g->fs_mask_upass) { g->punch_why = "general blend epilogue"; return VK_SUCCESS; } @@ -2074,6 +2086,12 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g { struct pvr_g5p_blend_isp isp; pvr_arch_g5p_blend_isp(g->fs_blend_mode, g->fs_discard, g->fs_write_mask, discard, &isp); + /* Apple's masked form (pco_g5p_fs_key::mask_upass): no destination term in the colour math, so OPAQUE, or + * FAST_PUNCH_THROUGH with a discard, and output mask 7 */ + if (g->fs_mask_upass && !discard) + isp.pass_type = !g->fs_discard ? ROGUE_TA_PASSTYPE_OPAQUE + : ROGUE_TA_PASSTYPE_FAST_PUNCH_THROUGH; + const unsigned upass = g->fs_mask_upass && !discard ? 7 : pvr_g5p_msaa_upass(g->samples); const enum ROGUE_TA_PASSTYPE pass = g->fs_depth_feedback ? ROGUE_TA_PASSTYPE_DEPTH_FEEDBACK : isp.pass_type; /* points rasterize as sprites (g3b-points: SPRITE_01UV, front and back) */ /* triangles in line/point polygon mode: AGXMetalA9 encodeISPPrimitiveType @@ -2102,7 +2120,7 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g pvr_csb_pack (&w[1], TA_STATE_ISPCTL, ctl) { /* 0 for a program that writes no colour (pvr_g5p_blend.h), else the sample rate's user pass * (pvr_g5p_msaa.h: 15, or 7 at 4x) */ - ctl.upass = isp.upass ? pvr_g5p_msaa_upass(g->samples) : 0; + ctl.upass = isp.upass ? upass : 0; ctl.bpres = true; ctl.scenable = scissor; ctl.tagwritedisable = isp.tag_write_disable; @@ -2134,7 +2152,7 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g * Mask-zero kill programs still need feedback, with upass0. */ struct ROGUE_TA_STATE_ISPCTL ctl; ROGUE_TA_STATE_ISPCTL_unpack(&w[1], &ctl); - ctl.upass = isp.upass ? pvr_g5p_msaa_upass(g->samples) : 0; + ctl.upass = isp.upass ? upass : 0; ctl.tagwritedisable = isp.tag_write_disable; /* Rogue TA_STATE_ISPCTL: VISTEST enables accumulated sample counting; * VISREG selects the counter in ISP_OCLQRY_BASE. Match the generic diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index c0e1f952565..22be560a6e2 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -278,6 +278,7 @@ struct pvr_g5p_pipeline { struct pvr_g5p_pipeline *punch; bool punch_variant; const char *punch_why; /* no punch variant: why (static string; g5p_pipeline_build_punch()) */ + bool fs_mask_upass; /* Apple's masked form: OPAQUE / FAST_PUNCH_THROUGH, upass 7 (pco_g5p_fs_key::mask_upass) */ /* the program writes memory (pco_g5p_program::side_effects): a fragment * program's draws bound the shaded pixels with the ISP scissor; a vertex * program's draws run even when no pixel is left (pvr_arch_g5p_gfx.c) */ From 97f25fd9c9365c7527d66aeab548ecaa7f74de50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:43 +0100 Subject: [PATCH 057/109] pco/g5p: the 2D-multisample sample, its own-sample operand and the 4x discard test Three encoder forms from Apple's A9 output: - G5P_TEX_2D_MS: the narrow sample of a 2D multisample state, its sample operand in the register after the coordinates (the 4x LOAD background of an iOS capture). Only that narrow form, with coordinate control, is admitted. - g5p_usc_ms_own_sample(): the operand "the task's own sample" (Apple's 4x loader, and texture2d_ms::read(c, sample_id)). It has the short move's layout, with the source kind "special value" and that value's index where the immediate move has a literal. - g5p_usc_discard_test_ms4(): the discard test at 4 raster samples, a packed ternary: -0.0 as literal A, the tested value as B, the all-samples mask as literal C, the float-equality opcode and the != relation. Like the one-sample test it kills where the condition is not 0.0. Every field of the two forms is a named constant or an argument. Assisted-by: LLM --- src/imagination/g5p/g5p_usc.c | 95 +++++++++++++++++++++++++++++++++-- src/imagination/g5p/g5p_usc.h | 22 +++++++- 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/src/imagination/g5p/g5p_usc.c b/src/imagination/g5p/g5p_usc.c index 261157f5726..dd0c9bab948 100644 --- a/src/imagination/g5p/g5p_usc.c +++ b/src/imagination/g5p/g5p_usc.c @@ -15,7 +15,7 @@ const char *const g5p_binop_name[G5P_BIN_COUNT] = {"fadd", "fmul", "iadd", "imul const char *const g5p_minmax_name[G5P_MINMAX_COUNT] = {"umin", "umax", "imin", "imax"}; const char *const g5p_convert_name[G5P_CONVERT_COUNT] = {"f2i32", "f2u32", "i2f32", "u2f32", "f2i32_rte"}; const char *const g5p_unary_name[G5P_UNARY_COUNT] = {"floor", "log2", "exp2", "not", "popcount", "bitreverse"}; -const char *const g5p_tex_dim_name[G5P_TEX_DIM_COUNT] = {"2d", "2d_array", "3d", "cube"}; +const char *const g5p_tex_dim_name[G5P_TEX_DIM_COUNT] = {"2d", "2d_array", "3d", "cube", "2d_ms"}; const char *const g5p_cmp_name[G5P_CMP_COUNT] = {"flt", "fgt", "feq", "ilt", "igt", "ult", "ugt", "ieq"}; const char *const g5p_sfu_name[G5P_SFU_COUNT] = {"rcp_approx", "rsqrt_approx", "sin_core", "cos_core"}; @@ -26,7 +26,7 @@ static const uint8_t convert_code[G5P_CONVERT_COUNT] = {0x60, 0x60, 0x68, 0x68, static const uint8_t cmp_code[G5P_CMP_COUNT] = {0x59, 0x5d, 0x55, 0xbe, 0xbf, 0xb6, 0xb7, 0xbd}; static const uint8_t sfu_code[G5P_SFU_COUNT] = {0x80, 0x81, 0x82, 0x86}; static const uint8_t unary_code[G5P_UNARY_COUNT] = {0x40, 0x84, 0x85, 0xc5, 0xe8, 0xe4}; -static const uint8_t dim_code[G5P_TEX_DIM_COUNT] = {0xf5, 0xf7, 0xfb, 0xfd}; +static const uint8_t dim_code[G5P_TEX_DIM_COUNT] = {0xf5, 0xf7, 0xfb, 0xfd, 0xf9}; static const uint8_t mov_trailer[4] = {0x54, 0x01, 0x38, 0x00}; /* Split immediate map (g5p_isa.IMM_MAP): bit -> (byte, bit). */ @@ -988,6 +988,8 @@ static unsigned texture_base(uint8_t *raw, const struct g5p_tex *t) CHECK(t->class_variant <= 1, "class_variant must be an integer in [0,1]"); CHECK(t->dim < G5P_TEX_DIM_COUNT, "unmeasured texture dimension"); CHECK(!t->read || t->dim == G5P_TEX_2D, "only 2d read has been isolated"); + CHECK(t->dim != G5P_TEX_2D_MS || (!t->read && !t->class_variant), + "the 2D-multisample fetch is witnessed only as the narrow class0 sample"); raw[0] = 0x1a | t->class_variant; raw[1] = dim_code[t->dim]; raw[2] = t->dst_half | (t->read ? 0x40 : 0); @@ -1003,13 +1005,53 @@ unsigned g5p_usc_sample(uint8_t *out, const struct g5p_tex *t) uint8_t raw[G5P_USC_MAX]; if (!texture_base(raw, t)) return 0; + CHECK(t->dim != G5P_TEX_2D_MS || (t->narrow && t->coord_control), + "the 2D-multisample sample is witnessed only narrow with coordinate control"); if (t->narrow) { - CHECK(!t->class_variant && t->dim == G5P_TEX_2D, "narrow sample witnessed only as 2d class0"); + CHECK(!t->class_variant && (t->dim == G5P_TEX_2D || t->dim == G5P_TEX_2D_MS), + "narrow sample witnessed only as 2d and 2d-multisample class0"); raw[0] &= ~2; } return finish(out, raw, 6); } +/* A move of a special value: the layout of g5p_usc_immediate()'s short move, with the source kind "special + * value" where the immediate sets "literal". Fields, by byte: + * 0 the single-source class of g5p_usc_move() and g5p_usc_immediate(); + * 1 the header: only the continuation into the destination parcel; + * 2 the destination half, 3 its bank (destination()); + * 4 bit 0: slot A is not a register; bits 1..6: the special value's index (where the immediate keeps its + * bits 0..5); bit 7: zero (the register form's group bit); + * 5 slot B unused (selector 0) with the continuation; + * 6 the source kind (G5P_MOV_SOURCE_*); the immediate's long-form bit 0 and literal bits stay zero; + * 7 the continuation, the immediate's literal bits zero; + * 8 the move's trailer (mov_trailer, as g5p_usc_move()). + * The A9 compiler emits this form for two indices: G5P_SPECIAL_OWN_SAMPLE, and 24 for the primitive's facing, + * which no encoder here takes. */ +#define G5P_CLASS_MOVE 0x98 +#define G5P_SRC_A_NOT_REGISTER 0x01 +#define G5P_SRC_B_UNUSED 0x00 +#define G5P_MOV_SOURCE_SPECIAL 0x10 /* observed with the two indices above; the immediate sets 0x20 */ +#define G5P_SPECIAL_OWN_SAMPLE 12 /* the sample the task shades */ + +unsigned g5p_usc_ms_own_sample(uint8_t *out, unsigned dst_half) +{ + CHECK(half_register(dst_half, "the sample operand's destination must be a 32-bit aligned half index"), + g5p_usc_error); + const unsigned dst_bank = 0; + uint8_t raw[12]; + raw[0] = G5P_CLASS_MOVE; + raw[1] = G5P_HEADER_FRAGMENT; + if (!destination(raw + 2, dst_half, dst_bank)) + return 0; + raw[4] = G5P_SRC_A_NOT_REGISTER | G5P_SPECIAL_OWN_SAMPLE << 1; + raw[5] = G5P_SRC_B_UNUSED | G5P_PARCEL_CONTINUES; + raw[6] = G5P_MOV_SOURCE_SPECIAL; + raw[7] = G5P_PARCEL_CONTINUES; + memcpy(raw + 8, mov_trailer, sizeof(mov_trailer)); + return finish(out, raw, 12); +} + unsigned g5p_usc_texture_lod(uint8_t *out, const struct g5p_tex *t) { CHECK(t->lod_bank == 1 || t->lod_bank == 3, "only LOD source banks 1 (shared) and 3 (temp) are measured"); @@ -1062,7 +1104,7 @@ unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t) CHECK((unsigned)t->tuple <= G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL_COMPARE_OFFSET, "unqualified texture recipe"); CHECK(!coupled || t->lod_shared == (t->tuple == G5P_TEX_TUPLE_COUPLED_SHARED_LEVEL), "unqualified independent texture continuation choices"); - static const uint8_t dim_low[G5P_TEX_DIM_COUNT] = {0x5, 0x7, 0xb, 0xd}; + static const uint8_t dim_low[G5P_TEX_DIM_COUNT] = {0x5, 0x7, 0xb, 0xd, 0x9}; static const uint8_t lod_mode[] = {0, 0x40, 0x20, 0x60}; CHECK(half_register(t->dst_half, "dst_half requires a measured 32-bit aligned half index"), g5p_usc_error); /* A9 Metal v_images31 and v_units16_sample witness the complete 7-bit @@ -1071,6 +1113,7 @@ unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t) CHECK(t->sampler_qword <= 124, "sampler_qword must be an integer in [0,124]"); CHECK(t->coord_dword <= 31 && t->lod_dword <= 31 && t->extras_half <= 63, "operand beyond r31"); CHECK(t->dim < G5P_TEX_DIM_COUNT, "unmeasured texture dimension"); + CHECK(t->dim != G5P_TEX_2D_MS, "no general-form 2D-multisample fetch is witnessed"); CHECK(t->mask == 0xf || (t->mask == 1 && t->compare), "only rgba and compare results are measured"); CHECK(!t->read || (!t->compare && !t->offset && t->lod == G5P_TEX_LOD_LEVEL && t->dim != G5P_TEX_CUBE), "read takes a level and no compare/offset (cube reads are refused by Apple)"); @@ -1446,6 +1489,50 @@ unsigned g5p_usc_discard_test(uint8_t *out, enum g5p_cmp op, unsigned dst_half, return finish(out, raw, n); } +/* The 4x discard test is a packed ternary of g5p_usc_literal_ternary()'s layout: the 16-bit-result class, + * A a long literal (literal_prefix()), B the tested value, C a short literal, then the opcode, the operation + * control and a last parcel of routing and relation. Its values: + * A -0.0f, the literal Apple's compiler compares against zero with (as the rsqrt and sign recipes); + * C the kill's sample mask, all four samples; + * opcode the float-equality compare of g5p_usc_compare_write(); + * control bit 1 the 16-bit result (as g5p_usc_binary()'s result_half flag); bit 4 observed with a short C + * literal (set in this form and in the cos-scale and bool-to-half recipes, clear in the rsqrt and + * sign recipes' long C literals; the rcp-refine recipe also sets it, with a register C); bit 0 the + * continuation into the last parcel; + * routing the low byte, the routing of the literal forms (g5p_usc_binary_literal2()); + * relation the high byte, as relation_control in g5p_usc_discard_test() and g5p_usc_compare_write(): 0, so + * with this opcode the test is A != B, as Apple's compiler emits it for (v != 0.0f) discard. The == + * relation (0x10) killed the samples a discard should keep on the device. Its continuation bit is + * clear: the parcel ends the instruction. */ +#define G5P_F32_NEG_ZERO 0x80000000u +#define G5P_MS4_ALL_SAMPLES 0xfu +#define G5P_TERNARY_CTL_C_SHORT 0x10 +#define G5P_ROUTING_LITERAL 0x70 +#define G5P_RELATION_NE 0x00 + +unsigned g5p_usc_discard_test_ms4(uint8_t *out, unsigned dst_half, struct g5p_src a) +{ + CHECK(!a.neg && !a.abs, "the 4x discard test is witnessed without source modifiers"); + const unsigned dst_bank = 0; + const int a_long = 1, c_long = 0; + uint8_t raw[G5P_USC_MAX], c[G5P_USC_MAX]; + unsigned n = literal_prefix(raw, G5P_CLASS_TERNARY_RESULT_HALF, dst_half, dst_bank, G5P_F32_NEG_ZERO, a.half, + a.group, G5P_HEADER_FRAGMENT, a_long); + if (!n) + return 0; + /* C's literal: its parcels from byte 4 on of a literal prefix (as g5p_usc_literal_ternary()). */ + unsigned m = literal_prefix(c, 0, 0, 0, G5P_MS4_ALL_SAMPLES, 0, 0, G5P_HEADER_FRAGMENT, c_long); + if (!m) + return 0; + memcpy(raw + n, c + 4, m - 4); + n += m - 4; + raw[n++] = cmp_code[G5P_FEQ]; + raw[n++] = G5P_TERNARY_CTL_RESULT_HALF | G5P_TERNARY_CTL_C_SHORT | G5P_PARCEL_CONTINUES; + raw[n++] = G5P_ROUTING_LITERAL; + raw[n++] = G5P_RELATION_NE; + return finish(out, raw, n); +} + unsigned g5p_usc_discard(uint8_t *out, int src_half) { if (src_half < 0) { diff --git a/src/imagination/g5p/g5p_usc.h b/src/imagination/g5p/g5p_usc.h index 82d98a13609..2412607764d 100644 --- a/src/imagination/g5p/g5p_usc.h +++ b/src/imagination/g5p/g5p_usc.h @@ -38,7 +38,10 @@ enum g5p_small_format { G5P_SMALL_FORMAT_COUNT }; enum g5p_unary { G5P_FLOOR, G5P_LOG2, G5P_EXP2, G5P_NOT, G5P_POPCOUNT, G5P_BITREVERSE, G5P_UNARY_COUNT }; -enum g5p_tex_dim { G5P_TEX_2D, G5P_TEX_2D_ARRAY, G5P_TEX_3D, G5P_TEX_CUBE, G5P_TEX_DIM_COUNT }; +/* G5P_TEX_2D_MS (code 0xf9): the 2D-multisample fetch, its sample operand in the register after the + * coordinates (Apple's 4x LOAD background and Metal texture2d_ms probes); only the narrow + * sample (g5p_usc_sample()) takes it. */ +enum g5p_tex_dim { G5P_TEX_2D, G5P_TEX_2D_ARRAY, G5P_TEX_3D, G5P_TEX_CUBE, G5P_TEX_2D_MS, G5P_TEX_DIM_COUNT }; /* ISA round 2 (isa-g5p.md "ISA round 2"): compare/write relations, SFU * cores and the literal kinds of encode_binary_literal. */ enum g5p_cmp { G5P_FLT, G5P_FGT, G5P_FEQ, G5P_ILT, G5P_IGT, G5P_ULT, G5P_UGT, G5P_IEQ, G5P_CMP_COUNT }; @@ -157,6 +160,12 @@ struct g5p_tex { unsigned length; /* 0 = minimum */ }; unsigned g5p_usc_sample(uint8_t *out, const struct g5p_tex *t); +/* The 2D-multisample fetch's sample operand "the task's own sample" (a sample-rate program reads the + * sample it shades): a 12-byte move of the special value with that index into dst_half, bank 0 (the fields + * are listed at the encoder). It is not g5p_usc_immediate()'s move: its source kind is "special value" where + * the immediate's is "literal". Witnesses: the 4x LOAD background (an iOS capture of a 4x colour load, half + * 4) and texture2d_ms::read(c, sample_id) (a Metal probe, half 12, which the fetch reads unchanged). */ +unsigned g5p_usc_ms_own_sample(uint8_t *out, unsigned dst_half); unsigned g5p_usc_texture_lod(uint8_t *out, const struct g5p_tex *t); unsigned g5p_usc_gather(uint8_t *out, const struct g5p_tex *t); @@ -300,6 +309,15 @@ unsigned g5p_usc_sfu_core(uint8_t *out, enum g5p_sfu op, unsigned dst_half, stru unsigned g5p_usc_square(uint8_t *out, unsigned dst_half, struct g5p_src s, struct g5p_ctl k); unsigned g5p_usc_discard_test(uint8_t *out, enum g5p_cmp op, unsigned dst_half, struct g5p_src a, struct g5p_src b, unsigned relation_control); +/* The discard test at 4 raster samples: A != 0.0 writes the kill's sample mask 15 (all four samples) into the + * destination, where the one-sample test writes 1 (our one-sample g5p_usc_discard_test() kills where A != 0.0 + * too). Apple's compiler, the same fragment program under raster_sample_count 4 (if (v != 0.0f) + * discard_fragment(), v in h0 and, read again, h4): a packed ternary with the 16-bit result, -0.0f as a long + * literal A, v as B and the mask 15 as a short literal C, the float-equality opcode and the != relation (the + * fields are listed at the encoder). At one sample the same compiler writes the mask 1 with a two-source form. + * With the == relation the form killed the samples it should keep on the device. The kill that follows is + * g5p_usc_discard() of the destination; an unconditional discard is g5p_usc_discard(-1) at 4 samples as at one. */ +unsigned g5p_usc_discard_test_ms4(uint8_t *out, unsigned dst_half, struct g5p_src a); unsigned g5p_usc_discard(uint8_t *out, int src_half); /* Fragment depth replacement (A9 depth(any) compiler witnesses). */ unsigned g5p_usc_depth_feedback(uint8_t *out, unsigned src_half); @@ -457,7 +475,7 @@ unsigned g5p_usc_address_add(uint8_t *out, unsigned dst_half, unsigned ptr_dword * `10 (0x23+8(n-1)) (0x80|data_half) (addr_half<<1|1) OO 88`. * Vertex tasks commit only the 4-byte form (xfb.md, device run 2). */ /* Host fragment store predicate and class70/71 memory side effect; - * helper-lane interpretation is inferred, pending graphics-storage rungs. */ + * helper-lane interpretation is inferred, pending graphics-storage cases. */ unsigned g5p_usc_fs_store_predicate(uint8_t *out); unsigned g5p_usc_fs_store(uint8_t *out,unsigned bytes,unsigned data_half,unsigned addr_half,unsigned slot); unsigned g5p_usc_global_store(uint8_t *out, unsigned bytes, unsigned data_half, unsigned addr_half, From ad4d2689207de009117498acf7d5e377c2c090d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:43 +0100 Subject: [PATCH 058/109] pvr/g5p: 4x LOAD, 4x depth/stencil load and store, 4x discard, as Apple From an iOS capture and Apple's offline A9 compiler: - LOAD of a 4-sample colour target uses Apple's 4x loader: it samples the target at the pixel centre with the own-sample operand and the narrow f9 sample, kicked at sample rate. Its texture state is the store's surface, W x H pixels, 4x twiddled layout. With a PBE resolve the loader reads the multisample source; a partial area or a VDM continuation with a resolve is refused. - 4x depth/stencil: Apple's 4x ZLS words are the one-sample words over the 4x plane, which the A9 path already builds. The 4x load/store refusals go. - 4x discard: a program's one conditional discard kills the four samples with the 4x test, run FAST_PUNCH_THROUGH as at one sample (ISPCTL upass 7). Several discards keep the one-sample test, and their 4x draws are refused by name. Programs at one sample are byte-identical. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 41 +++++++++++++++ src/imagination/g5p/pco_g5p_general.h | 9 ++++ src/imagination/vulkan/pvr_arch_g5p.c | 61 ++++++++++++++++++---- src/imagination/vulkan/pvr_arch_g5p_ds.c | 16 +----- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 19 ++++--- src/imagination/vulkan/pvr_arch_g5p_msaa.c | 10 ++-- src/imagination/vulkan/pvr_g5p.h | 5 ++ src/imagination/vulkan/pvr_g5p_ds.c | 12 ++--- src/imagination/vulkan/pvr_g5p_msaa.c | 12 +++++ src/imagination/vulkan/pvr_g5p_msaa.h | 9 +++- src/imagination/vulkan/pvr_g5p_nir.c | 46 ++++++++++++++++ 11 files changed, 194 insertions(+), 46 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 61a53c1fdb9..dab16f472da 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -5111,6 +5111,35 @@ static void emit_vertex_qualified_texture(struct gctx *c, nir_tex_instr *t) c->p->evidence |= PCO_G5P_EV_TEX_TYPES; } +/* The 4x LOAD background's sample (key ms_own_sample): Apple's loader form. The coordinates in a + * three-register run, the own-sample operand in its third register (the fetch reads it implicitly, so it is + * part of the run), then the narrow sample of dimension G5P_TEX_2D_MS with coordinate control. Only the narrow form + * is witnessed: anything else is refused. */ +static void emit_tex_ms_own_sample(struct gctx *c, nir_tex_instr *t, struct tex_bind *b, bool narrow) +{ + uint8_t raw[G5P_USC_MAX]; + if (!narrow || varying_coord_slot(c, t) >= 0 || c->mrt.targets != 1) { + refuse(c, "the 4x LOAD sample: only Apple's narrow form with computed coordinates is witnessed"); + return; + } + tex_flush(c); + nir_def *cd = nir_get_tex_src(t, nir_tex_src_coord); + struct ref x = resolve_value(c, cd, 0), y = resolve_value(c, cd, 1); + unsigned xr = reg_of(c, x, "texture coordinate"), yr = reg_of(c, y, "texture coordinate"); + const unsigned coord = alloc_run(c, 3, "multisample coordinates and sample operand"); + put(c, raw, g5p_usc_move(raw, 2 * coord, 2 * xr, 0, vgrp(c, x), 1), "coordinate x"); + put(c, raw, g5p_usc_move(raw, 2 * coord + 2, 2 * yr, 0, vgrp(c, y), 1), "coordinate y"); + put(c, raw, g5p_usc_ms_own_sample(raw, 2 * coord + 4), "the task's own sample"); + c->scratch |= BITFIELD64_RANGE(coord, 3); + struct g5p_tex s = {.image_qword = b->image / 2, .sampler_qword = b->sampler / 2, .coord_dword = coord, + .coord_control = true, .dim = G5P_TEX_2D_MS, .narrow = true, .dst_half = 0}; + put(c, raw, g5p_usc_sample(raw, &s), "texture sample (2D multisample)"); + c->tex_coord_pending |= BITFIELD64_RANGE(coord, 3); + c->tex_batch++; + tex_flush(c); + c->p->evidence |= PCO_G5P_EV_TEX_CAPTURE; +} + static void emit_tex(struct gctx *c, nir_tex_instr *t) { if (c->tex_batch == 4 || c->vs || !plain_tex(t) || size_query(t) || c->narrow_tex == t) @@ -5132,6 +5161,10 @@ static void emit_tex(struct gctx *c, nir_tex_instr *t) int slot = varying_coord_slot(c, t); bool narrow = c->narrow_tex == t; unsigned coord; + if (c->key && c->key->ms_own_sample) { + emit_tex_ms_own_sample(c, t, b, narrow); + return; + } if (slot >= 0) { tex_flush(c); coord = narrow ? 0 : alloc_run(c, 2, "texture coordinate"); @@ -5601,9 +5634,17 @@ static void fs_discards(struct gctx *c, bool defer) t = alloc_reg(c, -1); if (defer && t < (c->has_secondary ? 6 : 4)) refuse(c, "no free register above the outputs for the held discard condition"); + if (c->key && c->key->ms4_kill && c->ndiscards == 1 && !c->discard_always) { + /* 4 samples, one conditional discard: b != 0.0 writes the kill's sample mask 15 (Apple's 4x form, the + * same relation as the one-sample test below). + * Several discards keep the one-sample test; their draws are refused at 4x (pvr_arch_g5p_gfx.c). */ + unsigned gb = k.zero_b ? vgrp(c, k.a) : 1; + put(c, raw, g5p_usc_discard_test_ms4(raw, 2 * t, S(2 * b, gb)), "discard test (4 samples)"); + } else { unsigned z = zero_reg(c); unsigned gb = k.zero_b ? vgrp(c, k.a) : 1, gz = kgrp(c, K_ZERO); put(c, raw, g5p_usc_discard_test(raw, G5P_FEQ, 2 * t, S(2 * b, gb), S(2 * z, gz), 0), "discard test"); + } if (defer) { c->deferred_kill = t; /* stays busy until fs_kill() */ } else { diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index d66261b06e0..284ab420ab6 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -125,6 +125,15 @@ struct pco_g5p_fs_key { * keeping the destination alpha from a tile read issued before its kill. 1: the program may discard; * The qualified masked form is selected at pipeline creation. */ uint8_t mask_upass; + /* The 4x LOAD background only (pvr_g5p_compile_load_ms()): its one plain 2D sample reads the target's + * 2D-multisample state at the task's own sample, as Apple's loader (an iOS capture of a + * 4x colour load): the operand g5p_usc_ms_own_sample() in the register after the coordinates, then the + * narrow sample with dimension code f9. False everywhere else. */ + bool ms_own_sample; + /* 4 raster samples: a program's one conditional discard kills all four samples (g5p_usc_discard_test_ms4(), + * Apple's 4x form); several discards keep the one-sample test (Apple's several-discard 4x form is not + * ported: such draws are refused at 4x) */ + bool ms4_kill; }; #define PCO_G5P_HAS_MASK_UPASS 1 #define PCO_G5P_FS_KEY_DEFAULT ((struct pco_g5p_fs_key){.write_mask = 0xf}) diff --git a/src/imagination/vulkan/pvr_arch_g5p.c b/src/imagination/vulkan/pvr_arch_g5p.c index 536773eb1cb..3678eb55f38 100644 --- a/src/imagination/vulkan/pvr_arch_g5p.c +++ b/src/imagination/vulkan/pvr_arch_g5p.c @@ -236,6 +236,10 @@ g5p_free_programs(struct pvr_device *device, struct pvr_g5p_programs *p) vk_free(&device->vk.alloc, p->mrt_load[i]); } } + if (p->ms_load) { + g5p_free_programs(device, p->ms_load); + vk_free(&device->vk.alloc, p->ms_load); + } for (unsigned i = 0; i < ARRAY_SIZE(p->raw_clear_usc); i++) { pvr_bo_suballoc_free(p->raw_clear_usc[i]); pvr_bo_suballoc_free(p->raw_clear_kick[i].pvr_bo); @@ -263,14 +267,18 @@ g5p_free_programs(struct pvr_device *device, struct pvr_g5p_programs *p) * the backend compiles from NIR, its kick, its uploads' DMA code. A program * the backend refuses is not an error here: a LOAD render is refused by * name then. */ +/* g5p_upload_load()'s variant of the 4x LOAD */ +#define G5P_LOAD_MS4 20u + static VkResult g5p_upload_load(struct pvr_device *device, struct pvr_g5p_programs *p, unsigned raw) { struct pco_g5p_program prog; VkResult result = VK_SUCCESS; - /* raw 12..19: the compact LOAD of 5..8 targets, BGRA8 (12..15) then RGBA8 (16..19) */ - if (!(raw >= 12 ? pvr_g5p_compile_load_mrt(&prog, 5 + (raw - 12) % 4, raw < 16) : + /* raw 12..19: the compact LOAD of 5..8 targets, BGRA8 (12..15) then RGBA8 (16..19); G5P_LOAD_MS4: the 4x + * LOAD (Apple's loader, kicked at sample rate) */ + if (!(raw == G5P_LOAD_MS4 ? pvr_g5p_compile_load_ms(&prog) : raw >= 12 ? pvr_g5p_compile_load_mrt(&prog, 5 + (raw - 12) % 4, raw < 16) : raw == 10 || raw == 11 ? pvr_g5p_compile_load_scalar8(&prog, raw == 11) : (raw == 8 || raw == 9) ? pvr_g5p_compile_load_packed_sint(&prog, raw == 8 ? 8 : 16) : raw == 7 ? pvr_g5p_compile_load_rgb10a2ui(&prog) : raw == 6 ? pvr_g5p_compile_load_rgba8ui(&prog) : raw == 5 ? pvr_g5p_compile_load_rgba16ui(&prog) : raw ? pvr_g5p_compile_load_raw(&prog, raw == 2) : pvr_g5p_compile_load(&prog))) { snprintf(p->load_refusal, sizeof(p->load_refusal), "%s", prog.refusal); free(prog.code); @@ -284,7 +292,8 @@ g5p_upload_load(struct pvr_device *device, struct pvr_g5p_programs *p, unsigned for (unsigned i = 0; i < prog.num_uploads; i++) { const struct pco_g5p_upload *u = &prog.uploads[i]; if ((u->kind != PCO_G5P_UPLOAD_IMAGE && u->kind != PCO_G5P_UPLOAD_SAMPLER && - u->kind != PCO_G5P_UPLOAD_IMAGE_SIZE) || u->set || u->binding >= (raw >= 12 ? 8u : 1u)) { + u->kind != PCO_G5P_UPLOAD_IMAGE_SIZE) || u->set || + u->binding >= (raw >= 12 && raw != G5P_LOAD_MS4 ? 8u : 1u)) { snprintf(p->load_refusal, sizeof(p->load_refusal), "a load program upload of kind %u at %u.%u", u->kind, u->set, u->binding); free(prog.code); @@ -303,7 +312,8 @@ g5p_upload_load(struct pvr_device *device, struct pvr_g5p_programs *p, unsigned free(prog.code); if (result != VK_SUCCESS) return result; - result = g5p_upload_kick_temps(device, p->load_usc, p->load_temps, 1, &p->load_kick); + /* the 4x loader runs once per sample (Apple's DOUTU has src0[34:33] = 2, pvr_g5p_msaa_doutu()) */ + result = g5p_upload_kick_temps(device, p->load_usc, p->load_temps, raw == G5P_LOAD_MS4 ? 4 : 1, &p->load_kick); if (result != VK_SUCCESS) return result; uint32_t code[2 * ARRAY_SIZE(p->load_upload) + 2]; @@ -882,12 +892,45 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd if (lr != VK_SUCCESS) return lr; } + if (load && g->samples == 4) { + /* Apple's 4x loader (an iOS capture of a 4x colour load): kicked at sample rate, each instance + * samples the multisample surface at its own sample (pvr_g5p_compile_load_ms()), from the 4x texture + * state of that surface (pvr_g5p_msaa_tex_state()). One target (pvr_arch_g5p_msaa.c); integer targets + * are refused above at 4x. */ + const struct pvr_renderpass_hwsetup_eot_surface *eot = &hw_render->eot_surfaces[0]; + if (eot->need_resolve && (partial || sub_cmd->barrier_load)) + return G5P_REFUSE(cmd_buffer, "a 4x %s with a resolve: the pass stores no samples to reload", + partial ? "partial render area" : "VDM continuation"); + const struct pvr_image_view *src = info->attachments[eot->need_resolve ? eot->src_attachment_idx + : eot->attachment_idx]; + const struct pvr_image *image = pvr_image_view_get_image(src); + const struct pvr_image_plane *plane = pvr_single_plane_const(image); + const uint64_t addr = image->dev_addr.addr + plane->offset; + uint32_t key; + bool srgb; + if (image->vk.samples != VK_SAMPLE_COUNT_4_BIT || !g5p_pbe_key(src->vk.format, &key, &srgb) || addr % 64) + return G5P_REFUSE(cmd_buffer, "a 4x LOAD of %s at %#" PRIx64 " (a 64-byte based 4-sample image)", + vk_Format_to_str(src->vk.format), addr); + pvr_g5p_msaa_tex_state(load_state[0], key, addr, src->vk.extent.width, src->vk.extent.height, + vk_format_is_srgb(src->vk.format)); + load_state[0][3] = 0; + struct pvr_g5p_programs *programs = &device->g5p->programs; + VkResult lr = VK_SUCCESS; + simple_mtx_lock(&device->g5p->clear_lock); + if (!programs->ms_load) { + programs->ms_load = vk_zalloc(&device->vk.alloc, sizeof(*programs), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); + lr = programs->ms_load ? g5p_upload_load(device, programs->ms_load, G5P_LOAD_MS4) + : vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + } + load_program = programs->ms_load; + simple_mtx_unlock(&device->g5p->clear_lock); + if (lr != VK_SUCCESS) + return lr; + } if (load && !load_program->load_usc) - return G5P_REFUSE(cmd_buffer, "LOAD of a colour attachment%s: the load program: %s", - partial ? " (a partial render area)" : "", load_program->load_refusal); - if (load && g->samples == 4) - return G5P_REFUSE(cmd_buffer, "LOAD of a 4x colour attachment%s (the load program runs per pixel)", - partial ? " (a partial render area)" : ""); + return G5P_REFUSE(cmd_buffer, "LOAD of a %scolour attachment%s: the load program: %s", + g->samples == 4 ? "4x " : "", partial ? " (a partial render area)" : "", + load_program->load_refusal); g->targets = nt; /* per render: the clear floats and the PBE block, then their DOUTD data */ diff --git a/src/imagination/vulkan/pvr_arch_g5p_ds.c b/src/imagination/vulkan/pvr_arch_g5p_ds.c index 6ef6d750251..909c689f8f9 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_ds.c +++ b/src/imagination/vulkan/pvr_arch_g5p_ds.c @@ -70,20 +70,8 @@ PVR_PER_ARCH(g5p_ds_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_r image->total_size < (sampled_chain ? chain.size : plane.size)) return G5P_REFUSE(cmd_buffer, "a depth/stencil image of %" PRIu64 " bytes, the ZLS planes take %" PRIu64 " (pvr_g5p_ds_layout())", (uint64_t)image->total_size, plane.size); - if (image->vk.samples == VK_SAMPLE_COUNT_4_BIT && - (hw_render->depth_init == VK_ATTACHMENT_LOAD_OP_LOAD || hw_render->stencil_init == VK_ATTACHMENT_LOAD_OP_LOAD || - hw_render->depth_store || hw_render->stencil_store)) { - if (!pvr_g5p_capture_stub()) - return G5P_REFUSE(cmd_buffer, "a load or store of a 4-sample depth/stencil attachment " - "(no multisample ZLS capture: it stays on chip)"); - /* PVR_G5P_CAPTURE_STUB: the pass goes on, its ZLS load/store dropped (pvr_g5p_ds_zls()) */ - static bool told; - if (!told) { - told = true; - fprintf(stderr, "G5P capture stub: 4-sample depth/stencil load/store dropped (depth is not kept between " - "passes)\n"); - } - } + /* 4 samples: loaded and stored as at one sample over the 4x plane (pvr_g5p_ds_zls(), iOS + * captures of 4x depth store and load, and of 4x stencil) */ return VK_SUCCESS; } diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 3d92421a36b..1ef12fbc46b 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -595,6 +595,8 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo key.punch = g->punch_variant; g->fs_mask_upass = key.mask_upass; g->samples = s->ms ? MAX2(s->ms->rasterization_samples, VK_SAMPLE_COUNT_1_BIT) : VK_SAMPLE_COUNT_1_BIT; + /* 4x: a discard kills the four samples (Apple's 4x discard test) */ + key.ms4_kill = g->samples == 4; g->bresenham_line_probe = g5p_native_bresenham_line(s); for (uint32_t i = 0; i < create_info->stageCount; i++) { @@ -1039,13 +1041,14 @@ g5p_pipeline_check_draw_variants(struct pvr_device *device, #undef G5P_DYN_DS const bool testing = depth_test || stencil_test, writing = depth_write || stencil_test; const bool needs_punch = (testing && g->fs_discard_tokens > 1) || (writing && g->fs_discard_select); - /* The draws decide (g5p_cmd_draw()): a draw whose depth/stencil state needs the missing form is - * refused and skipped there, the command buffer valid, while the pipeline's other draws run. - * Refusing the pipeline instead loses a WebGPU device (wgpu-hal: any creation error but OOM). - * the default policy refuses at creation. */ + /* Strict mode rejects a pipeline at creation if any permitted dynamic + * depth/stencil state requires an unavailable program variant. Tolerant + * mode admits it and skips only draws that select the missing variant. */ + /* 4x: one conditional discard kills the four samples (Apple's 4x test, pco_g5p_fs_key::ms4_kill), run + * FAST_PUNCH_THROUGH as at one sample (an iOS 4x punch-through capture); several discards are not ported */ const char *why = needs_punch && !g->punch ? "discard under depth/stencil requires an unsupported punch-through variant" - : testing && (g->fs_discard || needs_punch) && g->samples != 1 - ? "discard under multisample depth/stencil is not hardware-qualified" : NULL; + : g->fs_discard && g->samples != 1 && (g->fs_discard_tokens != 1 || g->fs_discard_select) + ? "several discards at 4 samples (Apple's several-discard 4x form is not ported)" : NULL; if (!why) return VK_SUCCESS; if (!device->pdevice->skip_refused_work) @@ -1872,9 +1875,9 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g return G5P_REFUSE(cmd_buffer, "discard under depth/stencil requires an unsupported punch-through variant"); g = g->punch; } - if (testing && g->fs_discard && g->samples != 1) - return G5P_REFUSE(cmd_buffer, "discard under multisample depth/stencil is not hardware-qualified"); } + if (g->fs_discard && g->samples != 1 && (g->fs_discard_tokens != 1 || g->fs_discard_select)) + return G5P_REFUSE(cmd_buffer, "several discards at 4 samples (Apple's several-discard 4x form is not ported)"); result = pvr_arch_g5p_vertex_draw_check(cmd_buffer, g, draw); if (result != VK_SUCCESS) return result; diff --git a/src/imagination/vulkan/pvr_arch_g5p_msaa.c b/src/imagination/vulkan/pvr_arch_g5p_msaa.c index f47929e50dd..a978ca033c4 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_msaa.c +++ b/src/imagination/vulkan/pvr_arch_g5p_msaa.c @@ -19,9 +19,9 @@ /* - * The captured 4x pass: one colour target of 4-byte samples, cleared or - * don't-care, either stored with its samples (msaa4-store) or resolved by - * the PBE into a one-sample target at the end of each tile (msaa4-resolve), + * The captured 4x pass: one colour target of 4-byte samples, cleared, + * loaded or don't-care, either stored with its samples or resolved by + * the PBE into a one-sample target at the end of each tile, * a depth/stencil attachment on chip only. Upstream's hardware setup lists * the resolve as an end-of-tile surface with need_resolve and resolve_type * PVR_RESOLVE_TYPE_PBE. @@ -39,9 +39,7 @@ PVR_PER_ARCH(g5p_msaa_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr return G5P_REFUSE(cmd_buffer, "4x with %u end-of-tile surfaces (one target, stored or resolved: a " "STORE with a resolve needs the CPU resolve after the job)", hw_render->eot_surface_count); - for (unsigned i = 0; i < hw_render->color_init_count; i++) - if (hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD) - return G5P_REFUSE(cmd_buffer, "LOAD of a 4-sample attachment (no multisample background loader)"); + /* a LOAD of the target: Apple's 4x loader (pvr_arch_g5p.c) */ const struct pvr_renderpass_hwsetup_eot_surface *eot = &hw_render->eot_surfaces[0]; const struct pvr_image_view *iview = info->attachments[eot->need_resolve ? eot->src_attachment_idx diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index 22be560a6e2..3f1b72011e9 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -123,6 +123,8 @@ struct pvr_g5p_programs { struct pvr_g5p_programs *raw_load[11]; /* scalar R8 uses10/11, separate fromRGBA8 */ /* Lazy compact LOAD programs: 5..8 targets, BGRA8 [0..3] and RGBA8 [4..7] */ struct pvr_g5p_programs *mrt_load[8]; + /* Lazy 4x LOAD (one UNORM/float target, pvr_g5p_compile_load_ms()), kicked at sample rate */ + struct pvr_g5p_programs *ms_load; /* raw clear programs by total output words: 2, 4, 8 (two 32-bit integer targets) */ struct pvr_suballoc_bo *raw_clear_usc[4]; /*2/4/8 words, then scalar1 */ /* raw end-of-tile programs by target count: 1, 2 */ @@ -355,6 +357,9 @@ bool pvr_g5p_compile_load_rgb10a2ui(struct pco_g5p_program *out); bool pvr_g5p_compile_load_rgba8ui(struct pco_g5p_program *out); bool pvr_g5p_compile_load_rgba16ui(struct pco_g5p_program *out); bool pvr_g5p_compile_load_raw(struct pco_g5p_program *out, bool is_signed); +/* the 4x LOAD background (Apple's loader at sample rate, pvr_g5p_nir.c) */ +#define PVR_G5P_HAVE_LOAD_MS 1 +bool pvr_g5p_compile_load_ms(struct pco_g5p_program *out); /* The compact LOAD background of 5..8 UNORM8 targets: texelFetch of every * target (set 0, binding t), packed as the compact output. */ bool pvr_g5p_compile_load_mrt(struct pco_g5p_program *out, unsigned targets, bool bgra); diff --git a/src/imagination/vulkan/pvr_g5p_ds.c b/src/imagination/vulkan/pvr_g5p_ds.c index 151020e74ce..5b05cb80f5d 100644 --- a/src/imagination/vulkan/pvr_g5p_ds.c +++ b/src/imagination/vulkan/pvr_g5p_ds.c @@ -130,13 +130,11 @@ pvr_g5p_ds_zls(const struct pvr_g5p_ds_target *t, struct pvr_g5p_zls_words *out) if (!pvr_g5p_ds_layout(t->format, t->width, t->height, t->samples, &plane)) return "a depth/stencil attachment of 0 pixels, or not 1 or 4 samples"; const bool d = t->depth && f->depth, s = t->stencil && f->stencil; - bool zload = d && t->zload, zstore = d && t->zstore; - bool sload = s && t->sload, sstore = s && t->sstore; - if (t->samples == 4 && (zload || zstore || sload || sstore)) { - if (!pvr_g5p_capture_stub()) - return "a load or store of a 4-sample depth/stencil attachment (no multisample ZLS capture: it stays on chip)"; - zload = zstore = sload = sstore = false; /* PVR_G5P_CAPTURE_STUB: dropped, logged at the pass */ - } + const bool zload = d && t->zload, zstore = d && t->zstore; + const bool sload = s && t->sload, sstore = s && t->sstore; + /* 4 samples: the same words over the 4x plane (iOS captures of 4x depth store, depth load + * and stencil: ZLSCTL 0xe2004 / 0xaa006 / 0x26002 and ZLS_PIXELS 0x1f803f at 64x64, as at one sample; the + * stencil plane at depth + 0x10000, four times the one-sample plane) */ const struct ROGUE_CR_ISP_ZLSCTL ctl = { .loadtwiddled = true, diff --git a/src/imagination/vulkan/pvr_g5p_msaa.c b/src/imagination/vulkan/pvr_g5p_msaa.c index c685a67b06d..a12cb40091e 100644 --- a/src/imagination/vulkan/pvr_g5p_msaa.c +++ b/src/imagination/vulkan/pvr_g5p_msaa.c @@ -117,6 +117,18 @@ pvr_g5p_msaa_pbe_store(uint64_t q[3], uint32_t key, uint64_t va, uint32_t width, q[2] = 0; } +/* Texture state q0 bit 55: the 4-sample image (Apple's 4x LOAD state, the PBE store state has it clear) */ +#define G5P_TEX_Q0_MS4 (1ull << 55) + +void +pvr_g5p_msaa_tex_state(uint64_t q[3], uint32_t key, uint64_t va, uint32_t width, uint32_t height, bool srgb) +{ + q[0] = (uint64_t)(height - 1) << 41 | (uint64_t)(width - 1) << 27 | + ((key & ~G5P_PBE_LAYOUT_MASK) | G5P_PBE_LAYOUT_MS_TWIDDLED) | G5P_TEX_Q0_MS4; + q[1] = va >> 1 | (srgb ? G5P_PBE_Q1_SRGB : 0); + q[2] = 0; +} + void pvr_g5p_msaa_pbe_resolve(uint64_t q[3]) { diff --git a/src/imagination/vulkan/pvr_g5p_msaa.h b/src/imagination/vulkan/pvr_g5p_msaa.h index 67e153aba61..ef80653dae9 100644 --- a/src/imagination/vulkan/pvr_g5p_msaa.h +++ b/src/imagination/vulkan/pvr_g5p_msaa.h @@ -59,6 +59,11 @@ uint64_t pvr_g5p_msaa_sample_offset(uint32_t width, uint32_t height, uint32_t cp void pvr_g5p_msaa_pbe_store(uint64_t q[3], uint32_t key, uint64_t va, uint32_t width, uint32_t height, bool srgb); void pvr_g5p_msaa_pbe_resolve(uint64_t q[3]); +/* The texture state the 4x LOAD background samples (the 2D-multisample fetch, g5p_usc.h + * G5P_TEX_2D_MS): the store's surface, W x H pixels, layout 4 twiddled and q0 bit 55, no PBE bit + * (Apple's 4x colour load of a 64x64 BGRA8 target differs from its PBE store words in exactly those + * fields). */ +void pvr_g5p_msaa_tex_state(uint64_t q[3], uint32_t key, uint64_t va, uint32_t width, uint32_t height, bool srgb); /* The end-of-tile program of a one-target 4x store: the one-target program * (g5p_mrt_eot_usc(out, 1)) with the emitpix's sample field 2 instead of 1 @@ -100,8 +105,8 @@ struct pvr_cmd_buffer; struct pvr_renderpass_hwsetup_render; /* job_init: refuses, by name, a multisample render this lane does not - * build (only 1 and 4 samples; at 4: one colour target, no LOAD, no - * multisample depth/stencil load or store). VK_SUCCESS at one sample. */ + * build (only 1 and 4 samples; at 4: one colour target, stored or resolved; + * its LOAD is Apple's 4x loader, pvr_arch_g5p.c). VK_SUCCESS at one sample. */ VkResult PVR_PER_ARCH(g5p_msaa_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_renderpass_hwsetup_render *hw_render); diff --git a/src/imagination/vulkan/pvr_g5p_nir.c b/src/imagination/vulkan/pvr_g5p_nir.c index f92c5d451ea..4bc078ac35f 100644 --- a/src/imagination/vulkan/pvr_g5p_nir.c +++ b/src/imagination/vulkan/pvr_g5p_nir.c @@ -1038,6 +1038,52 @@ pvr_g5p_compile_load_raw(struct pco_g5p_program *out, bool is_signed) return g5p_compile_load_type(out, is_signed ? 2 : 1); } +/* + * The 4x LOAD background (a loaded 4-sample colour attachment), Apple's loader (an iOS capture + * of a 4x colour load: the 1x loader plus the own-sample operand, dimension f9, kicked at + * sample rate): a sample of the target's 2D-multisample state at the pixel centre in normalized + * coordinates, color = texture(prev, gl_FragCoord.xy / textureSize(prev, 0)), compiled with the key's + * ms_own_sample, so the sample reads the sample the task shades (pco_g5p_general.c + * emit_tex_ms_own_sample()). Its uploads: the image state, the sampler, the size block. + */ +bool +pvr_g5p_compile_load_ms(struct pco_g5p_program *out) +{ + static util_once_flag once = UTIL_ONCE_FLAG_INIT; + util_call_once(&once, g5p_init_options); + memset(out, 0, sizeof(*out)); + glsl_type_singleton_init_or_ref(); + + nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, &g5p_options, "g5p-load-ms4"); + nir_variable *prev = nir_variable_create(b.shader, nir_var_uniform, + glsl_sampler_type(GLSL_SAMPLER_DIM_2D, false, false, GLSL_TYPE_FLOAT), + "prev"); + prev->data.descriptor_set = 0; + prev->data.binding = 0; + nir_variable *color = nir_variable_create(b.shader, nir_var_shader_out, glsl_vec4_type(), "color"); + color->data.location = FRAG_RESULT_DATA0; + nir_deref_instr *deref = nir_build_deref_var(&b, prev); + nir_def *size = nir_i2f32(&b, nir_trim_vector(&b, nir_txs(&b, .texture_deref = deref, .lod = nir_imm_int(&b, 0), + .dim = GLSL_SAMPLER_DIM_2D, + .dest_type = nir_type_int32), 2)); + nir_def *uv = nir_fdiv(&b, nir_trim_vector(&b, nir_load_frag_coord(&b), 2), size); + nir_def *texel = nir_tex(&b, uv, .texture_deref = deref, .sampler_deref = deref, .dim = GLSL_SAMPLER_DIM_2D, + .dest_type = nir_type_float32); + nir_store_var(&b, color, texel, 0xf); + + bool sample_mask = false; + nir_shader *nir = graphics_lower(g5p_nir_prepare(b.shader), 1, &sample_mask, false, NULL); + struct pco_g5p_fs_key key = PCO_G5P_FS_KEY_DEFAULT; + key.last_read_groups = true; + key.ms_own_sample = true; + bool ok = nir && pco_g5p_compile_general_link(nir, true, NULL, &key, NULL, out); + if (!nir) + snprintf(out->refusal, sizeof(out->refusal), "the 4x load program's NIR"); + ralloc_free(nir); + glsl_type_singleton_decref(); + return ok; +} + bool pvr_g5p_compile_load_mrt(struct pco_g5p_program *out, unsigned targets, bool bgra) { From 1a2fc7b14f8243bdea8419cb8b44e210cc1acebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:43 +0100 Subject: [PATCH 059/109] pco/g5p: vertex attributes above r31, as Apple Vertex attributes may now take up to 64 dwords (were 32). Apple packs the attributes from r0 and reads those above r31 in place, in bank 1, with the extension word: 44 attribute dwords put a8..a10 in r32..r43, and Apple's compiler reads them with an OR of two bank-1 sources, e.g. r44 <- r40 | r36. - An input dword above r31 is a value parked there, like a fragment home. Each read moves it into a register with that OR form (g5p_usc_vs_or_wide(), both sources' bank bits in the header). - The vertex and instance index may land above r31 the same way. - An 8- or 16-bit input above r31 is moved down first, then converted. - The packed layout's limit is PVR_G5P_VERTEX_INPUT_MAX_DWORDS = 64. Assisted-by: LLM --- src/imagination/g5p/g5p_usc.c | 29 ++++++++++++++ src/imagination/g5p/g5p_usc.h | 6 +++ src/imagination/g5p/pco_g5p_general.c | 43 +++++++++++++++++---- src/imagination/vulkan/pvr_g5p_vertex.h | 5 ++- src/imagination/vulkan/pvr_g5p_vertex_nir.c | 5 +-- 5 files changed, 77 insertions(+), 11 deletions(-) diff --git a/src/imagination/g5p/g5p_usc.c b/src/imagination/g5p/g5p_usc.c index dd0c9bab948..d461f02a7ad 100644 --- a/src/imagination/g5p/g5p_usc.c +++ b/src/imagination/g5p/g5p_usc.c @@ -2290,6 +2290,35 @@ unsigned g5p_usc_alu2_wide(uint8_t *out, unsigned op, unsigned dst, unsigned a, return put_words(out, w, 5); } +#define G5P_CLASS_BINARY 0x99 /* g5p_usc_binary()'s class byte with the 32-bit result */ +#define G5P_HEADER_SRC_A_BANK1 0x20 /* source A in register bank 1 (g9p_w0()'s W0 bit 13) */ +unsigned g5p_usc_vs_or_wide(uint8_t *out, unsigned dst_half, unsigned dst_bank, unsigned a, unsigned a_group, + unsigned b, unsigned b_group) +{ + CHECK(dst_half < 256 && !(dst_half & 1) && dst_bank <= 1 && a < 128 && b < 128, "registers r0..r127"); + /* g5p_usc_binary()'s layout for A | B with both sources' bank bits in the header; a source above r31 keeps + * r & 31 in its selector, sets A's selector bit 0 and puts r >> 5 in an extension parcel (A at bit 1, B at + * bit 9 of the parcel, as g5p_usc_alu2_wide()). */ + const bool wide = a >= 32 || b >= 32; + uint8_t raw[10]; + raw[0] = G5P_CLASS_BINARY; + raw[1] = G5P_HEADER_FRAGMENT | G5P_HEADER_SRC_A_BANK1 | G5P_HEADER_SRC_B_BANK1; + if (!destination(raw + 2, dst_half, dst_bank)) + return 0; + SEL(sa, 2 * (a & 31), a_group & 1, wide); + SEL(sb, 2 * (b & 31), b_group & 1, G5P_PARCEL_CONTINUES); + raw[4] = sa; + raw[5] = sb; + unsigned n = 6; + if (wide) { + raw[n++] = (a >> 5) << 1; + raw[n++] = (b >> 5) << 1 | G5P_PARCEL_CONTINUES; + } + raw[n++] = binop_code[G5P_BIN_OR]; + raw[n++] = 0; + return finish(out, raw, n); +} + /* Graphics storage differential host compiles, compute-gfx-storage-isa.md. * Unlike compute/vertex class10, fragment stores use the pixel side-effect * port70/71 and the observed active-fragment predicate. */ diff --git a/src/imagination/g5p/g5p_usc.h b/src/imagination/g5p/g5p_usc.h index 2412607764d..7e19c7f7f65 100644 --- a/src/imagination/g5p/g5p_usc.h +++ b/src/imagination/g5p/g5p_usc.h @@ -517,6 +517,12 @@ unsigned g5p_usc_cf_compare_imm(uint8_t *out, enum g5p_cmp op, unsigned reg, uns * r31 takes the extension word. op: the binop code. */ unsigned g5p_usc_alu2_wide(uint8_t *out, unsigned op, unsigned dst, unsigned a, unsigned a_group, unsigned b, unsigned b_group); +/* The vertex form: A | B with both sources in bank 1 (both bank bits in the header), any of r0..r127 (above r31 the extension + * word), into dst_half of bank dst_bank. Apple's A9 vertex programs read attributes above r31 in place this way + * (Apple's compiler, 44 attribute dwords: r44 <- r40 | r36 with the extension word). With a == b it copies + * a register (the G9P home move). */ +unsigned g5p_usc_vs_or_wide(uint8_t *out, unsigned dst_half, unsigned dst_bank, unsigned a, unsigned a_group, + unsigned b, unsigned b_group); /* Derivatives (Apple's A9 dfdx/dfdy, tools/mesa-a9-hypr-re f_ddx_*, f_ddy_*): * the class-98 unary form with op 44 and a modifier word, 0200 across x and diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index dab16f472da..3672bd04195 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -62,6 +62,9 @@ * buffer starts at CODE_INITIAL and doubles. */ #define CODE_INITIAL 16384 #define NUM_DWORDS 32 +/* Vertex input dwords: the PDS fetches up to this many into bank 1. Those above r31 are read through a move + * (g5p_usc_vs_or_wide(), Apple's vertex form for attributes above r31), as the fragment homes are. */ +#define VS_INPUT_DWORDS 64 #define F32_ONE 0x3f800000u #define LAST_END (INT_MAX - 1) /* read by the output block */ #define DISCARD_AT (INT_MAX - 2) /* read by the deferred discards, before the output */ @@ -2444,7 +2447,11 @@ static unsigned reg_of(struct gctx *c, struct ref r, const char *what) case VAL_HOME: { /* parked above r31: back into a register for this instruction */ unsigned reg = alloc_reg(c, -1); - put(c, raw, g5p_usc_alu2_wide(raw, G5P_OP_OR, reg, v->reg, 0, v->reg, 0), "move from a home above r31"); + if (c->vs) /* a vertex input fetched above r31 (setup_inputs()): v->reg is its bank-1 register */ + put(c, raw, g5p_usc_vs_or_wide(raw, H(c, reg), vbank(c, reg), v->reg, 0, v->reg, 0), + "move a vertex input from above r31"); + else + put(c, raw, g5p_usc_alu2_wide(raw, G5P_OP_OR, reg, v->reg, 0, v->reg, 0), "move from a home above r31"); c->scratch |= BITFIELD64_BIT(reg); return reg; } @@ -5239,6 +5246,14 @@ static void vs_converts(struct gctx *c, struct vs_convert *cv, unsigned n) c->alloc_out = true; /* converted attributes stay in bank 1, where they are fetched */ qsort(cv, n, sizeof(*cv), vs_convert_order); for (unsigned i = 0; i < n; i++) { + if (cv[i].reg >= NUM_DWORDS) { + /* a dword fetched above r31 (sorted last, after the low converts freed their registers): moved into a + * register first (Apple's vertex OR form, g5p_usc_vs_or_wide()), its converts then read it there */ + const unsigned hi = cv[i].reg, lo = alloc_reg(c, -1); + put(c, raw, g5p_usc_vs_or_wide(raw, H(c, lo), vbank(c, lo), hi, 0, hi, 0), "move a vertex input from above r31"); + for (unsigned k = i; k < n && cv[k].reg == hi; k++) + cv[k].reg = lo; + } const unsigned half = 2 * cv[i].reg + (cv[i].byte >> 1); bool last_half = true, last_reg = true; for (unsigned k = i + 1; k < n; k++) { @@ -5280,8 +5295,8 @@ static void setup_inputs(struct gctx *c) !nir_src_is_const(in->src[0]) || nir_src_as_uint(in->src[0]) || in->def.bit_size != 32) refuse(c, "load_input: only direct 32-bit generic attributes"); unsigned l = loc - VERT_ATTRIB_GENERIC0, first = nir_intrinsic_component(in); - if (4 * l + 4 > NUM_DWORDS) - refuse(c, "vertex input location %u: its fetch lands beyond r%u", l, NUM_DWORDS - 1); + if (4 * l + 4 > VS_INPUT_DWORDS) + refuse(c, "vertex input location %u: its fetch lands beyond r%u", l, VS_INPUT_DWORDS - 1); c->p->input_mask |= 1u << l; for (unsigned k = 0; k < in->def.num_components; k++) { unsigned comp = first + k; @@ -5303,7 +5318,9 @@ static void setup_inputs(struct gctx *c) if (fmt >= PCO_G5P_VSF_COUNT || in->def.num_components != 1 || (fmt >= PCO_G5P_VSF_U16 && (byte & 1))) refuse(c, "vertex input location %u: converted load %#x", l, base); - raw_regs |= BITFIELD64_BIT(4 * l + comp); + /* a fetched dword above r31: vs_converts() moves it down first (not in the low-register masks) */ + if (4 * l + comp < NUM_DWORDS) + raw_regs |= BITFIELD64_BIT(4 * l + comp); if (v->uses) { if (ncv == ARRAY_SIZE(cv)) refuse(c, "more than %u converted vertex input components", (unsigned)ARRAY_SIZE(cv)); @@ -5312,6 +5329,13 @@ static void setup_inputs(struct gctx *c) continue; } v->input = true; + if (4 * l + comp >= NUM_DWORDS) { + /* above r31: read through a move at each use */ + v->state = VAL_HOME; + v->reg = 4 * l + comp; + c->p->evidence |= PCO_G5P_EV_VS_ALU; + continue; + } if (v->uses) hold(c, v, 4 * l + comp); } @@ -5345,12 +5369,17 @@ static void setup_inputs(struct gctx *c) for (unsigned i = 0; i < 2; i++) { if (!ids[i]) continue; - if (next >= NUM_DWORDS) + if (next >= VS_INPUT_DWORDS) refuse(c, "%s index: no register after the last attribute", i ? "instance" : "vertex"); struct val *v = V(c, ids[i], 0); - v->state = VAL_REG; v->input = true; - hold(c, v, next); + if (next >= NUM_DWORDS) { /* after attributes above r31: read through a move, as they are */ + v->state = VAL_HOME; + v->reg = next; + } else { + v->state = VAL_REG; + hold(c, v, next); + } c->p->id_mask |= 1u << i; c->p->id_reg[i] = next++; c->p->evidence |= PCO_G5P_EV_VS_IDS; diff --git a/src/imagination/vulkan/pvr_g5p_vertex.h b/src/imagination/vulkan/pvr_g5p_vertex.h index 7d771511e5f..99abbcf4329 100644 --- a/src/imagination/vulkan/pvr_g5p_vertex.h +++ b/src/imagination/vulkan/pvr_g5p_vertex.h @@ -26,8 +26,11 @@ struct nir_shader; struct vk_vertex_input_state; -/* The backend's unified store for vertex inputs: dwords r0..r31 */ +/* The backend's unified store for vertex inputs: dwords r0..r31 (the one-location-per-vec4 layout's range) */ #define PVR_G5P_VERTEX_INPUT_DWORDS 32 +/* The packed layout's limit: attributes above r31 are read through a move (pco_g5p_general.c VS_INPUT_DWORDS, + * Apple's vertex form for attributes above r31: 44 dwords from r0 in applegpu_g5p) */ +#define PVR_G5P_VERTEX_INPUT_MAX_DWORDS 64 /* * A pipeline's vertex input layout. pvr_g5p_vertex_layout_init() takes the diff --git a/src/imagination/vulkan/pvr_g5p_vertex_nir.c b/src/imagination/vulkan/pvr_g5p_vertex_nir.c index 14994e55402..ffebebbc7b3 100644 --- a/src/imagination/vulkan/pvr_g5p_vertex_nir.c +++ b/src/imagination/vulkan/pvr_g5p_vertex_nir.c @@ -285,9 +285,8 @@ packed_fetches(struct pvr_g5p_vertex_layout *l, struct attrib *a, unsigned n, un } slot += a[i].dwords; } - if (slot > PVR_G5P_VERTEX_INPUT_DWORDS) - return REFUSE(l, "vertex attributes of %u dwords (the unified store holds %u)", slot, - PVR_G5P_VERTEX_INPUT_DWORDS); + if (slot > PVR_G5P_VERTEX_INPUT_MAX_DWORDS) + return REFUSE(l, "vertex attributes of %u dwords (%u are built)", slot, PVR_G5P_VERTEX_INPUT_MAX_DWORDS); for (unsigned v = 0; 4 * v < slot; v++) l->components[v] = MIN2(4, slot - 4 * v); return true; From 2f286bd3ce15b0ff2101860ebbf0987a5ae73d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:43 +0100 Subject: [PATCH 060/109] pco/g5p: general-blend discards in Apple's order A discard in a program with a general blend epilogue now runs as Apple's straight-blend discard (pass type PUNCH_THROUGH, ISPCTL upass 15): the kill is held and issued after the epilogue's tile read (test, tile read, kill, blend), and the general-blend punch variant is built. The punch-through form that once hung the A9 under Aquarium killed before the tile read, which a general epilogue no longer does. A general epilogue whose kill cannot be held (several discards, MRT, compact or integer output) is refused by name. The premultiplied form keeps its bytes. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 24 ++++++++++++++--------- src/imagination/vulkan/pvr_arch_g5p_gfx.c | 10 +--------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 3672bd04195..67a9fc4eb5a 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -5730,8 +5730,12 @@ static void fs_output(struct gctx *c) put(c, raw, g5p_usc_depth_feedback(raw, 2 * reg), "fragment depth feedback"); c->p->depth_feedback = true; } - /* a discarding program whose epilogue reads the tile: the kill waits for that read (fs_put_epilogue()) */ - const bool defer = (c->key->mask_upass) && c->ndiscards + c->discard_always == 1 && + /* a discarding program whose epilogue reads the tile: the kill waits for that read (fs_put_epilogue()). + * A general-blend epilogue always takes Apple's order (a blending Metal probe: test, tile read, + * kill, blend), the form its default PUNCH_THROUGH runs in; the premultiplied production form keeps its + * bytes. */ + const bool defer = (c->key->mask_upass || c->blend == BLEND_GENERAL) && + c->ndiscards + c->discard_always == 1 && c->key->color_output == PCO_G5P_COLOR_F16 && !compact_output(c->key) && !c->key->mrt_blend_state && any_color(c) && (c->blend == 1 || c->blend == 2 || c->blend == BLEND_GENERAL); @@ -5877,13 +5881,15 @@ static void fs_output(struct gctx *c) if (c->ndiscards + c->discard_always > 1) refuse(c, "discard in a program with a general blend epilogue needs one merged kill"); /* A kill before a general epilogue runs PUNCH_THROUGH with a general - * blend, base program or punch variant alike. r39-6.1 hard-hung the - * iPad in Aquarium with that form under depth writes (not bisected - * yet), so it is off: such a program is refused at pipeline - * creation, the r39-6 rule. */ - if ((c->ndiscards || c->discard_always) && c->key->mask_upass != 1) - refuse(c, "discard in a program with a general blend epilogue " - "(general-blend punch-through is off)"); + * blend, base program or punch variant alike, in Apple's order: the + * tile read, then the kill, then the blend (fs_output()'s defer; + * Apple's straight-alpha punch-through capture runs its draw in the PUNCH_THROUGH pass). + * Aquarium hard-hung the iPad with the old + * order (the kill before the tile read), which is no longer used. */ + /* Opaque mode retains the masked form for non-discarding programs only. */ + if ((c->ndiscards || c->discard_always) && c->deferred_kill == -2 && c->key->mask_upass != 1) + refuse(c, "discard in a program with a general blend epilogue: no held kill for Apple's order " + "(several discards, MRT, compact or integer output)"); uint8_t buf[512]; const char *err; struct g5p_blend_key bk = general_key(c->key); diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 1ef12fbc46b..68d7fc3b21b 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -981,15 +981,7 @@ g5p_pipeline_build_punch(struct pvr_device *device, { if (g->fs_discard_tokens < 2 && !g->fs_discard_select) return VK_SUCCESS; - /* The general-blend punch variant (r39-6.1's form: PUNCH_THROUGH + - * general blend + depth write) is not built: r39-6.1 hard-hung the iPad - * in Aquarium with it, cause not yet bisected. Without the variant, - * g5p_pipeline_check_draw_variants() refuses the pipeline at creation - * when a depth/stencil state could need it, the non-fatal form r39-3 had. */ - if (g->fs_blend_mode == 4 /* general */ && !g->fs_mask_upass) { - g->punch_why = "general blend epilogue"; - return VK_SUCCESS; - } + /* General-blend punch variants use Apple's tile-read-before-kill order. */ struct pvr_g5p_pipeline *p = vk_zalloc(&device->vk.alloc, sizeof(*p), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!p) From 46a745797a04381c17266d4d2d83ac5cad6fc4e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:44 +0100 Subject: [PATCH 061/109] g9p: native vector shared-memory accesses on the A9 A9 compute took one 32-bit value per shared-memory access, and a uvec4 store to shared memory stopped GravityMark. Apple's local load/store word carries a size code (W0[15:11]: 4 = 32-bit, 5 = 2x32, 7 = 4x32), with the data in consecutive registers and the address unit equal to the access size. The witnessed offset-half form shifts by log2(unit). Shared accesses are split by nir_lower_mem_access_bit_sizes into 1-, 2- and 4-dword pieces by alignment, and a9_shared() emits Apple's form for each: the static and register-indexed address forms, loads into a register run, stores from one. Sub-dword access stays refused. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_compute.h | 19 +++++++++ src/imagination/g9p/g9p_a9_intrinsics.h | 45 +++++++++++++-------- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/g9p/g9p_compile.c | 12 ++++-- src/imagination/g9p/g9p_isa.h | 52 +++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 20 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index a08e6e2746d..5284ea982e3 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -1,4 +1,19 @@ /* SPDX-License-Identifier: MIT */ +/* Apple's shared-memory size codes require size-aligned 1/2/4-dword runs. */ +static nir_mem_access_size_align +a9_shared_size_align(nir_intrinsic_op op, uint8_t bytes, uint8_t bit_size, + uint32_t align_mul, uint32_t align_offset, + bool offset_is_const, enum gl_access_qualifier access, + const void *data) +{ + unsigned align = nir_combined_align(align_mul, align_offset); + if (bit_size != 32 || align < 4) + return (nir_mem_access_size_align) { 1, bit_size, bit_size / 8 }; + unsigned comps = bytes / 4; + unsigned n = comps >= 4 && align >= 16 ? 4 : comps >= 2 && align >= 8 ? 2 : 1; + return (nir_mem_access_size_align) { n, 32, 4 * n, nir_mem_access_shift_method_scalar }; +} + bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_ctx) { const nir_lower_compute_system_values_options csv = { @@ -9,6 +24,10 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c lower_front(nir); NIR_PASS(_, nir, nir_lower_vars_to_explicit_types, nir_var_mem_shared, glsl_get_natural_size_align_bytes); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_shared, nir_address_format_32bit_offset); + const nir_lower_mem_access_bit_sizes_options shared_opts = { + .modes = nir_var_mem_shared, .callback = a9_shared_size_align, + }; + NIR_PASS(_, nir, nir_lower_mem_access_bit_sizes, &shared_opts); if (nir->info.shared_size > 16384) { if (!out->a9_refusal[0]) snprintf(out->a9_refusal, sizeof(out->a9_refusal), "shared allocation exceeds 16384 bytes"); diff --git a/src/imagination/g9p/g9p_a9_intrinsics.h b/src/imagination/g9p/g9p_a9_intrinsics.h index e885110bbe1..1c812d16957 100644 --- a/src/imagination/g9p/g9p_a9_intrinsics.h +++ b/src/imagination/g9p/g9p_a9_intrinsics.h @@ -14,37 +14,50 @@ static bool a9_shared(struct g9p_ctx *ctx, nir_intrinsic_instr *in) nir_src offset = in->src[load ? 0 : 1]; unsigned bits = load ? in->def.bit_size : nir_src_bit_size(in->src[0]); unsigned comps = load ? in->def.num_components : nir_src_num_components(in->src[0]); - if (!ctx->out->a9 || bits != 32 || comps != 1 || (nir_intrinsic_base(in) & 3)) + unsigned base = 16 + nir_intrinsic_base(in); + unsigned unit = 4 * comps, shift_bits = util_logbase2(unit); + unsigned size = comps == 1 ? G9P_SIZE_U32 : comps == 2 ? G9P_SIZE_2X32 : G9P_SIZE_4X32; + if (!ctx->out->a9 || bits != 32 || (comps != 1 && comps != 2 && comps != 4) || + base % unit || (!load && nir_intrinsic_write_mask(in) != BITFIELD_MASK(comps))) return false; - unsigned word = 4 + nir_intrinsic_base(in) / 4; + unsigned word = base / unit; bool constant = nir_src_is_const(offset); int index = -1; if (constant) { unsigned byte = nir_src_as_uint(offset); - if (byte & 3) return false; - word += byte / 4; + if (byte % unit) return false; + word += byte / unit; } else { int src = src_reg(ctx, nir_get_scalar(offset.ssa, 0)); - int shift = const_reg(ctx, 2); + int shift = const_reg(ctx, shift_bits); index = alloc_temp(ctx); if (src < 0 || shift < 0 || index < 0) return false; g9p_shift(ctx->c, index, src, shift, true); - G9P_EMIT(ctx->c, 0x1189, 0x0300 | 2 * index, - 0x0101 | index << 10, 0x0120, 0x0fa1, 0x4000); + g9p_shared_index(ctx->c, index, index, shift_bits); } - int reg = load ? alloc_reg(ctx) : src_reg(ctx, nir_get_scalar(in->src[0].ssa, 0)); - if (reg < 0 || word >= 65536) return false; - unsigned w1 = 0x0100 | (!load ? 0x80 : 0) | 2 * reg; - if (constant) { - G9P_EMIT(ctx->c, 0x2314, w1, 0x8100 | (word & 255), 0x0200 | (word >> 8)); + int reg; + if (load) { + reg = comps == 1 ? alloc_reg(ctx) : alloc_range(ctx, comps); + } else if (comps == 1) { + reg = src_reg(ctx, nir_get_scalar(in->src[0].ssa, 0)); } else { - w1 |= (2 * index) << 9; - if (word < 256) G9P_EMIT(ctx->c, 0x2314, w1, 0x8800 | word); - else G9P_EMIT(ctx->c, 0x2314, w1, 0x8900 | (word & 255), word >> 8); + reg = alloc_range(ctx, comps); + if (reg < 0) return false; + ctx->temps |= BITFIELD64_RANGE(reg, comps); + for (unsigned j = 0; j < comps; j++) { + int r = src_reg(ctx, nir_get_scalar(in->src[0].ssa, j)); + if (r < 0) return false; + g9p_alu2(ctx->c, G9P_OR, reg + j, r, r, false); + } } + if (reg < 0 || 2 * reg + 2 * comps - 1 >= 128 || word >= 4096) return false; + g9p_shared_access(ctx->c, size, !load, reg, word, constant ? -1 : 2 * index); g9p_wait(ctx->c); release(ctx, &in->instr); - if (load) ctx->reg[key(&in->def, 0)] = reg; + if (load) for (unsigned j = 0; j < comps; j++) { + ctx->reg[key(&in->def, j)] = reg + j; + if (!ctx->last_use[key(&in->def, j)]) ctx->busy &= ~BITFIELD64_BIT(reg + j); + } return true; } /* Keep the election's local-index source live for every task barrier. */ diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index dbe019eb9e3..492e35a6e9e 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -37,6 +37,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index a95c0675531..e20c4130797 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -41,6 +41,10 @@ #include "util/format/u_format.h" #include "g9p_compile.h" #include "g9p_isa.h" +#include "util/u_debug.h" +#include "util/log.h" + +DEBUG_GET_ONCE_BOOL_OPTION(g9p_pressure, "G9P_PRESSURE", false) /* r0-r29 for values, the mask stack after them (G9P_REGS: debug override) */ static unsigned @@ -4889,7 +4893,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, walk_liveness(&ctx, &impl->body, true); /* G9P_PRESSURE: the most scalars live at once, from the liveness above */ - if (getenv("G9P_PRESSURE")) { + if (debug_get_option_g9p_pressure()) { unsigned *delta = rzalloc_array(mem_ctx, unsigned, ctx.ip + 2), *dec = rzalloc_array(mem_ctx, unsigned, ctx.ip + 2); unsigned live = 0, peak = 0, peak_ip = 0, decls = 0; @@ -4920,7 +4924,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, if (live > peak) peak = live, peak_ip = i; } - fprintf(stderr, "g9p: pressure peak %u at instruction %u of %u (%u NIR register scalars)\n", + mesa_logi( "g9p: pressure peak %u at instruction %u of %u (%u NIR register scalars)\n", peak, peak_ip, ctx.ip, decls); } @@ -5027,9 +5031,9 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, bool free = true; for (unsigned i = 0; free && i < num_homed; i++) - free = homed[i].home != 32 + (int)h + (getenv("G9P_HOME_SKIP") ? atoi(getenv("G9P_HOME_SKIP")) : 0) || homed[i].end < start || end < homed[i].start; + free = homed[i].home != 32 + (int)h || homed[i].end < start || end < homed[i].start; if (free) - home = 32 + h + (getenv("G9P_HOME_SKIP") ? atoi(getenv("G9P_HOME_SKIP")) : 0); + home = 32 + h; } if (home < 0) { fprintf(stderr, "g9p: out of registers above r31 at instruction %u\n", ctx.oom_ip); diff --git a/src/imagination/g9p/g9p_isa.h b/src/imagination/g9p/g9p_isa.h index 915429ca13e..19bf8632b78 100644 --- a/src/imagination/g9p/g9p_isa.h +++ b/src/imagination/g9p/g9p_isa.h @@ -263,6 +263,58 @@ g9p_tg_load(struct g9p_code *c, unsigned dst, unsigned word) 0x0200 | (word >> 8)); } +/* + * Threadgroup-memory access (g9p_tg_load()'s class), Apple's A9 compiler's forms: W0 the class byte, the load's + * bank bits (g9p_mem_at()) and the access size at bit 11 (G9P_SIZE_*); W1 the data register's half, the store + * bit and the index register's half at bit 9; then the word offset. With no index it takes two words, W2 with + * G9P_SHARED_OFFSET and the offset's low byte, W3 with G9P_SHARED_OFFSET_HIGH and its high bits; after an index, + * W2 adds G9P_SHARED_INDEXED and holds an offset below 256 alone, a larger one in the same two-word layout without + * G9P_SHARED_OFFSET_HIGH. Those three bits are named by their observed roles. + */ +#define G9P_CLASS_SHARED 0x14 +#define G9P_MEM_W0_BANKS 0x0300 +#define G9P_SIZE_SHIFT 11 +#define G9P_SIZE_4X32 7 +#define G9P_MEM_STORE 0x0080 +#define G9P_MEM_INDEX_SHIFT 9 +#define G9P_SHARED_OFFSET 0x8000 +#define G9P_SHARED_INDEXED 0x0800 +#define G9P_SHARED_OFFSET_HIGH 0x0200 +static inline void +g9p_shared_access(struct g9p_code *c, unsigned size, bool store, unsigned reg, unsigned word, int index_half) +{ + const uint16_t w0 = size << G9P_SIZE_SHIFT | G9P_MEM_W0_BANKS | G9P_CLASS_SHARED; + uint16_t w1 = G9P_CONT | (store ? G9P_MEM_STORE : 0) | 2 * reg; + + if (index_half < 0) { + G9P_EMIT(c, w0, w1, G9P_SHARED_OFFSET | G9P_CONT | (word & 255), G9P_SHARED_OFFSET_HIGH | (word >> 8)); + return; + } + w1 |= (unsigned)index_half << G9P_MEM_INDEX_SHIFT; + if (word < 256) + G9P_EMIT(c, w0, w1, G9P_SHARED_OFFSET | G9P_SHARED_INDEXED | word); + else + G9P_EMIT(c, w0, w1, G9P_SHARED_OFFSET | G9P_SHARED_INDEXED | G9P_CONT | (word & 255), word >> 8); +} + +/* + * A shared-memory index after its shift, as Apple's A9 compiler emits it: r[@dst] = 0 + r[@src], an integer add + * of the literal 0 (g9p_add_imm()'s literal form) in g9p_addr()'s class, with @shift in g9p_addr()'s shift field. + * G9P_INDEX_SHIFT_CTL is named by its observed role (seen in this form only). + */ +#define G9P_CLASS_ADDR 0x89 /* g9p_addr()'s class */ +#define G9P_SRC_A_IMM 0x0001 /* source A is a literal, its value from bit 1 (g9p_add_imm()) */ +#define G9P_IMM_SHORT 0x0020 /* g9p_add_imm()'s short-literal word */ +#define G9P_INDEX_SHIFT_CTL 0x0e00 +#define G9P_ADDR_SHIFT_BIT 13 /* g9p_addr()'s shift field */ +static inline void +g9p_shared_index(struct g9p_code *c, unsigned dst, unsigned src, unsigned shift) +{ + G9P_EMIT(c, G9P_CLASS_ADDR | G9P_CONT | G9P_W0_SRC_B_BANK1, G9P_CONT | G9P_W1_BANK1 | 2 * dst, + G9P_SRC_A_IMM | G9P_CONT | src << G9P_SRC_B_SHIFT, G9P_CONT | G9P_IMM_SHORT, + G9P_CONT | G9P_INDEX_SHIFT_CTL | G9P_IADD, shift << G9P_ADDR_SHIFT_BIT); +} + /* * dst = flat varying component @comp (4 per varying in the DOUTI program's * order), the provoking vertex's value bit for bit; follow with g9p_wait(). From dbb4ae8e28824a5e03f0ce524d1fb79bd453e96f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:44 +0100 Subject: [PATCH 062/109] pvr/g5p: expose and lower single-invocation subgroups The A9 has no SIMD-group functions (Metal offers them from the A13), but pvr advertised arithmetic, ballot, shuffle and vote, which GravityMark uses when present, and the A9 compute compiler implemented none. On the A9 the subgroup size is now 1 (min = max = 1, maxComputeWorkgroupSubgroups 256), compute stages only, with the operations that are trivial at size 1 (no quad). A NIR pass lowers each subgroup intrinsic to its size-1 value: ids 0, masks of one bit, votes and reductions as the source, exclusive scans as the identity. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_compute.h | 85 ++++++++++++++++++++ src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/vulkan/pvr_physical_device.c | 9 ++- 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index 5284ea982e3..4be1fa1fa2f 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -1,4 +1,87 @@ /* SPDX-License-Identifier: MIT */ +/* One invocation is the whole subgroup on G5P. */ +static bool +a9_lower_subgroups_size1(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + b->cursor = nir_before_instr(&in->instr); + nir_def *v, *bit, *parts[NIR_MAX_VEC_COMPONENTS]; + unsigned n = in->def.num_components, bits = in->def.bit_size; + switch (in->intrinsic) { + case nir_intrinsic_load_subgroup_size: + v = nir_imm_int(b, 1); break; + case nir_intrinsic_load_subgroup_invocation: + case nir_intrinsic_first_invocation: + case nir_intrinsic_last_invocation: + v = nir_imm_int(b, 0); break; + case nir_intrinsic_load_num_subgroups: + v = nir_imm_int(b, b->shader->info.workgroup_size[0] * + b->shader->info.workgroup_size[1] * + b->shader->info.workgroup_size[2]); break; + case nir_intrinsic_load_subgroup_id: + v = nir_load_local_invocation_index(b); break; + case nir_intrinsic_load_subgroup_eq_mask: + case nir_intrinsic_load_subgroup_ge_mask: + case nir_intrinsic_load_subgroup_le_mask: + case nir_intrinsic_ballot: + parts[0] = in->intrinsic == nir_intrinsic_ballot ? + nir_b2iN(b, in->src[0].ssa, bits) : nir_imm_intN_t(b, 1, bits); + for (unsigned j = 1; j < n; j++) parts[j] = nir_imm_intN_t(b, 0, bits); + v = nir_vec(b, parts, n); break; + case nir_intrinsic_load_subgroup_gt_mask: + case nir_intrinsic_load_subgroup_lt_mask: + case nir_intrinsic_ballot_bit_count_exclusive: + v = nir_imm_zero(b, n, bits); break; + case nir_intrinsic_vote_ieq: + case nir_intrinsic_vote_feq: + case nir_intrinsic_elect: + v = nir_imm_true(b); break; + case nir_intrinsic_vote_any: + case nir_intrinsic_vote_all: + case nir_intrinsic_read_invocation: + case nir_intrinsic_read_first_invocation: + case nir_intrinsic_shuffle: + case nir_intrinsic_shuffle_xor: + case nir_intrinsic_shuffle_up: + case nir_intrinsic_shuffle_down: + case nir_intrinsic_rotate: + case nir_intrinsic_quad_broadcast: + case nir_intrinsic_quad_swap_horizontal: + case nir_intrinsic_quad_swap_vertical: + case nir_intrinsic_quad_swap_diagonal: + case nir_intrinsic_quad_vote_any: + case nir_intrinsic_quad_vote_all: + case nir_intrinsic_reduce: + case nir_intrinsic_inclusive_scan: + v = in->src[0].ssa; break; + case nir_intrinsic_inverse_ballot: + case nir_intrinsic_ballot_bitfield_extract: + case nir_intrinsic_ballot_bit_count_reduce: + case nir_intrinsic_ballot_bit_count_inclusive: + case nir_intrinsic_ballot_find_lsb: + case nir_intrinsic_ballot_find_msb: + bit = nir_iand_imm(b, nir_channel(b, in->src[0].ssa, 0), 1); + if (in->intrinsic == nir_intrinsic_inverse_ballot) + v = nir_ine_imm(b, bit, 0); + else if (in->intrinsic == nir_intrinsic_ballot_bitfield_extract) + v = nir_iand(b, nir_ieq_imm(b, in->src[1].ssa, 0), nir_ine_imm(b, bit, 0)); + else if (in->intrinsic == nir_intrinsic_ballot_find_lsb || + in->intrinsic == nir_intrinsic_ballot_find_msb) + v = nir_bcsel(b, nir_ine_imm(b, bit, 0), nir_imm_int(b, 0), nir_imm_int(b, -1)); + else + v = nir_u2uN(b, bit, bits); + break; + case nir_intrinsic_exclusive_scan: { + nir_const_value identity = nir_alu_binop_identity(nir_intrinsic_reduction_op(in), bits); + nir_def *scalar = nir_build_imm(b, 1, bits, &identity); + for (unsigned j = 0; j < n; j++) parts[j] = scalar; + v = nir_vec(b, parts, n); break; + } + default: return false; + } + nir_def_replace(&in->def, v); + return true; +} + /* Apple's shared-memory size codes require size-aligned 1/2/4-dword runs. */ static nir_mem_access_size_align a9_shared_size_align(nir_intrinsic_op op, uint8_t bytes, uint8_t bit_size, @@ -22,6 +105,8 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c memset(out, 0, sizeof(*out)); out->a9 = true; lower_front(nir); + NIR_PASS(_, nir, nir_shader_intrinsics_pass, a9_lower_subgroups_size1, + nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_lower_vars_to_explicit_types, nir_var_mem_shared, glsl_get_natural_size_align_bytes); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_shared, nir_address_format_32bit_offset); const nir_lower_mem_access_bit_sizes_options shared_opts = { diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index 492e35a6e9e..b7f44c991a6 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -38,6 +38,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index 132fc8e830d..4770f588e3f 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -833,7 +833,7 @@ static bool pvr_physical_device_get_properties( .nonCoherentAtomSize = 1U, /* Vulkan 1.1 */ - .subgroupSize = ROGUE_MAX_INSTANCES_PER_TASK, + .subgroupSize = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 1U : ROGUE_MAX_INSTANCES_PER_TASK, .subgroupSupportedStages = VK_SHADER_STAGE_COMPUTE_BIT, .subgroupSupportedOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_VOTE_BIT | @@ -973,9 +973,10 @@ static bool pvr_physical_device_get_properties( .integerDotProductAccumulatingSaturating64BitMixedSignednessAccelerated = false, /* Vulkan 1.3 / VK_EXT_subgroup_size_control */ - .minSubgroupSize = ROGUE_MAX_INSTANCES_PER_TASK, - .maxSubgroupSize = ROGUE_MAX_INSTANCES_PER_TASK, - .maxComputeWorkgroupSubgroups = 128U / ROGUE_MAX_INSTANCES_PER_TASK, + .minSubgroupSize = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 1U : ROGUE_MAX_INSTANCES_PER_TASK, + .maxSubgroupSize = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 1U : ROGUE_MAX_INSTANCES_PER_TASK, + .maxComputeWorkgroupSubgroups = PVR_HAS_FEATURE(dev_info, apple_g5p) ? + 256U : 128U / ROGUE_MAX_INSTANCES_PER_TASK, .requiredSubgroupSizeStages = VK_SHADER_STAGE_COMPUTE_BIT, /* Vulkan 1.2 / VK_KHR_timeline_semaphore */ From e65f5b53d4e2823197064f0460623c5b9b719237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:44 +0100 Subject: [PATCH 063/109] g9p: shared atomics with Apple's LL/SC loop on the A9 Apple compiles threadgroup atomics as an LL/SC retry loop, the same loop the A9 already runs in its fenced shared barrier. g5p_shared_atomic() copies that structure; only the address (the static or register-indexed form) and the update differ. Supported: add, min/max (signed and unsigned), and/or/xor, exchange, and compare-exchange (the store always runs, so the CAS is strong, as Apple's is). The result is the loaded value. 32-bit scalars only; float atomics are refused by name. Assisted-by: LLM --- src/imagination/g5p/g5p_shared_atomic.h | 76 +++++++++++++++++++++++++ src/imagination/g9p/g9p_a9_intrinsics.h | 47 +++++++++++++++ src/imagination/g9p/g9p_a9_tex.h | 1 + 3 files changed, 124 insertions(+) create mode 100644 src/imagination/g5p/g5p_shared_atomic.h diff --git a/src/imagination/g5p/g5p_shared_atomic.h b/src/imagination/g5p/g5p_shared_atomic.h new file mode 100644 index 00000000000..56a98f6581c --- /dev/null +++ b/src/imagination/g5p/g5p_shared_atomic.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef G5P_SHARED_ATOMIC_H +#define G5P_SHARED_ATOMIC_H +#include "compiler/nir/nir.h" +#include "../g9p/g9p_isa.h" + +/* The exclusive (load-linked, store-conditional) form of g9p_shared_access(): Apple's A9 LL/SC loop sets the size + * field to 16, W0 bit 15 alone (observed role). The retry jump is g5p_barrier.h's LL/SC retry: G9P_JMP_ANY with + * the condition bits 5..6 set. */ +#define G5P_SHARED_EXCLUSIVE 16u +#define G5P_JMP_SC_RETRY (G9P_JMP_ANY | 3u << 5) + +static inline void +g5p_shared_exclusive(struct g9p_code *c, bool store, unsigned r, + unsigned k, int offset_half) +{ + g9p_shared_access(c, G5P_SHARED_EXCLUSIVE, store, r, k, offset_half); +} + +/* The device-proven g5p_shared_barrier_fenced() LL/SC retry structure. + * SC always runs, including compare failure, so cmpxchg is strong. */ +static inline void +g5p_shared_atomic(struct g9p_code *c, nir_atomic_op op, unsigned old, + unsigned val, unsigned cmp, unsigned next, unsigned k, + int offset_half, unsigned st) +{ + assert(!c->fragment && k < 4096 && old < 32 && next < 32); + c->uses_stack = true; + g9p_wait(c); + /* push two levels */ + G9P_EMIT(c, G9P_W0_STACK, g9p_st_dst(c, st), g9p_st_src(c, st), + G9P_STACK_CTL | G9P_STACK_PUSH | 2 << G9P_STACK_LEVELS_SHIFT); + unsigned ll = c->n; + g5p_shared_exclusive(c, false, old, k, offset_half); + g9p_wait(c); + switch (op) { + case nir_atomic_op_iadd: + g9p_alu2(c, G9P_IADD, next, old, val, false); break; + case nir_atomic_op_iand: + g9p_alu2(c, G9P_AND, next, old, val, false); break; + case nir_atomic_op_ior: + g9p_alu2(c, G9P_OR, next, old, val, false); break; + case nir_atomic_op_ixor: + g9p_alu2(c, G9P_XOR, next, old, val, false); break; + case nir_atomic_op_xchg: + g9p_alu2(c, G9P_OR, next, val, val, false); break; + case nir_atomic_op_imin: + case nir_atomic_op_umin: + case nir_atomic_op_imax: + case nir_atomic_op_umax: + case nir_atomic_op_cmpxchg: { + bool cas = op == nir_atomic_op_cmpxchg; + bool is_min = op == nir_atomic_op_imin || op == nir_atomic_op_umin; + unsigned cond = cas ? G9P_EQ : + op == nir_atomic_op_imin || op == nir_atomic_op_imax ? G9P_SLT : G9P_ULT; + unsigned x = cas ? val : is_min ? old : val; + unsigned y = cas ? old : is_min ? val : old; + /* Same compare/select sequence as g9p_compile.c emit_select(). */ + g9p_add_imm(c, next, y, 0); + g9p_cmp_reg(c, cond, old, cas ? cmp : val, false); + g9p_push_if(c, st, false); + g9p_add_imm(c, next, x, 0); + g9p_pop_levels(c, st, 1); + break; + } + default: UNREACHABLE("unsupported shared atomic"); + } + g5p_shared_exclusive(c, true, next, k, offset_half); + unsigned retry = g9p_jump(c, G5P_JMP_SC_RETRY, 0); + g9p_jump_patch(c, retry, 2 * ((int)ll - (int)retry)); + /* pop both levels, the stack register still live (no last-read bit) */ + G9P_EMIT(c, G9P_W0_STACK, g9p_st_dst(c, st), g9p_st_src(c, st), + G9P_STACK_CTL | G9P_STACK_POP | 2 << G9P_STACK_LEVELS_SHIFT); + g9p_wait(c); +} +#endif diff --git a/src/imagination/g9p/g9p_a9_intrinsics.h b/src/imagination/g9p/g9p_a9_intrinsics.h index 1c812d16957..18baa37fa67 100644 --- a/src/imagination/g9p/g9p_a9_intrinsics.h +++ b/src/imagination/g9p/g9p_a9_intrinsics.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: MIT */ #include "../g5p/g5p_barrier.h" #include "../g5p/g5p_atomic.h" +#include "../g5p/g5p_shared_atomic.h" #include "../g5p/g5p_image.h" /* Static threadgroup operands: A9 host-compiler forms (compute.md §4). * TG0/1 hold the workgroup IDs x/y, TG2 the software-barrier counter, TG3 the ID z (G9P_A9_TG_WGID_Z). @@ -127,6 +128,51 @@ static bool a9_barrier(struct g9p_ctx *ctx, nir_intrinsic_instr *in) g9p_wait(ctx->c); return true; } +static bool a9_shared_atomic(struct g9p_ctx *ctx, nir_intrinsic_instr *in) +{ + if (!ctx->out->a9 || ctx->stage != MESA_SHADER_COMPUTE || + in->def.bit_size != 32 || in->def.num_components != 1 || + (nir_intrinsic_base(in) & 3)) return false; + nir_atomic_op op = nir_intrinsic_atomic_op(in); + bool cas = in->intrinsic == nir_intrinsic_shared_atomic_swap; + if (cas && op != nir_atomic_op_cmpxchg) return false; + switch (op) { + case nir_atomic_op_iadd: + case nir_atomic_op_imin: case nir_atomic_op_umin: + case nir_atomic_op_imax: case nir_atomic_op_umax: + case nir_atomic_op_iand: case nir_atomic_op_ior: case nir_atomic_op_ixor: + case nir_atomic_op_xchg: break; + case nir_atomic_op_cmpxchg: if (!cas) return false; break; + default: + mesa_logw("g5p: unsupported shared atomic op %u", op); + return false; + } + unsigned word = 4 + nir_intrinsic_base(in) / 4; + int index = -1; + if (nir_src_is_const(in->src[0])) { + unsigned byte = nir_src_as_uint(in->src[0]); + if (byte & 3) return false; + word += byte / 4; + } else { + int src = src_reg(ctx, nir_get_scalar(in->src[0].ssa, 0)); + int shift = const_reg(ctx, 2); + index = alloc_temp(ctx); + if (src < 0 || shift < 0 || index < 0) return false; + g9p_shift(ctx->c, index, src, shift, true); + g9p_shared_index(ctx->c, index, index, 2); /* a dword index */ + } + int val = src_reg(ctx, nir_get_scalar(in->src[cas ? 2 : 1].ssa, 0)); + int cmp = cas ? src_reg(ctx, nir_get_scalar(in->src[1].ssa, 0)) : 0; + int old = alloc_reg(ctx), next = alloc_temp(ctx); + if (val < 0 || cmp < 0 || old < 0 || next < 0 || word >= 4096) return false; + g5p_shared_atomic(ctx->c, op, old, val, cmp, next, word, + index < 0 ? -1 : 2 * index, ctx->st); + release(ctx, &in->instr); + ctx->reg[key(&in->def, 0)] = old; + if (!ctx->last_use[key(&in->def, 0)]) ctx->busy &= ~BITFIELD64_BIT(old); + return true; +} + static bool a9_ssbo_atomic(struct g9p_ctx *ctx, nir_intrinsic_instr *in) { if (!ctx->out->a9 || ctx->stage != MESA_SHADER_COMPUTE || @@ -222,6 +268,7 @@ static bool a9_image(struct g9p_ctx *ctx, nir_intrinsic_instr *in) #define G9P_A9_INTRINSICS \ if (ctx->out->a9) { switch (intr->intrinsic) { \ case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_load: return a9_image(ctx, intr); \ + case nir_intrinsic_shared_atomic: case nir_intrinsic_shared_atomic_swap: return a9_shared_atomic(ctx, intr); \ case nir_intrinsic_ssbo_atomic_swap: return a9_ssbo_cas(ctx, intr); \ case nir_intrinsic_ssbo_atomic: return a9_ssbo_atomic(ctx, intr); \ case nir_intrinsic_load_shared: case nir_intrinsic_store_shared: return a9_shared(ctx, intr); \ diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index b7f44c991a6..b2f0104ea4b 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -39,6 +39,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } From 6017514c51cee20cadb501c867df4c3758e2d698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:44 +0100 Subject: [PATCH 064/109] g9p: lower the integer and packing ops the G9P backend has no emitter for g9p_compile.c has no emitter for bit_count, bitfield_reverse, the 4x8 pack/unpack ops or ufind_msb/ifind_msb, so shaders that use them (findMSB, findLSB, bitCount, packUnorm4x8 ...) were refused on both Apple GPUs. The shared NIR options now lower bit_count, bitfield_reverse and the unorm/snorm 4x8 packs. A small pass lowers ufind_msb to a branchless binary search and ifind_msb to ufind_msb of the complement for negative values. nir_lower_alu runs in the optimisation loop. The A9's half-float packing keeps its own encoding. A10 programs without these ops are unchanged; A10 shaders with them now compile (compile-checked only). Assisted-by: LLM --- src/imagination/g9p/g9p_a9_compute.h | 2 + src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/g9p/g9p_compile.c | 76 ++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index 4be1fa1fa2f..7804372011f 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -99,6 +99,7 @@ a9_shared_size_align(nir_intrinsic_op op, uint8_t bytes, uint8_t bit_size, bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_ctx) { + nir->options = &nir_options_a9; const nir_lower_compute_system_values_options csv = { .lower_cs_local_id_to_index = true, .global_id_is_32bit = true, }; @@ -107,6 +108,7 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c lower_front(nir); NIR_PASS(_, nir, nir_shader_intrinsics_pass, a9_lower_subgroups_size1, nir_metadata_control_flow, NULL); + NIR_PASS(_, nir, nir_lower_alu); NIR_PASS(_, nir, nir_lower_vars_to_explicit_types, nir_var_mem_shared, glsl_get_natural_size_align_bytes); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_shared, nir_address_format_32bit_offset); const nir_lower_mem_access_bit_sizes_options shared_opts = { diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index b2f0104ea4b..21dfb838458 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -40,6 +40,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index e20c4130797..c06a47cfbb8 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -133,6 +133,8 @@ static const nir_shader_compiler_options nir_options = { .lower_fround_even = true, .lower_flrp32 = true, .lower_fsat = true, + .lower_bit_count = true, + .lower_bitfield_reverse = true, .lower_find_lsb = true, .lower_ifind_msb = true, .lower_iadd_sat = true, @@ -140,6 +142,52 @@ static const nir_shader_compiler_options nir_options = { .lower_device_index_to_zero = true, .lower_bitfield_insert = true, .lower_bitfield_extract = true, + .lower_pack_unorm_4x8 = true, + .lower_unpack_unorm_4x8 = true, + .lower_pack_snorm_4x8 = true, + .lower_unpack_snorm_4x8 = true, + .lower_pack_snorm_2x16 = true, + .lower_unpack_snorm_2x16 = true, + .lower_pack_unorm_2x16 = true, + .lower_unpack_unorm_2x16 = true, + .lower_extract_byte = true, + .lower_extract_word = true, + .lower_insert_byte = true, + .lower_insert_word = true, + .max_unroll_iterations = 0, +}; + +/* A9 compute options. */ +static const nir_shader_compiler_options nir_options_a9 = { + /* multiply-adds as one instruction (g9p_ffma()), not a mul and an add */ + .float_mul_add32 = nir_float_muladd_support_has_ffma | nir_float_muladd_support_fuse, + .lower_int64_options = ~0U, + .lower_fdiv = true, + .lower_iabs = true, + .lower_mul_high = true, + .lower_isign = true, + .lower_fmod = true, + .lower_fpow = true, + .lower_fsqrt = true, + .lower_ffract = true, + .lower_fceil = true, + .lower_ftrunc = true, + .lower_fround_even = true, + .lower_flrp32 = true, + .lower_fsat = true, + .lower_bit_count = true, + .lower_bitfield_reverse = true, + .lower_find_lsb = true, + .lower_ifind_msb = true, + .lower_iadd_sat = true, + .lower_mul_2x32_64 = true, + .lower_device_index_to_zero = true, + .lower_bitfield_insert = true, + .lower_bitfield_extract = true, + .lower_pack_unorm_4x8 = true, + .lower_unpack_unorm_4x8 = true, + .lower_pack_snorm_4x8 = true, + .lower_unpack_snorm_4x8 = true, .lower_pack_snorm_2x16 = true, .lower_unpack_snorm_2x16 = true, .lower_pack_unorm_2x16 = true, @@ -4000,6 +4048,27 @@ phis_scalar(const nir_instr *instr, const void *data) return 1; } +/* No native find-MSB emitter yet: branchless binary search, including zero + * and GLSL's signed negative-input convention. */ +static bool +lower_find_msb(nir_builder *b, nir_alu_instr *alu, void *data) +{ + if (alu->op != nir_op_ufind_msb && alu->op != nir_op_ifind_msb) + return false; + b->cursor = nir_before_instr(&alu->instr); + nir_def *x = nir_ssa_for_alu_src(b, alu, 0); + if (alu->op == nir_op_ifind_msb) + x = nir_bcsel(b, nir_ilt_imm(b, x, 0), nir_inot(b, x), x); + nir_def *t = x, *r = nir_imm_int(b, 0); + for (unsigned shift = x->bit_size / 2; shift; shift /= 2) { + nir_def *upper = nir_ugt_imm(b, t, (1ull << shift) - 1); + t = nir_bcsel(b, upper, nir_ushr_imm(b, t, shift), t); + r = nir_iadd(b, r, nir_bcsel(b, upper, nir_imm_int(b, shift), nir_imm_int(b, 0))); + } + nir_def_replace(&alu->def, nir_bcsel(b, nir_ieq_imm(b, x, 0), nir_imm_int(b, -1), r)); + return true; +} + static void optimize(nir_shader *nir) { @@ -4015,6 +4084,11 @@ optimize(nir_shader *nir) NIR_PASS(progress, nir, nir_opt_cse); NIR_PASS(progress, nir, nir_opt_constant_folding); NIR_PASS(progress, nir, nir_opt_algebraic); + /* Algebraic lowering of find_lsb/ifind_msb introduces ufind_msb. */ + NIR_PASS(progress, nir, nir_shader_alu_pass, lower_find_msb, + nir_metadata_control_flow, NULL); + /* Algebraic lowering of extended multiplies can introduce mul_high. */ + NIR_PASS(progress, nir, nir_lower_alu); NIR_PASS(progress, nir, nir_opt_dead_cf); NIR_PASS(progress, nir, nir_opt_remove_phis); } while (progress); @@ -4226,6 +4300,8 @@ lower_front(nir_shader *nir) glsl_get_natural_size_align_bytes); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_shared, nir_address_format_32bit_offset); } + NIR_PASS(_, nir, nir_shader_alu_pass, lower_find_msb, + nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_shader_intrinsics_pass, zero_descriptor_offset, nir_metadata_control_flow, NULL); } From 49529a7b3edf843f060a00cc3a8d939473d48a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:44 +0100 Subject: [PATCH 065/109] g9p: compute image stores for the A9's packed and raw formats A9 compute stored only raw R32_UINT images. Storage formats are now packed in NIR at compile time and written with Apple's forms: a raw four-dword store, a packed two-dword store with the native half packing (Apple's f32-to-f16 conversion), and the 2D dimension code, or the array one for array and cube-face writes, as Apple's A9 kernels encode them. Unsupported formats, image loads other than R32_UINT, and image atomics are refused by name. Assisted-by: LLM --- src/imagination/g5p/g5p_image.h | 42 +++++---- src/imagination/g9p/g9p_a9_alu.h | 25 ++++++ src/imagination/g9p/g9p_a9_compute.h | 45 ++++++++++ src/imagination/g9p/g9p_a9_intrinsics.h | 114 +++++++++++++++++------- src/imagination/g9p/g9p_a9_tex.h | 1 + src/imagination/g9p/g9p_compile.c | 9 +- 6 files changed, 185 insertions(+), 51 deletions(-) diff --git a/src/imagination/g5p/g5p_image.h b/src/imagination/g5p/g5p_image.h index 709f6f41487..f128ae00828 100644 --- a/src/imagination/g5p/g5p_image.h +++ b/src/imagination/g5p/g5p_image.h @@ -2,26 +2,36 @@ #ifndef G5P_IMAGE_H #define G5P_IMAGE_H #include "../g9p/g9p_isa.h" -/* RE §4 raw32-bit-channel image access. Format conversion is not implicit: - * only the pipeline's witnessed/probed raw format may use this form. - * - * The coordinate halfword's bit 15 (the sample encoder's coord_control) is - * set on reads only. Apple's A9 compiler (build/gpu/compute-isa-re out-img, - * out-image32: every texture write kernel) writes through integer - * coordinates as `f51a 01c0 0900 ...` with that bit clear; the bit appears on - * a write only with opcode f518 after the coordinates were converted to - * float. The old `f51a ... 8900` store mixed the two: its pixels reached - * memory, but a later job's imageLoad read pre-store data for the last - * workgroups written (release9d imgdiag1: 384 of 4096 words, always whole - * 8x8 groups of the tail, with or without glFinish between the jobs). */ +/* Apple's captured image and array shaders pack 16/8-bit channels into two dwords (the sample class with bit 1 + * clear, as g5p_usc_sample()'s narrow result) or supply four raw dwords; the captured array shaders use the 2D + * array dimension with integer x,y,layer coordinates. W0: that class byte and the dimension byte (all four + * components in its high nibble and the dimension in its low one, as g5p_usc_sample_ext()); W1: the data + * register's half, the integer-coordinate bit 6 (texture_base()'s read bit), the store bit 7 and the descriptor's + * qword at bit 9. Keep raw32's qualified coordinate control and completion unchanged. */ +#define G5P_IMAGE_CLASS 0x1a /* texture_base()'s sample class */ +#define G5P_IMAGE_WIDE 0x02 /* clear: 16/8-bit channels packed into two dwords */ +#define G5P_IMAGE_RGBA 0xf0 +#define G5P_IMAGE_DIM_2D 0x5 +#define G5P_IMAGE_INTEGER_COORDS 0x40 +#define G5P_IMAGE_STORE 0x80 static inline void -g5p_image_raw32(struct g9p_code *c, bool store, unsigned data, - unsigned coords, unsigned descriptor_dword, unsigned lod_dword) +g5p_image_access(struct g9p_code *c, bool store, bool packed, bool array, + unsigned data, unsigned coords, unsigned descriptor_dword, unsigned lod_dword) { - assert(!c->fragment && data+3<32 && coords+1<32 && !(descriptor_dword&1) && lod_dword<32); - G9P_EMIT(c,0xf51a,0x0100 | (descriptor_dword/2)<<9 | (store?0xc0:0x40) | 2*data, + assert(!c->fragment && data + (packed ? 1 : 3) < 32 && coords + (array ? 2 : 1) < 32 && + !(descriptor_dword & 1) && lod_dword < 32); + const unsigned dim = G5P_IMAGE_RGBA | (array ? G9P_TEX_2D_ARRAY : G5P_IMAGE_DIM_2D); + G9P_EMIT(c, dim << 8 | (G5P_IMAGE_CLASS & ~(packed ? G5P_IMAGE_WIDE : 0)), + G9P_CONT | (descriptor_dword / 2) << 9 | G5P_IMAGE_INTEGER_COORDS | (store ? G5P_IMAGE_STORE : 0) | + 2 * data, (store?0x0100:0x8100) | coords<<9 | (store?0:8),0x0140 | lod_dword, store?0x8100:0x8180,0x0008); if(!store)g9p_wait(c); } +static inline void +g5p_image_raw32(struct g9p_code *c, bool store, unsigned data, + unsigned coords, unsigned descriptor_dword, unsigned lod_dword) +{ + g5p_image_access(c, store, false, false, data, coords, descriptor_dword, lod_dword); +} #endif diff --git a/src/imagination/g9p/g9p_a9_alu.h b/src/imagination/g9p/g9p_a9_alu.h index 70706829cbf..3d676ff76c8 100644 --- a/src/imagination/g9p/g9p_a9_alu.h +++ b/src/imagination/g9p/g9p_a9_alu.h @@ -70,6 +70,31 @@ a9_alu_fma_literals(struct g9p_ctx *ctx, nir_alu_instr *alu, nir_scalar s[3], bo ctx->reg[key(&alu->def, 0)] = dst; return true; } +/* wa_h/wa_f8: Apple's f32-to-f16 conversion writes one half of a bank-1 + * dword. Convert both scalar sources without a software float pack. */ +static bool +a9_alu_pack_half(struct g9p_ctx *ctx, nir_alu_instr *alu, nir_scalar s[3]) +{ + int dst = alloc_reg(ctx); + if (dst < 0) return false; + ctx->temps |= BITFIELD64_BIT(dst); + for (unsigned i = 0; i < 2; i++) { + int r = src_reg(ctx, s[i]); + if (r < 0 || (r = one_src(ctx, r)) < 0) return false; + unsigned at = ctx->c->n; + g9p_f16_of(ctx->c, dst, r); + ctx->c->w[at + 1] |= i; /* the high half: bit 0 of the destination half index, as Apple's half packs */ + } + release(ctx, &alu->instr); + ctx->busy |= BITFIELD64_BIT(dst); + ctx->reg[key(&alu->def, 0)] = dst; + return true; +} +# define G9P_A9_ALU_PACK_HALF() do { \ + if (ctx->out->a9 && ctx->stage == MESA_SHADER_COMPUTE && \ + alu->op == nir_op_pack_half_2x16_split) \ + return a9_alu_pack_half(ctx, alu, s); \ + } while (0) # define G9P_A9_BRANCH_IMMEDIATE() do { \ uint32_t value; \ if (ctx->out->a9 && !is_float && a >= 0 && \ diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index 7804372011f..92882d79adc 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -97,6 +97,49 @@ a9_shared_size_align(nir_intrinsic_op op, uint8_t bytes, uint8_t bit_size, return (nir_mem_access_size_align) { n, 32, 4 * n, nir_mem_access_shift_method_scalar }; } +/* Specialise storage writes before ALU lowering. Two data components mark + * Apple's packed 0x18 form; raw 0x1a always has four dwords. Cube image + * coordinates are already x,y,face (or 6*cube+face) in SPIR-V/NIR. */ +static bool +a9_pack_image_store(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + if (in->intrinsic != nir_intrinsic_image_deref_store) + return false; + nir_variable *v = nir_deref_instr_get_variable(nir_src_as_deref(in->src[0])); + if (!v || nir_src_bit_size(in->src[3]) != 32) + return false; + b->cursor = nir_before_instr(&in->instr); + nir_def *value = in->src[3].ssa, *lo, *hi = nir_imm_int(b, 0); + switch (v->data.image.format) { + case PIPE_FORMAT_R32_UINT: case PIPE_FORMAT_R32_SINT: case PIPE_FORMAT_R32_FLOAT: + value = nir_vec4(b, nir_channel(b, value, 0), hi, hi, hi); + break; + case PIPE_FORMAT_R32G32B32A32_UINT: case PIPE_FORMAT_R32G32B32A32_SINT: + case PIPE_FORMAT_R32G32B32A32_FLOAT: + return false; + case PIPE_FORMAT_R8G8B8A8_UNORM: + lo = nir_pack_unorm_4x8(b, value); + value = nir_vec2(b, lo, hi); + break; + case PIPE_FORMAT_R16G16B16A16_FLOAT: + hi = nir_pack_half_2x16(b, nir_channels(b, value, 0xc)); + FALLTHROUGH; + case PIPE_FORMAT_R16G16_FLOAT: + lo = nir_pack_half_2x16(b, nir_channels(b, value, 0x3)); + value = nir_vec2(b, lo, hi); + break; + case PIPE_FORMAT_R16_FLOAT: + lo = nir_pack_half_2x16(b, nir_vec2(b, nir_channel(b, value, 0), nir_imm_float(b, 0))); + value = nir_vec2(b, lo, hi); + break; + default: + return false; /* a9_image() records the format by name */ + } + nir_src_rewrite(&in->src[3], value); + in->num_components = value->num_components; + return true; +} + bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_ctx) { nir->options = &nir_options_a9; @@ -108,6 +151,8 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c lower_front(nir); NIR_PASS(_, nir, nir_shader_intrinsics_pass, a9_lower_subgroups_size1, nir_metadata_control_flow, NULL); + NIR_PASS(_, nir, nir_shader_intrinsics_pass, a9_pack_image_store, + nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_lower_alu); NIR_PASS(_, nir, nir_lower_vars_to_explicit_types, nir_var_mem_shared, glsl_get_natural_size_align_bytes); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_shared, nir_address_format_32bit_offset); diff --git a/src/imagination/g9p/g9p_a9_intrinsics.h b/src/imagination/g9p/g9p_a9_intrinsics.h index 18baa37fa67..58efcaf6d50 100644 --- a/src/imagination/g9p/g9p_a9_intrinsics.h +++ b/src/imagination/g9p/g9p_a9_intrinsics.h @@ -226,47 +226,93 @@ static bool a9_ssbo_cas(struct g9p_ctx *ctx, nir_intrinsic_instr *in) } static bool a9_image(struct g9p_ctx *ctx, nir_intrinsic_instr *in) { - bool store=in->intrinsic==nir_intrinsic_image_deref_store; - if(!ctx->out->a9 || ctx->stage!=MESA_SHADER_COMPUTE || - nir_intrinsic_image_dim(in)!=GLSL_SAMPLER_DIM_2D || nir_intrinsic_image_array(in))return false; - nir_deref_instr *deref=nir_src_as_deref(in->src[0]); - nir_variable *v=nir_deref_instr_get_variable(deref); - if(!v || deref->deref_type!=nir_deref_type_var || v->data.image.format!=PIPE_FORMAT_R32_UINT || - !nir_src_is_const(in->src[store?4:3]) || nir_src_as_uint(in->src[store?4:3])!=0)return false; - if(ctx->out->num_textures && (ctx->out->textures[0].set!=v->data.descriptor_set || - ctx->out->textures[0].binding!=v->data.binding))return false; - ctx->out->num_textures=1; - ctx->out->textures[0].set=v->data.descriptor_set; - ctx->out->textures[0].binding=v->data.binding; - int co=alloc_pair(ctx); - if(co<0 || co>30)return false; - ctx->temps |= BITFIELD64_RANGE(co,2); - for(unsigned j=0;j<2;j++) { - int r=src_reg(ctx,nir_get_scalar(in->src[1].ssa,j)); - if(r<0)return false; - g9p_alu2(ctx->c,G9P_OR,co+j,r,r,false); + bool store = in->intrinsic == nir_intrinsic_image_deref_store; + enum glsl_sampler_dim dim = nir_intrinsic_image_dim(in); + bool array = nir_intrinsic_image_array(in); + if (!ctx->out->a9 || ctx->stage != MESA_SHADER_COMPUTE) + return false; + if ((dim != GLSL_SAMPLER_DIM_2D && !(store && dim == GLSL_SAMPLER_DIM_CUBE)) || + (!store && array)) { + char why[128]; + snprintf(why, sizeof(why), "image %s dimension %s is_array=%u unsupported", + store ? "store" : "load", (dim == GLSL_SAMPLER_DIM_CUBE ? "cube" : dim == GLSL_SAMPLER_DIM_3D ? "3D" : + dim == GLSL_SAMPLER_DIM_2D ? "2D" : "other"), array); + a9_tex_refuse(ctx, why); + return false; + } + nir_deref_instr *deref = nir_src_as_deref(in->src[0]); + nir_variable *v = nir_deref_instr_get_variable(deref); + if (!v || deref->deref_type != nir_deref_type_var) { + a9_tex_refuse(ctx, "image access without a constant storage binding"); + return false; } - int data=alloc_range(ctx,4); - if(data<0 || data>28)return false; - if(store) { - if(nir_src_bit_size(in->src[3])!=32 || nir_src_num_components(in->src[3])!=4)return false; - ctx->temps |= BITFIELD64_RANGE(data,4); - for(unsigned j=0;j<4;j++) { - int r=src_reg(ctx,nir_get_scalar(in->src[3].ssa,j)); - if(r<0)return false; - g9p_alu2(ctx->c,G9P_OR,data+j,r,r,false); + enum pipe_format format = v->data.image.format; + bool packed = false, supported = format == PIPE_FORMAT_R32_UINT; + if (store) switch (format) { + case PIPE_FORMAT_R32_UINT: case PIPE_FORMAT_R32_SINT: case PIPE_FORMAT_R32_FLOAT: + case PIPE_FORMAT_R32G32B32A32_UINT: case PIPE_FORMAT_R32G32B32A32_SINT: + case PIPE_FORMAT_R32G32B32A32_FLOAT: + supported = true; break; + case PIPE_FORMAT_R8G8B8A8_UNORM: case PIPE_FORMAT_R16G16B16A16_FLOAT: + case PIPE_FORMAT_R16G16_FLOAT: case PIPE_FORMAT_R16_FLOAT: + supported = packed = true; break; + default: break; + } + if (!supported) { + char why[160]; + snprintf(why, sizeof(why), "image %s format %s (%s)", store ? "store" : "load", + util_format_short_name(format), store ? "no A9 pack path yet" : "R32_UINT only"); + a9_tex_refuse(ctx, why); + return false; + } + if (!nir_src_is_const(in->src[store ? 4 : 3]) || + nir_src_as_uint(in->src[store ? 4 : 3]) != 0) { + a9_tex_refuse(ctx, "image access at a nonzero or dynamic level"); + return false; + } + if (ctx->out->num_textures && (ctx->out->textures[0].set != v->data.descriptor_set || + ctx->out->textures[0].binding != v->data.binding)) { + a9_tex_refuse(ctx, "more than one storage image binding"); + return false; + } + ctx->out->num_textures = 1; + ctx->out->textures[0].set = v->data.descriptor_set; + ctx->out->textures[0].binding = v->data.binding; + unsigned coords = array || dim == GLSL_SAMPLER_DIM_CUBE ? 3 : 2; + int co = alloc_range(ctx, coords); + if (co < 0 || co + coords > 32) return false; + ctx->temps |= BITFIELD64_RANGE(co, coords); + for (unsigned j = 0; j < coords; j++) { + int r = src_reg(ctx, nir_get_scalar(in->src[1].ssa, j)); + if (r < 0) return false; + g9p_alu2(ctx->c, G9P_OR, co + j, r, r, false); + } + unsigned words = packed ? 2 : 4; + int data = alloc_range(ctx, words); + if (data < 0 || data + words > 32) return false; + if (store) { + if (nir_src_bit_size(in->src[3]) != 32 || nir_src_num_components(in->src[3]) != words) + return false; + ctx->temps |= BITFIELD64_RANGE(data, words); + for (unsigned j = 0; j < words; j++) { + int r = src_reg(ctx, nir_get_scalar(in->src[3].ssa, j)); + if (r < 0) return false; + g9p_alu2(ctx->c, G9P_OR, data + j, r, r, false); } - } else if(in->def.bit_size!=32 || in->def.num_components>4)return false; - g5p_image_raw32(ctx->c,store,data,co,40,31); - release(ctx,&in->instr); - if(!store) for(unsigned j=0;j<4;j++) { - ctx->reg[key(&in->def,j)]=data+j; - if(j>=in->def.num_components || !ctx->last_use[key(&in->def,j)])ctx->busy &= ~BITFIELD64_BIT(data+j); + } else if (in->def.bit_size != 32 || in->def.num_components > 4) return false; + g5p_image_access(ctx->c, store, packed, coords == 3, data, co, 40, 31); + release(ctx, &in->instr); + if (!store) for (unsigned j = 0; j < 4; j++) { + ctx->reg[key(&in->def, j)] = data + j; + if (j >= in->def.num_components || !ctx->last_use[key(&in->def, j)]) + ctx->busy &= ~BITFIELD64_BIT(data + j); } return true; } #define G9P_A9_INTRINSICS \ if (ctx->out->a9) { switch (intr->intrinsic) { \ + case nir_intrinsic_image_deref_atomic: case nir_intrinsic_image_deref_atomic_swap: \ + a9_tex_refuse(ctx, "image atomics unsupported on A9"); return false; \ case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_load: return a9_image(ctx, intr); \ case nir_intrinsic_shared_atomic: case nir_intrinsic_shared_atomic_swap: return a9_shared_atomic(ctx, intr); \ case nir_intrinsic_ssbo_atomic_swap: return a9_ssbo_cas(ctx, intr); \ diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index 21dfb838458..d67478c1112 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -41,6 +41,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index c06a47cfbb8..97ab0136447 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -1217,6 +1217,7 @@ emit_alu(struct g9p_ctx *ctx, nir_alu_instr *alu) } for (unsigned i = 0; i < num_inputs; i++) s[i] = nir_get_scalar(alu->src[i].src.ssa, alu->src[i].swizzle[0]); + G9P_A9_ALU_PACK_HALF(); G9P_A9_ALU_LITERAL_FMA(); /* compared where it is used (emit_branch_cmp()) */ if (branch_compare(&alu->def)) @@ -4862,7 +4863,8 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, /* A9 compute reads texels with the sampler's read form and the fetch sampler (g9p_a9_tex.h) */ if (!(out->a9 && stage == MESA_SHADER_COMPUTE)) NIR_PASS(_, nir, nir_shader_tex_pass, lower_txf, nir_metadata_control_flow, NULL); - NIR_PASS(_, nir, nir_shader_instructions_pass, lower_half_packing, nir_metadata_control_flow, NULL); + NIR_PASS(_, nir, nir_shader_instructions_pass, lower_half_packing, nir_metadata_control_flow, + (void *)(uintptr_t)(out->a9 && stage == MESA_SHADER_COMPUTE)); } /* integer division through floats: no such instruction (GLSL ES 1.00 int and ivec /) */ { @@ -5387,6 +5389,11 @@ lower_half_packing(nir_builder *b, nir_instr *instr, void *data) nir_def *src = nir_ssa_for_alu_src(b, alu, 0); if (alu->op == nir_op_pack_half_2x16) { + if (data) { + nir_def_replace(&alu->def, nir_pack_half_2x16_split(b, nir_channel(b, src, 0), + nir_channel(b, src, 1))); + return true; + } nir_def_replace(&alu->def, nir_ior(b, f32_to_half(b, nir_channel(b, src, 0)), nir_ishl_imm(b, f32_to_half(b, nir_channel(b, src, 1)), 16))); } else { From 1113ab97003b17958077f24ea393cb8376c8488f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:44 +0100 Subject: [PATCH 066/109] pvr/g5p: storage image formats; cube storage as face arrays Optimal-tiling STORAGE_IMAGE is advertised for exactly the ten formats the A9 compute compiler stores. Cube storage descriptors become 2D arrays (cube coordinates are already face-linear in NIR), and the storage image layout supports array views. STORAGE_WRITE_WITHOUT_FORMAT and storage atomics stay unadvertised. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_tex.h | 1 + .../vulkan/pvr_arch_descriptor_set.c | 2 +- src/imagination/vulkan/pvr_arch_g5p_tex.c | 28 +++++++++++++++---- src/imagination/vulkan/pvr_g5p_format_table.h | 26 +++++++++-------- src/imagination/vulkan/pvr_g5p_formats.c | 7 +++++ src/imagination/vulkan/pvr_g5p_hooks.h | 7 +++++ src/imagination/vulkan/pvr_g5p_image.c | 1 + src/imagination/vulkan/pvr_g5p_tex.h | 5 +++- 8 files changed, 59 insertions(+), 18 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index d67478c1112..398c51a9538 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -42,6 +42,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } diff --git a/src/imagination/vulkan/pvr_arch_descriptor_set.c b/src/imagination/vulkan/pvr_arch_descriptor_set.c index 68091566380..9669970f9cb 100644 --- a/src/imagination/vulkan/pvr_arch_descriptor_set.c +++ b/src/imagination/vulkan/pvr_arch_descriptor_set.c @@ -214,7 +214,7 @@ write_storage_image(const struct pvr_descriptor_set *set, struct pvr_image_descriptor storage_image_desc = image_view->image_state[is_cube ? PVR_TEXTURE_STATE_STORAGE : PVR_TEXTURE_STATE_SAMPLE]; - + PVR_G5P_HOOK(storage_image, dev_info, image_view, &storage_image_desc) if (!PVR_G5P_NATIVE_STORAGE(dev_info) && !PVR_HAS_FEATURE(dev_info, tpu_extended_integer_lookup)) { struct ROGUE_TEXSTATE_STRIDE_IMAGE_WORD1 word1; ROGUE_TEXSTATE_STRIDE_IMAGE_WORD1_unpack(&storage_image_desc.words[1], diff --git a/src/imagination/vulkan/pvr_arch_g5p_tex.c b/src/imagination/vulkan/pvr_arch_g5p_tex.c index cca160c64e3..7d8321db3bf 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_tex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_tex.c @@ -38,7 +38,7 @@ g5p_tex_format(VkFormat format, struct pvr_g5p_tex_format *out) /* The three qwords of a view, or the reason it has none */ static const char * -g5p_view_state(const struct pvr_image_view *iview, uint64_t q[3]) +g5p_view_state(const struct pvr_image_view *iview, bool storage, uint64_t q[3]) { const struct pvr_image *image = pvr_image_view_get_image(iview); const struct pvr_image_plane *plane = pvr_single_plane_const(image); @@ -57,7 +57,10 @@ g5p_view_state(const struct pvr_image_view *iview, uint64_t q[3]) const bool meta_2d = (iview->vk.create_flags & VK_IMAGE_VIEW_CREATE_DRIVER_INTERNAL_BIT_MESA) && iview->vk.view_type == VK_IMAGE_VIEW_TYPE_2D_ARRAY && iview->vk.layer_count == 1 && image->vk.image_type == VK_IMAGE_TYPE_2D; - const VkImageViewType view_type = meta_2d ? VK_IMAGE_VIEW_TYPE_2D : iview->vk.view_type; + const bool storage_cube = storage && (iview->vk.view_type == VK_IMAGE_VIEW_TYPE_CUBE || + iview->vk.view_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY); + const VkImageViewType view_type = storage_cube ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : + meta_2d ? VK_IMAGE_VIEW_TYPE_2D : iview->vk.view_type; if (image->memlayout == PVR_MEMLAYOUT_LINEAR) { const struct pvr_mip_level *level = &plane->mip_levels[0]; @@ -164,11 +167,11 @@ g5p_view_state(const struct pvr_image_view *iview, uint64_t q[3]) * The sampled state of a view: q0, q1 in words[], q2 in meta[0..1], * meta[2..3] zero, the 32 bytes the backend's image upload reads. */ -const char * -PVR_PER_ARCH(g5p_pack_image_view)(const struct pvr_image_view *iview, struct pvr_image_descriptor *desc) +static const char * +g5p_pack_view(const struct pvr_image_view *iview, bool storage, struct pvr_image_descriptor *desc) { uint64_t q[3]; - const char *why = g5p_view_state(iview, q); + const char *why = g5p_view_state(iview, storage, q); static_assert(sizeof(desc->words) + sizeof(desc->meta) == 32, "G5P image state is 32 bytes"); memset(desc, 0, sizeof(*desc)); @@ -191,6 +194,21 @@ PVR_PER_ARCH(g5p_pack_image_view)(const struct pvr_image_view *iview, struct pvr return NULL; } +const char * +PVR_PER_ARCH(g5p_pack_image_view)(const struct pvr_image_view *iview, struct pvr_image_descriptor *desc) +{ + return g5p_pack_view(iview, false, desc); +} + +/* Vulkan layer_count already counts faces (six for each cube). Storage + * descriptors use that same face order as a 2D array; sampled state stays cube. */ +const char * +PVR_PER_ARCH(g5p_pack_storage_image_view)(const struct pvr_image_view *iview, + struct pvr_image_descriptor *desc) +{ + return g5p_pack_view(iview, true, desc); +} + /* * Both forms of a sampler's word: the general one (Vulkan NEAREST * mipmapping as Apple's mip-nearest) in words[0], for standalone samplers diff --git a/src/imagination/vulkan/pvr_g5p_format_table.h b/src/imagination/vulkan/pvr_g5p_format_table.h index 8f7bab5c983..68797f3d243 100644 --- a/src/imagination/vulkan/pvr_g5p_format_table.h +++ b/src/imagination/vulkan/pvr_g5p_format_table.h @@ -64,6 +64,7 @@ enum pvr_g5p_fmt_flags { PVR_G5P_FMT_BLEND = 1u << 3, /* blending into it (the fragment program's) */ PVR_G5P_FMT_COMPRESSED = 1u << 4, /* blocks (4x4 ETC2/EAC, 4x4..12x12 ASTC), twiddled only */ PVR_G5P_FMT_INTEGER = 1u << 5, /* integer texel: texelFetch/unfiltered */ + PVR_G5P_FMT_STORAGE = 1u << 6, /* compute writes with a declared format */ /* narrow integer render rows remain unadvertised probes */ PVR_G5P_FMT_PROBE_SAMPLED = 1u << 8, /* the sampled bits are a probe */ PVR_G5P_FMT_PROBE_RENDER = 1u << 9, /* the render (and blend) bits are a probe */ @@ -115,7 +116,7 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { /* 8_8_8_8: pass/pass-rgba (BGRA8, RGBA8), fmt-srgb8 (q1 bit 45). The * A8B8G8R8 packed formats are the R8G8B8A8 bytes on a little-endian GPU. */ G5P_FMT(B8G8R8A8_UNORM, 4, false, G5P_SF | G5P_RT, G5P1B, 0x28, G5P_SEL_BGRA), - G5P_FMT(R8G8B8A8_UNORM, 4, false, G5P_SF | G5P_RT, G5P1B, 0x28, G5P_SEL_RGBA), + G5P_FMT(R8G8B8A8_UNORM, 4, false, G5P_SF | G5P_RT | PVR_G5P_FMT_STORAGE, G5P1B, 0x28, G5P_SEL_RGBA), G5P_FMT(A8B8G8R8_UNORM_PACK32, 4, false, G5P_SF | G5P_RT, G5P1B, 0x28, G5P_SEL_RGBA), G5P_FMT(B8G8R8A8_SRGB, 4, true, G5P_SF | G5P_RT, G5P1B, 0x28, G5P_SEL_BGRA), G5P_FMT(R8G8B8A8_SRGB, 4, true, G5P_SF | G5P_RT, G5P1B, 0x28, G5P_SEL_RGBA), @@ -144,13 +145,13 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { G5P_FMT(R16G16B16A16_SNORM, 8, false, G5P_SF, G5P1B, 0xb2, G5P_SEL_RGBA), /* half float (t9-t11, fmt-rg16f, fmt-rgba16f), rendered with the two-word * program (RGBA16F: an 8-byte PBE pixel) */ - G5P_FMT(R16_SFLOAT, 2, false, G5P_SF | G5P_RT, G5P1B, 0x209, G5P_SEL_R001), - G5P_FMT(R16G16_SFLOAT, 4, false, G5P_SF | G5P_RT, G5P1B, 0x223, G5P_SEL_RG01), - G5P_FMT(R16G16B16A16_SFLOAT, 8, false, G5P_SF | G5P_RT, G5P1B, 0x232, G5P_SEL_RGBA), + G5P_FMT(R16_SFLOAT, 2, false, G5P_SF | G5P_RT | PVR_G5P_FMT_STORAGE, G5P1B, 0x209, G5P_SEL_R001), + G5P_FMT(R16G16_SFLOAT, 4, false, G5P_SF | G5P_RT | PVR_G5P_FMT_STORAGE, G5P1B, 0x223, G5P_SEL_RG01), + G5P_FMT(R16G16B16A16_SFLOAT, 8, false, G5P_SF | G5P_RT | PVR_G5P_FMT_STORAGE, G5P1B, 0x232, G5P_SEL_RGBA), /* 32-bit float (fmt-r32f, t12-t13): the A9 does not filter 32-bit float */ - G5P_FMT(R32_SFLOAT, 4, false, G5P_S, G5P1B, 0x221, G5P_SEL_R001), + G5P_FMT(R32_SFLOAT, 4, false, G5P_S | PVR_G5P_FMT_STORAGE, G5P1B, 0x221, G5P_SEL_R001), G5P_FMT(R32G32_SFLOAT, 8, false, G5P_S, G5P1B, 0x231, G5P_SEL_RG01), - G5P_FMT(R32G32B32A32_SFLOAT, 16, false, G5P_S, G5P1B, 0x238, G5P_SEL_RGBA), + G5P_FMT(R32G32B32A32_SFLOAT, 16, false, G5P_S | PVR_G5P_FMT_STORAGE, G5P1B, 0x238, G5P_SEL_RGBA), /* packed float (t14-t15). Apple packs RG11B10F in a 66-byte program; the * PBE packs it from the two-word program's f16 too, exactly. */ G5P_FMT(B10G11R11_UFLOAT_PACK32, 4, false, G5P_SF | G5P_RT, G5P1B, 0x225, G5P_SEL_RGB1), @@ -190,9 +191,9 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { G5P_FMT(R16_UINT, 2, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x109, G5P_SEL_R001), G5P_FMT(R16G16_UINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x123, G5P_SEL_RG01), G5P_FMT(R16G16B16A16_UINT, 8, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x132, G5P_SEL_RGBA), - G5P_FMT(R32_UINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x121, G5P_SEL_R001), + G5P_FMT(R32_UINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_STORAGE, G5P1B, 0x121, G5P_SEL_R001), G5P_FMT(R32G32_UINT, 8, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x131, G5P_SEL_RG01), - G5P_FMT(R32G32B32A32_UINT, 16, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x138, G5P_SEL_RGBA), + G5P_FMT(R32G32B32A32_UINT, 16, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_STORAGE, G5P1B, 0x138, G5P_SEL_RGBA), G5P_FMT(R8_SINT, 1, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x180, G5P_SEL_R001), G5P_FMT(R8G8_SINT, 2, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x18a, G5P_SEL_RG01), G5P_FMT(R8G8B8A8_SINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x1a8, G5P_SEL_RGBA), @@ -200,9 +201,9 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { G5P_FMT(R16_SINT, 2, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x189, G5P_SEL_R001), G5P_FMT(R16G16_SINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x1a3, G5P_SEL_RG01), G5P_FMT(R16G16B16A16_SINT, 8, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x1b2, G5P_SEL_RGBA), - G5P_FMT(R32_SINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x1a1, G5P_SEL_R001), + G5P_FMT(R32_SINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_STORAGE, G5P1B, 0x1a1, G5P_SEL_R001), G5P_FMT(R32G32_SINT, 8, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x1b1, G5P_SEL_RG01), - G5P_FMT(R32G32B32A32_SINT, 16, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x1b8, G5P_SEL_RGBA), + G5P_FMT(R32G32B32A32_SINT, 16, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_STORAGE, G5P1B, 0x1b8, G5P_SEL_RGBA), /* ETC2/EAC (g3b-compressed t0-t7; textypes etc2, etc2-8x8 and eac exact * on the device), twiddled 4x4 blocks; sRGB is q1 bit 45 as everywhere. * Uploads are upstream's block-unit copies through the CPU twiddle; blits @@ -298,7 +299,8 @@ pvr_g5p_format_flags(const struct pvr_g5p_format *f, bool probe) * compressed sources decoded per block (pvr_arch_g5p_layout.c); * - COLOR_ATTACHMENT (BLEND, BLIT_DST) for rendered rows, in either tiling * (a linear 2D target is what the PBE writes); - * - nothing else: no storage images (no compute on G5P), no texel buffers. + * - STORAGE_IMAGE for the declared-format compute pack paths, optimal only; + * no formatless image writes/reads or storage atomics, no texel buffers. * linear: the features of VK_IMAGE_TILING_LINEAR. */ static inline VkFormatFeatureFlags2 @@ -322,6 +324,8 @@ pvr_g5p_format_features(const struct pvr_g5p_format *f, bool linear, bool probe) if (flags & PVR_G5P_FMT_BLEND) r |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT; } + if ((flags & PVR_G5P_FMT_STORAGE) && !linear) + r |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT; return r; } diff --git a/src/imagination/vulkan/pvr_g5p_formats.c b/src/imagination/vulkan/pvr_g5p_formats.c index 9421cfda431..09f1d2919dd 100644 --- a/src/imagination/vulkan/pvr_g5p_formats.c +++ b/src/imagination/vulkan/pvr_g5p_formats.c @@ -97,6 +97,13 @@ pvr_g5p_image_format_check(const VkPhysicalDeviceImageFormatInfo2 *info, VkImage why = "3D block-compressed images (the A9's 3D order is built for texels only)"; else if (info->type == VK_IMAGE_TYPE_3D && tiling != VK_IMAGE_TILING_OPTIMAL) why = "linear 3D images (Apple samples 3D twiddled only)"; + if (!why && (info->usage & VK_IMAGE_USAGE_STORAGE_BIT)) { + const struct pvr_g5p_format *f = pvr_g5p_format_find(info->format); + if (!(f->flags & PVR_G5P_FMT_STORAGE)) + why = "no A9 storage pack path for this format"; + else if (info->type != VK_IMAGE_TYPE_2D || tiling != VK_IMAGE_TILING_OPTIMAL) + why = "A9 storage images require optimal 2D images (including arrays/cube faces)"; + } if (!why) return VK_SUCCESS; mesa_logd("G5P: no %s image of %s, tiling %s, usage %#x, flags %#x: %s", vk_ImageType_to_str(info->type), diff --git a/src/imagination/vulkan/pvr_g5p_hooks.h b/src/imagination/vulkan/pvr_g5p_hooks.h index 27b38d91392..97682e7cd79 100644 --- a/src/imagination/vulkan/pvr_g5p_hooks.h +++ b/src/imagination/vulkan/pvr_g5p_hooks.h @@ -47,6 +47,13 @@ pvr_arch_g5p_pack_image_view( \ iview, &(iview)->image_state[PVR_TEXTURE_STATE_SAMPLE]); +/* Only storage descriptors reinterpret cube faces as a 2D array. */ +#define PVR_G5P_HOOK_storage_image(dev_info, iview, desc) \ + if (PVR_HAS_FEATURE(dev_info, apple_g5p) && \ + ((iview)->vk.view_type == VK_IMAGE_VIEW_TYPE_CUBE || \ + (iview)->vk.view_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY)) \ + pvr_arch_g5p_pack_storage_image_view(iview, desc); + /* Release integration of the vertex-store capability, preserving shared lines. */ /* Also the polygon-mode and indirect lanes' feature lines (their * pvr_g5p_publish_hooks.h, which this base no longer has): line and point diff --git a/src/imagination/vulkan/pvr_g5p_image.c b/src/imagination/vulkan/pvr_g5p_image.c index b207212d4ea..7b162f12c26 100644 --- a/src/imagination/vulkan/pvr_g5p_image.c +++ b/src/imagination/vulkan/pvr_g5p_image.c @@ -62,6 +62,7 @@ pvr_g5p_image_layout(struct pvr_image *image, unsigned stride_align) return; const bool twiddled = is3d || image->vk.mip_levels > 1 || + (image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT) || (image->vk.create_flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) || (image->vk.array_layers > 1 && (image->vk.usage & VK_IMAGE_USAGE_SAMPLED_BIT)) || vk_format_is_compressed(image->vk.format); diff --git a/src/imagination/vulkan/pvr_g5p_tex.h b/src/imagination/vulkan/pvr_g5p_tex.h index 5e96d790931..4a130bfdcfd 100644 --- a/src/imagination/vulkan/pvr_g5p_tex.h +++ b/src/imagination/vulkan/pvr_g5p_tex.h @@ -40,7 +40,7 @@ struct pvr_sampler_descriptor; /* What the texture state needs of a format (pvr_arch_g5p_tex.c looks it * up). Lane (f)'s format table, pvr_arch_g5p_formats.c, is to provide it; * until then pvr_arch_g5p_tex.c has the formats of the rendered and - * textured rungs only. */ + * textured cases only. */ struct pvr_g5p_tex_format { VkFormat format; uint32_t key; /* PVR_G5P_TEX_KEY(): format code and channel selectors */ @@ -51,12 +51,15 @@ struct pvr_g5p_tex_format { const char *PVR_PER_ARCH(g5p_pack_image_view)(const struct pvr_image_view *iview, struct pvr_image_descriptor *desc); +const char *PVR_PER_ARCH(g5p_pack_storage_image_view)(const struct pvr_image_view *iview, + struct pvr_image_descriptor *desc); const char *PVR_PER_ARCH(g5p_pack_sampler)(const VkSamplerCreateInfo *info, struct pvr_sampler_descriptor *desc); void PVR_PER_ARCH(g5p_combine_sampler)(struct pvr_combined_image_sampler_descriptor *desc, const struct pvr_image_view *iview); # define pvr_arch_g5p_pack_image_view PVR_PER_ARCH(g5p_pack_image_view) +# define pvr_arch_g5p_pack_storage_image_view PVR_PER_ARCH(g5p_pack_storage_image_view) # define pvr_arch_g5p_pack_sampler PVR_PER_ARCH(g5p_pack_sampler) # define pvr_arch_g5p_combine_sampler PVR_PER_ARCH(g5p_combine_sampler) From af708e0c93e7f3d6e60a729ebfc45204d0f57b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:45 +0100 Subject: [PATCH 067/109] g9p: array fetches and explicit array and cube samples in A9 compute texelFetch and textureLod on 2D arrays (integer layers) and textureLod on cubes (with its face iterator) compile in A9 compute, with Apple's witnessed operand forms and the dispatch uploads they need. Assisted-by: LLM --- src/imagination/g9p/g9p_a9.h | 5 +- src/imagination/g9p/g9p_a9_compute.h | 30 +++++++- src/imagination/g9p/g9p_a9_tex.h | 74 ++++++++++++++++--- src/imagination/vulkan/pvr_arch_g5p_compute.c | 5 ++ .../vulkan/pvr_g5p_compute_state.h | 2 +- 5 files changed, 101 insertions(+), 15 deletions(-) diff --git a/src/imagination/g9p/g9p_a9.h b/src/imagination/g9p/g9p_a9.h index ea7cc18b5d1..bd158eb4a31 100644 --- a/src/imagination/g9p/g9p_a9.h +++ b/src/imagination/g9p/g9p_a9.h @@ -2,13 +2,14 @@ #ifndef G9P_A9_H #define G9P_A9_H /* Compute texture words the dispatch puts in the common store (g9p_a9_tex.h) */ -#define G9P_A9_MAX_UPLOADS 7 -enum g9p_a9_upload_kind { G9P_A9_UP_IMAGE = 1, G9P_A9_UP_SAMPLER, G9P_A9_UP_FETCH_SAMPLER, G9P_A9_UP_SIZE }; +#define G9P_A9_MAX_UPLOADS 10 +enum g9p_a9_upload_kind { G9P_A9_UP_IMAGE = 1, G9P_A9_UP_SAMPLER, G9P_A9_UP_FETCH_SAMPLER, G9P_A9_UP_SIZE, G9P_A9_UP_LOD }; struct g9p_a9_upload { unsigned char kind, set, binding, element; unsigned char combined; /* a sampler in a combined image sampler: its words at +32 */ unsigned char level; /* a size block's level */ unsigned char common, dwords; + uint32_t value; /* literal common-store LOD */ }; /* g9p_shader::a9_bank1_regs: the kernel record's register field allocates bank 1 (g9p_cdm.h) */ #define G9P_A9_HAS_BANK1_REGS 1 diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index 92882d79adc..bfc85f465ad 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -140,6 +140,23 @@ a9_pack_image_store(nir_builder *b, nir_intrinsic_instr *in, void *data) return true; } +/* The texture unit takes an integer array layer even for filtered samples. */ +static bool +a9_lower_array_layer(nir_builder *b, nir_tex_instr *tex, void *data) +{ + if (tex->op != nir_texop_txl || tex->sampler_dim != GLSL_SAMPLER_DIM_2D || !tex->is_array) + return false; + int ci = nir_tex_instr_src_index(tex, nir_tex_src_coord); + if (ci < 0) return false; + b->cursor = nir_before_instr(&tex->instr); + nir_def *co = tex->src[ci].src.ssa; + nir_def *layer = nir_f2u32(b, nir_fround_even(b, nir_fmax(b, nir_channel(b, co, 2), + nir_imm_float(b, 0)))); + nir_src_rewrite(&tex->src[ci].src, + nir_vec3(b, nir_channel(b, co, 0), nir_channel(b, co, 1), layer)); + return true; +} + bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_ctx) { nir->options = &nir_options_a9; @@ -166,14 +183,23 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c return false; } out->a9_local_dwords = 4 + DIV_ROUND_UP(nir->info.shared_size, 4); + NIR_PASS(_, nir, nir_shader_tex_pass, a9_lower_array_layer, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_lower_compute_system_values, &csv); NIR_PASS(_, nir, nir_lower_undef_to_zero, NULL); if (!compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx)) return false; /* the fixed common-store places of texture words (g9p_a9_tex.h) */ - bool sizes = false; - for (unsigned i = 0; i < out->a9_num_uploads; i++) + bool sizes = false, lods = false; + for (unsigned i = 0; i < out->a9_num_uploads; i++) { sizes |= out->a9_uploads[i].kind == G9P_A9_UP_SIZE; + lods |= out->a9_uploads[i].kind == G9P_A9_UP_LOD; + } + if (lods && out->num_buffers > 13) { + if (!out->a9_refusal[0]) + snprintf(out->a9_refusal, sizeof(out->a9_refusal), + "more than 13 buffers with shared texture levels (common 28-30)"); + return false; + } if (out->a9_num_uploads && out->num_buffers > 15) { if (!out->a9_refusal[0]) snprintf(out->a9_refusal, sizeof(out->a9_refusal), diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index 398c51a9538..af5befb9f07 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: MIT */ #include "../g5p/g5p_usc.h" +#include "util/half_float.h" /* * A9 compute samples (WGSL textureSampleLevel/Grad/Load/Dimensions in a * compute shader). Apple's A9 kernels (sample(s, uv, level(l)), @@ -43,6 +44,7 @@ a9_tex_refuse(struct g9p_ctx *ctx, const char *why) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) if (!ctx->out->a9_refusal[0]) + if (!ctx->out->a9_refusal[0]) snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); } @@ -83,6 +85,36 @@ a9_tex_upload(struct g9p_ctx *ctx, unsigned kind, unsigned set, unsigned binding return slots[used]; } +/* ra_f/sla_f/slc_f use common-store integer/f16 levels, not the + * temporary continuation of the older 2D samples. Three literal slots + * precede the storage-image level at 31; buffer pointers must end at 27. */ +static int +a9_tex_shared_lod(struct g9p_ctx *ctx, nir_tex_instr *tex, bool read) +{ + int li = nir_tex_instr_src_index(tex, nir_tex_src_lod); + if (li >= 0 && !nir_src_is_const(tex->src[li].src)) { + a9_tex_refuse(ctx, "dynamic LOD on a compute 2D array or cube (no A9 witness yet)"); + return -1; + } + uint32_t value = li < 0 ? 0 : read ? nir_src_as_uint(tex->src[li].src) : + _mesa_float_to_half(nir_src_as_float(tex->src[li].src)); + unsigned used = 0; + for (unsigned i = 0; i < ctx->out->a9_num_uploads; i++) { + const struct g9p_a9_upload *u = &ctx->out->a9_uploads[i]; + if (u->kind != G9P_A9_UP_LOD) continue; + if (u->value == value) return u->common; + used++; + } + if (used == 3 || ctx->out->a9_num_uploads == G9P_A9_MAX_UPLOADS) { + a9_tex_refuse(ctx, "more than 3 shared texture levels in a compute shader"); + return -1; + } + ctx->out->a9_uploads[ctx->out->a9_num_uploads++] = (struct g9p_a9_upload) { + .kind = G9P_A9_UP_LOD, .common = 28 + used, .dwords = 1, .value = value, + }; + return 28 + used; +} + static bool a9_tex_var(nir_tex_instr *tex, nir_tex_src_type type, unsigned *set, unsigned *binding, unsigned *element) { @@ -185,9 +217,18 @@ a9_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) a9_tex_refuse(ctx, "texture operation not supported in compute shaders yet"); return false; } - if (tex->sampler_dim != GLSL_SAMPLER_DIM_2D || tex->is_array || tex->is_shadow) { - a9_tex_refuse(ctx, tex->is_shadow ? "a depth-compare sample in a compute shader" : - "only 2D (not array, cube or 3D) samples in compute shaders so far"); + const bool array = tex->sampler_dim == GLSL_SAMPLER_DIM_2D && tex->is_array; + const bool cube = tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE; + const bool layered = array || cube; + if (tex->is_shadow || + (tex->sampler_dim != GLSL_SAMPLER_DIM_2D && !cube) || + (cube && (tex->is_array || tex->op != nir_texop_txl)) || (array && grad)) { + char why[160]; + const char *dim = cube ? "cube" : tex->sampler_dim == GLSL_SAMPLER_DIM_3D ? "3D" : + array ? "2D array" : tex->sampler_dim == GLSL_SAMPLER_DIM_2D ? "2D" : "other"; + snprintf(why, sizeof(why), "compute %s on %s is_array=%u is_shadow=%u unsupported", + (read ? "txf" : grad ? "txd" : "txl"), dim, tex->is_array, tex->is_shadow); + a9_tex_refuse(ctx, why); return false; } if (nir_alu_type_get_base_type(tex->dest_type) != nir_type_float) { @@ -222,17 +263,21 @@ a9_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) return false; /* coordinates: a fresh block, consumed by the sample */ - const int block = alloc_range(ctx, 2); - if (block < 0 || block > 30) + const unsigned coords = layered ? 3 : 2; + const int block = alloc_range(ctx, coords); + if (block < 0 || block + coords > 32) return false; - ctx->temps |= BITFIELD64_RANGE(block, 2); - for (unsigned c = 0; c < 2; c++) + ctx->temps |= BITFIELD64_RANGE(block, coords); + for (unsigned c = 0; c < coords; c++) if (!a9_tex_copy(ctx, block + c, tex->src[ci].src.ssa, c)) return false; /* level (f16 for samples, u32 for reads) or gradients */ int lod; - if (grad) { + if (layered) { + lod = a9_tex_shared_lod(ctx, tex, read); + if (lod < 0) return false; + } else if (grad) { const int dxi = nir_tex_instr_src_index(tex, nir_tex_src_ddx); const int dyi = nir_tex_instr_src_index(tex, nir_tex_src_ddy); lod = alloc_range(ctx, 4); @@ -262,6 +307,13 @@ a9_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) g9p_f16_of(ctx->c, lod, r); } } + if (cube) { + /* slc_f's cube coordinates (g9p_cube_coords()): bank0 r0..2 directions -> bank1 coords. + * Bank0 homes start at r32, so this scratch cannot overwrite them. */ + for (unsigned c = 0; c < 3; c++) + g9p_mov_to_high(ctx->c, c, block + c); + g9p_cube_coords(ctx->c, 2 * block, 0); + } release(ctx, &tex->instr); const int dst = alloc_range(ctx, 4); if (dst < 0 || dst > 28) @@ -269,9 +321,11 @@ a9_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) struct g5p_tex_ext x = { .dst_half = 2 * dst, .image_qword = image / 2, .sampler_qword = sampler / 2, - .coord_dword = block, .dim = G5P_TEX_2D, .mask = 0xf, .read = read, + .coord_dword = block, .dim = cube ? G5P_TEX_CUBE : array ? G5P_TEX_2D_ARRAY : G5P_TEX_2D, .mask = 0xf, .read = read, .lod = grad ? G5P_TEX_LOD_GRAD : G5P_TEX_LOD_LEVEL, .lod_dword = lod, - .tuple = grad ? G5P_TEX_TUPLE_COUPLED_GRAD : G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL, + .lod_shared = layered, + .tuple = layered ? G5P_TEX_TUPLE_COUPLED_SHARED_LEVEL : + grad ? G5P_TEX_TUPLE_COUPLED_GRAD : G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL, }; uint8_t raw[G5P_USC_MAX]; const unsigned n = g5p_usc_sample_ext(raw, &x); diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 672d36226c5..de6f2e62096 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -162,6 +162,7 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( state->a9_uploads[i].level = u->level; state->a9_uploads[i].common = u->common; state->a9_uploads[i].dwords = u->dwords; + state->a9_uploads[i].value = u->value; } state->uses_workgroup_id = shader.uses_workgroup_id; memcpy(state->local_size, shader.local_size, sizeof(state->local_size)); @@ -266,6 +267,10 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, for(unsigned i=0;ia9_uploads[0]) *u=&state->a9_uploads[i]; uint32_t *out=&uniforms[tex0+8*i]; + if (u->kind == G9P_A9_UP_LOD) { + out[0] = u->value; + continue; + } if(u->kind==G9P_A9_UP_FETCH_SAMPLER) { const uint64_t policy=pvr_g5p_fetch_sampler_word(); memcpy(out,&policy,sizeof(policy)); diff --git a/src/imagination/vulkan/pvr_g5p_compute_state.h b/src/imagination/vulkan/pvr_g5p_compute_state.h index 671de5f87cd..5ee4be651cc 100644 --- a/src/imagination/vulkan/pvr_g5p_compute_state.h +++ b/src/imagination/vulkan/pvr_g5p_compute_state.h @@ -2,6 +2,6 @@ #ifndef PVR_G5P_COMPUTE_STATE_H #define PVR_G5P_COMPUTE_STATE_H #define PVR_G5P_COMPUTE_STATE_FIELDS unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ - unsigned a9_num_uploads; struct { unsigned char kind, set, binding, element, combined, level, common, dwords; } a9_uploads[7]; + unsigned a9_num_uploads; struct { unsigned char kind, set, binding, element, combined, level, common, dwords; uint32_t value; } a9_uploads[10]; #define PVR_G5P_NATIVE_STORAGE(d) PVR_HAS_FEATURE((d), apple_g5p) #endif From afdaf50fdfa3418851056ae9dcef61b7c4f59bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:04:45 +0100 Subject: [PATCH 068/109] g9p: keep the final compute refusal reason across allocator retries The A9 compute compiler retries register allocation, and a later attempt's message replaced the reason that the final attempt failed for. The last attempt's reason now survives, with a thread-local fallback for the last message. Shared-memory and loop refusals are named, and duplicate shared-allocation accounting is gone. Assisted-by: LLM --- src/imagination/g9p/g9p_a9.h | 1 + src/imagination/g9p/g9p_a9_compute.h | 15 ++- src/imagination/g9p/g9p_a9_tex.h | 20 +--- src/imagination/g9p/g9p_compile.c | 104 ++++++++++++++---- src/imagination/vulkan/pvr_arch_g5p_compute.c | 2 +- 5 files changed, 91 insertions(+), 51 deletions(-) diff --git a/src/imagination/g9p/g9p_a9.h b/src/imagination/g9p/g9p_a9.h index bd158eb4a31..d7b27d97d04 100644 --- a/src/imagination/g9p/g9p_a9.h +++ b/src/imagination/g9p/g9p_a9.h @@ -30,6 +30,7 @@ struct g9p_a9_upload { if ((ctx)->out->a9 && (ctx)->stage == MESA_SHADER_COMPUTE) \ (ctx)->out->a9_bodyless = !(n); \ } while (0) +const char *g9p_a9_last_refusal(void); struct g9p_shader; bool g9p_a9_compile_compute(nir_shader *, struct g9p_shader *, void *); #endif diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index bfc85f465ad..38ecb33a0b1 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -165,22 +165,21 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c }; memset(out, 0, sizeof(*out)); out->a9 = true; + a9_last_refusal[0] = 0; lower_front(nir); NIR_PASS(_, nir, nir_shader_intrinsics_pass, a9_lower_subgroups_size1, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_shader_intrinsics_pass, a9_pack_image_store, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_lower_alu); - NIR_PASS(_, nir, nir_lower_vars_to_explicit_types, nir_var_mem_shared, glsl_get_natural_size_align_bytes); - NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_shared, nir_address_format_32bit_offset); + /* lower_front() already allocated and lowered shared variables. Doing + * it again double-counts the allocation after addresses became explicit. */ const nir_lower_mem_access_bit_sizes_options shared_opts = { .modes = nir_var_mem_shared, .callback = a9_shared_size_align, }; NIR_PASS(_, nir, nir_lower_mem_access_bit_sizes, &shared_opts); if (nir->info.shared_size > 16384) { - if (!out->a9_refusal[0]) - snprintf(out->a9_refusal, sizeof(out->a9_refusal), "shared allocation exceeds 16384 bytes"); - return false; + return g9p_fail(out, "shared memory %u bytes, above the A9's 16384", nir->info.shared_size); } out->a9_local_dwords = 4 + DIV_ROUND_UP(nir->info.shared_size, 4); NIR_PASS(_, nir, nir_shader_tex_pass, a9_lower_array_layer, nir_metadata_control_flow, NULL); @@ -198,20 +197,20 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c if (!out->a9_refusal[0]) snprintf(out->a9_refusal, sizeof(out->a9_refusal), "more than 13 buffers with shared texture levels (common 28-30)"); - return false; + return backend_failure(out, &(struct g9p_ctx){0}); } if (out->a9_num_uploads && out->num_buffers > 15) { if (!out->a9_refusal[0]) snprintf(out->a9_refusal, sizeof(out->a9_refusal), "%u buffers with sampled textures (buffer words reach the texture words at common 32)", out->num_buffers); - return false; + return backend_failure(out, &(struct g9p_ctx){0}); } if (sizes && out->num_textures) { if (!out->a9_refusal[0]) snprintf(out->a9_refusal, sizeof(out->a9_refusal), "a texture size query together with a storage image (both at common 40-47)"); - return false; + return backend_failure(out, &(struct g9p_ctx){0}); } return true; } diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index af5befb9f07..b5261db812e 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -27,25 +27,7 @@ static const uint8_t a9_tex_size_dw[] = { 40, 44 }; static void a9_tex_refuse(struct g9p_ctx *ctx, const char *why) { - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - if (!ctx->out->a9_refusal[0]) - snprintf(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal), "%s", why); + g9p_fail(ctx->out, "%s", why); } /* The common dword of an upload, added if new; -1 when its slots are full */ diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index 97ab0136447..5ea5442f678 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -19,6 +19,8 @@ #include #include #include "c11/threads.h" +#include +#include "util/log.h" #include #include @@ -46,6 +48,43 @@ DEBUG_GET_ONCE_BOOL_OPTION(g9p_pressure, "G9P_PRESSURE", false) +/* Pipeline compilation may run on several threads. Keep the fallback for + * the current compile, never a different pipeline's last diagnostic. */ +static thread_local char a9_last_refusal[192]; +const char *g9p_a9_last_refusal(void) +{ + return a9_last_refusal[0] ? a9_last_refusal : "A9 backend failed without a diagnostic"; +} + +static bool +g9p_fail(struct g9p_shader *out, const char *fmt, ...) +{ + char why[192]; + va_list ap; + va_start(ap, fmt); + vsnprintf(why, sizeof(why), fmt, ap); + va_end(ap); + if (out->a9) { + if (!out->a9_refusal[0]) + snprintf(out->a9_refusal, sizeof(out->a9_refusal), "%s", why); + snprintf(a9_last_refusal, sizeof(a9_last_refusal), "%s", out->a9_refusal); + } + if (out->a9) { + mesa_logw("G5P refuses: %s", why); + } else { + /* Preserve main's non-A9 diagnostics. */ + if (!strcmp(why, "shader has no entry point")) + return false; + if (!strncmp(why, "too many loops", 14)) + snprintf(why, sizeof(why), "too many loops"); + char *suffix = strstr(why, " (no parking victim)"); + if (!suffix) suffix = strstr(why, " (bank-0 homes exhausted)"); + if (suffix) *suffix = 0; + fprintf(stderr, "g9p: %s\n", why); + } + return false; +} + /* r0-r29 for values, the mask stack after them (G9P_REGS: debug override) */ static unsigned g9p_num_regs(void) @@ -318,6 +357,23 @@ struct g9p_ctx { unsigned num_exit_jumps, max_exit_jumps; }; +/* Every compile_back failure reaches this helper, including control flow + * failures outside emit_instr(). An instruction's precise reason wins. */ +static bool +backend_failure(struct g9p_shader *out, const struct g9p_ctx *ctx) +{ + if (out->a9) { + if (out->a9_refusal[0]) { + char why[sizeof(out->a9_refusal)]; + memcpy(why, out->a9_refusal, sizeof(why)); + return g9p_fail(out, "%s", why); + } + return ctx->oom ? g9p_fail(out, "out of registers at instruction %u", ctx->oom_ip) : + g9p_fail(out, "backend control flow or register allocation failed at instruction %u", ctx->ip); + } + return false; +} + static unsigned key(const nir_def *def, unsigned comp) { @@ -3764,6 +3820,10 @@ emit_cf_list(struct g9p_ctx *ctx, struct exec_list *list) static bool generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) { + if (ctx->out->a9) { + ctx->out->a9_refusal[0] = a9_last_refusal[0] = 0; + ctx->out->a9_num_uploads = 0; + } uint16_t *body; unsigned body_n; @@ -4922,7 +4982,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, impl = nir_shader_get_entrypoint(nir); if (!impl) - return false; + return g9p_fail(out, "shader has no entry point"); nir_index_ssa_defs(impl); ctx.tex_total = count_textures(impl); if (getenv("G9P_DEBUG")) @@ -4963,8 +5023,8 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.ip = 0; walk_liveness(&ctx, &impl->body, false); if (ctx.loop_index > (out->a9 ? G9P_MAX_LOOPS : 64)) { - fprintf(stderr, "g9p: too many loops\n"); - return false; + g9p_fail(out, "too many loops (%u, maximum %u)", ctx.loop_index, out->a9 ? G9P_MAX_LOOPS : 64); + return backend_failure(out, &ctx); } ctx.ip = 0; ctx.loop_index = 0; @@ -5047,14 +5107,13 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, /* A9 compute parks values above its bank-1 limit in bank 0. */ if (!ctx.oom || (stage == MESA_SHADER_COMPUTE && !(out->a9)) || num_homed >= 1024 || g9p_no_homes) - return false; + return backend_failure(out, &ctx); stuck = ctx.oom_ip == last_oom_ip ? stuck + 1 : 0; last_oom_ip = ctx.oom_ip; /* each retry parks one more value; 40 live scalars need more than 16 (drawbench fs 2) */ if (stuck >= 64) { - fprintf(stderr, "g9p: out of registers at instruction %u (parking does not help)\n", - ctx.oom_ip); - return false; + g9p_fail(out, "out of registers at instruction %u (parking does not help)", ctx.oom_ip); + return backend_failure(out, &ctx); } for (unsigned d = 0; d < ctx.num_decls; d++) { const unsigned k = key(ctx.decls[d], 0); @@ -5089,8 +5148,8 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.reg[best - 1] >= 0) victim = best - 1; if (victim < 0) { - fprintf(stderr, "g9p: out of registers at instruction %u\n", ctx.oom_ip); - return false; + g9p_fail(out, "out of registers at instruction %u (no parking victim)", ctx.oom_ip); + return backend_failure(out, &ctx); } { const unsigned start = victim_decl ? ctx.reg_first[victim / 4] : ctx.def_ip[victim / 4]; @@ -5114,8 +5173,8 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, home = 32 + h; } if (home < 0) { - fprintf(stderr, "g9p: out of registers above r31 at instruction %u\n", ctx.oom_ip); - return false; + g9p_fail(out, "out of registers above r31 at instruction %u (bank-0 homes exhausted)", ctx.oom_ip); + return backend_failure(out, &ctx); } homed[num_homed++] = (__typeof__(*homed)){ start, end, (int8_t)home }; ctx.home[victim] = home; @@ -5124,9 +5183,8 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, } st = ctx.max_reg + 1; if (ctx.c->high_bank0 && ctx.value_cap && st + 1 > ctx.value_cap + 1) { - if (!out->a9_refusal[0]) - snprintf(out->a9_refusal, sizeof(out->a9_refusal), "bank-1 cap exceeded"); - return false; + g9p_fail(out, "bank-1 cap exceeded"); + return backend_failure(out, &ctx); } /* * Vertex shaders: the mask stack below r16, selects use it too (dEQP vertex @@ -5144,22 +5202,22 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, for (unsigned k = 0; k < ctx.out->num_buffers; k++) changed |= ctx.out->buffers[k].dwords != ctx.prev_dwords[k]; if (changed && !generate(&ctx, impl, 15)) - return false; + return backend_failure(out, &ctx); st = MAX2(ctx.max_reg, 15); } else if (stage == MESA_SHADER_VERTEX && st > 15) { ctx.reserve_st_low = true; if (!generate(&ctx, impl, 15)) { /* without r15, address work can run out of low registers */ if (!ctx.oom || ctx.reserve_low) - return false; + return backend_failure(out, &ctx); ctx.reserve_low = true; ctx.oom = false; if (!generate(&ctx, impl, 15)) - return false; + return backend_failure(out, &ctx); } st = MAX2(ctx.max_reg, 15); } else if (!generate(&ctx, impl, st)) - return false; + return backend_failure(out, &ctx); /* * Buffers over the uniform operands' 512 dwords: only the dwords it reads, in as few runs @@ -5209,8 +5267,8 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, runs++; } if (span > 512) { - fprintf(stderr, "g9p: fragment shader reads %u dwords of buffers in %u runs\n", span, runs); - return false; + g9p_fail(out, "fragment shader reads %u dwords of buffers in %u runs", span, runs); + return backend_failure(out, &ctx); } unsigned next = g9p_fs_buf_base(out->num_textures); @@ -5239,7 +5297,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, } } if (!generate(&ctx, impl, st)) - return false; + return backend_failure(out, &ctx); } } @@ -5255,8 +5313,8 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, out->code = ctx.c->w; out->code_size = 2 * ctx.c->n; if (stage != MESA_SHADER_COMPUTE && st + 1 > 32) { - fprintf(stderr, "g9p: %u registers, a graphics task has 32\n", out->num_regs); - return false; + g9p_fail(out, "%u registers, a graphics task has 32", out->num_regs); + return backend_failure(out, &ctx); } if (getenv("G9P_DEBUG")) { fprintf(stderr, "g9p: %u registers, %u bytes of code\n", out->num_regs, diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index de6f2e62096..d34624153b3 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -58,7 +58,7 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( /* Shader capture and diagnostic stubs (pvr_g5p_shader_dump.h). */ const bool stub = pvr_g5p_capture_stub(); char reason[sizeof(shader.a9_refusal)]; - snprintf(reason, sizeof(reason), "%s", shader.a9_refusal[0] ? shader.a9_refusal : "no reason recorded"); + snprintf(reason, sizeof(reason), "%s", shader.a9_refusal[0] ? shader.a9_refusal : g9p_a9_last_refusal()); pvr_g5p_shader_dump(&create_info->stage, create_info->layout, "comp", NULL, 0, stub ? "STUB" : "REFUSE", reason); if (!stub) { From 5c8891c08e72215f06c1b5e292d8c624762eacb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:05:00 +0100 Subject: [PATCH 069/109] pvr/g5p: identify compute dispatches with PVR_DEBUG=compute_log Assisted-by: LLM --- src/imagination/common/pvr_debug.c | 1 + src/imagination/common/pvr_debug.h | 2 + src/imagination/vulkan/pvr_arch_cmd_buffer.c | 10 +- src/imagination/vulkan/pvr_arch_g5p_compute.c | 116 +++++++++++++++++- src/imagination/vulkan/pvr_arch_g5p_tex.c | 11 +- src/imagination/vulkan/pvr_cmd_buffer.h | 2 + src/imagination/vulkan/pvr_g5p_compute.h | 10 +- .../vulkan/pvr_g5p_compute_hooks.h | 2 +- .../vulkan/pvr_g5p_compute_state.h | 2 +- src/imagination/vulkan/pvr_g5p_shader_dump.c | 25 ++++ src/imagination/vulkan/pvr_g5p_shader_dump.h | 2 + .../winsys/powervr/pvr_drm_job_compute.c | 26 +++- src/imagination/vulkan/winsys/pvr_winsys.h | 4 + 13 files changed, 201 insertions(+), 12 deletions(-) diff --git a/src/imagination/common/pvr_debug.c b/src/imagination/common/pvr_debug.c index 9b6d8d1fd19..ecdb55e777f 100644 --- a/src/imagination/common/pvr_debug.c +++ b/src/imagination/common/pvr_debug.c @@ -41,6 +41,7 @@ static const struct debug_named_value debug_control[] = { { "shader_dump", PVR_DEBUG_SHADER_DUMP, "Capture shader compilation inputs and results." }, { "capture_stub", PVR_DEBUG_CAPTURE_STUB, "Replace refused shaders with diagnostic stubs; renders incorrect pixels." }, { "errors", PVR_DEBUG_ERRORS, "Log every Vulkan error the driver returns, with its origin." }, + { "compute_log", PVR_DEBUG_COMPUTE_LOG, "Log compute pipelines and dispatches." }, DEBUG_NAMED_VALUE_END }; /* clang-format on */ diff --git a/src/imagination/common/pvr_debug.h b/src/imagination/common/pvr_debug.h index 98b820fc833..578c3ef7b6b 100644 --- a/src/imagination/common/pvr_debug.h +++ b/src/imagination/common/pvr_debug.h @@ -45,6 +45,8 @@ extern uint32_t PVR_DEBUG; #define PVR_DEBUG_ERRORS BITFIELD_BIT(8) +#define PVR_DEBUG_COMPUTE_LOG BITFIELD_BIT(9) + void pvr_process_debug_variable(void); #endif /* PVR_DEBUG_H */ diff --git a/src/imagination/vulkan/pvr_arch_cmd_buffer.c b/src/imagination/vulkan/pvr_arch_cmd_buffer.c index 01f00920c45..ab350dff23c 100644 --- a/src/imagination/vulkan/pvr_arch_cmd_buffer.c +++ b/src/imagination/vulkan/pvr_arch_cmd_buffer.c @@ -227,6 +227,11 @@ static void pvr_cmd_buffer_free_sub_cmd(struct pvr_cmd_buffer *cmd_buffer, case PVR_SUB_CMD_TYPE_COMPUTE: case PVR_SUB_CMD_TYPE_QUERY: vk_free(&cmd_buffer->vk.pool->alloc, sub_cmd->compute.g9p_query); + for (struct pvr_g5p_dispatch_log *l = sub_cmd->compute.g5p_log_first, *next; l; l = next) { + next = l->next; + free(l->line); + free(l); + } pvr_csb_finish(&sub_cmd->compute.control_stream); break; @@ -7147,6 +7152,7 @@ static VkResult pvr_cmd_upload_push_consts(struct pvr_cmd_buffer *cmd_buffer, static void pvr_cmd_dispatch( struct pvr_cmd_buffer *const cmd_buffer, const pvr_dev_addr_t indirect_addr, + const uint64_t indirect_offset, const uint32_t base_group[static const PVR_WORKGROUP_DIMENSIONS], const uint32_t workgroup_size[static const PVR_WORKGROUP_DIMENSIONS]) { @@ -7224,7 +7230,7 @@ void PVR_PER_ARCH(CmdDispatchBase)(VkCommandBuffer commandBuffer, return; pvr_cmd_dispatch(cmd_buffer, - PVR_DEV_ADDR_INVALID, + PVR_DEV_ADDR_INVALID, 0, (uint32_t[]){ baseGroupX, baseGroupY, baseGroupZ }, (uint32_t[]){ groupCountX, groupCountY, groupCountZ }); } @@ -7239,7 +7245,7 @@ void PVR_PER_ARCH(CmdDispatchIndirect)(VkCommandBuffer commandBuffer, PVR_CHECK_COMMAND_BUFFER_BUILDING_STATE(cmd_buffer); pvr_cmd_dispatch(cmd_buffer, - PVR_DEV_ADDR_OFFSET(buffer->dev_addr, offset), + PVR_DEV_ADDR_OFFSET(buffer->dev_addr, offset), offset, (uint32_t[]){ 0, 0, 0 }, (uint32_t[]){ 1, 1, 1 }); } diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index d34624153b3..a7483427ceb 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -18,6 +18,8 @@ #include "pvr_csb.h" #include "pds/pvr_rogue_pds_defs.h" #include "pvr_descriptor_set.h" +#include "pvr_image.h" +#include "vk_enum_to_str.h" #include "pvr_device.h" #include "pvr_physical_device.h" #include "pvr_pipeline.h" @@ -38,6 +40,8 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( struct pvr_compute_pipeline *pipeline) { struct pvr_g9p_compute_state *state = &pipeline->g9p; + state->a9_compute_log = pvr_g5p_compute_log(); + pvr_g5p_shader_id(&create_info->stage, state->a9_pipeline_id); void *mem_ctx = ralloc_context(NULL); struct g9p_shader shader; nir_shader *nir; @@ -54,8 +58,11 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( goto out; } - if (!g9p_a9_compile_compute(nir, &shader, mem_ctx)) { - /* Shader capture and diagnostic stubs (pvr_g5p_shader_dump.h). */ + const bool compiled = g9p_a9_compile_compute(nir, &shader, mem_ctx); + for (unsigned i = 0; i < 3; i++) state->a9_log_local[i] = nir->info.workgroup_size[i]; + state->a9_log_shared = nir->info.shared_size; + if (!compiled) { + /* PVR_DEBUG=shader_dump / PVR_DEBUG=capture_stub (pvr_g5p_shader_dump.h) */ const bool stub = pvr_g5p_capture_stub(); char reason[sizeof(shader.a9_refusal)]; snprintf(reason, sizeof(reason), "%s", shader.a9_refusal[0] ? shader.a9_refusal : g9p_a9_last_refusal()); @@ -67,6 +74,7 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( "G5P compute: shader not supported by the backend yet"); goto out; } + state->a9_capture_stub = true; mesa_logw("G5P capture stub: compute shader refused (%s): empty kernel, dispatches omitted", reason); VkPipelineShaderStageCreateInfo stub_info; VkShaderModuleCreateInfo stub_module; @@ -75,7 +83,10 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( g9p_spirv_options(), g9p_nir_options(), mem_ctx, &nir); if (result != VK_SUCCESS) goto out; - if (!g9p_a9_compile_compute(nir, &shader, mem_ctx)) { + const bool compiled = g9p_a9_compile_compute(nir, &shader, mem_ctx); + for (unsigned i = 0; i < 3; i++) state->a9_log_local[i] = nir->info.workgroup_size[i]; + state->a9_log_shared = nir->info.shared_size; + if (!compiled) { result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, "G5P compute: the capture stub was refused"); goto out; } @@ -181,6 +192,97 @@ emit_record(struct pvr_csb *csb, const struct g5p_cdm_record *rec) memcpy(w, rec->w, sizeof(rec->w)); } +/* Capture complete bound compute resources, including a stub's original + * descriptors. No descriptor or pipeline pointers survive command recording. */ +static VkResult +g5p_record_dispatch(struct pvr_cmd_buffer *cmd, struct pvr_sub_cmd_compute *sub, + uint64_t indirect_va, uint64_t indirect_offset, const uint32_t groups[3]) +{ + const struct pvr_g9p_compute_state *s = &cmd->state.compute_pipeline->g9p; + struct pvr_g5p_dispatch_log *l = calloc(1, sizeof(*l)); + if (!l) return vk_error(cmd, VK_ERROR_OUT_OF_HOST_MEMORY); + size_t size; + FILE *f = open_memstream(&l->line, &size); + if (!f) { free(l); return vk_error(cmd, VK_ERROR_OUT_OF_HOST_MEMORY); } + memcpy(l->pipeline, s->a9_pipeline_id, sizeof(l->pipeline)); + l->stub = s->a9_capture_stub; + fprintf(f, "pipe=%s stub=%u local=%ux%ux%u shared=%u ", l->pipeline, l->stub, + s->a9_log_local[0], s->a9_log_local[1], s->a9_log_local[2], s->a9_log_shared); + if (indirect_va) + fprintf(f, "groups=indirect buffer_va=%#" PRIx64 " offset=%" PRIu64 " args_va=%#" PRIx64, + indirect_va - indirect_offset, indirect_offset, indirect_va); + else + fprintf(f, "groups=%ux%ux%u", groups[0], groups[1], groups[2]); + fprintf(f, " res=["); + const struct pvr_descriptor_state *desc = &cmd->state.compute_desc_state; + const char *sep = ""; + for (unsigned si = 0; si < PVR_MAX_DESCRIPTOR_SETS; si++) { + const struct pvr_descriptor_set *set = desc->sets[si]; + if (!set || !set->layout || !set->mapping) continue; + for (unsigned bi = 0; bi < set->layout->binding_count; bi++) { + const struct pvr_descriptor_set_layout_binding *b = &set->layout->bindings[bi]; + if (!(b->stage_flags & VK_SHADER_STAGE_COMPUTE_BIT)) continue; + for (unsigned e = 0; e < b->descriptor_count; e++) { + const uint8_t *d = (uint8_t *)set->mapping + b->offset + e * b->stride; + switch (b->type) { + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: { + struct pvr_buffer_descriptor buf; + if (b->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC || + b->type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) + buf = set->dynamic_buffers[b->dynamic_buffer_idx + e]; + else + memcpy(&buf, d, sizeof(buf)); + fprintf(f, "%sbuffer %u/%u[%u] VA=%#" PRIx64 " range=%u", sep, si, bi, e, buf.addr + buf.offset, buf.size); + sep = "; "; + break; + } + case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: + case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: { + const struct pvr_image_view *v; + uintptr_t ptr; + memcpy(&ptr, d + 24, sizeof(ptr)); + v = (const struct pvr_image_view *)ptr; + if (!v) { + fprintf(f, "%simage %u/%u[%u] unbound", sep, si, bi, e); + } else { + const struct pvr_image *im = pvr_image_view_get_image(v); + uint64_t q[3]; + memcpy(q, d, sizeof(q)); + const bool linear = !(q[0] & PVR_G5P_TEX_TWIDDLED); + uint64_t va = (q[1] & ((UINT64_C(1) << 39) - 1) & ~UINT64_C(31)) << 1; + if (linear && (q[1] >> 63)) va |= ((q[2] >> 58) & 3) << 4; + fprintf(f, "%s%s %u/%u[%u] VA=%#" PRIx64 " bytes=%" PRIu64 + " %ux%ux%u format=%s view=%s", sep, + b->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE ? "storage" : "sampled", + si, bi, e, va, im->total_size, + u_minify(im->vk.extent.width, v->vk.base_mip_level), + u_minify(im->vk.extent.height, v->vk.base_mip_level), v->vk.layer_count, + vk_Format_to_str(v->vk.format), vk_ImageViewType_to_str(v->vk.view_type)); + if (b->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) + fprintf(f, " dim=%#x", (q[0] & 7) == PVR_G5P_TEX_LAYOUT_2D_ARRAY ? 0xf7 : 0xf5); + } + sep = "; "; + break; + } + default: break; + } + } + } + } + fprintf(f, "]"); + const bool failed = ferror(f); + if (fclose(f) || failed) { free(l->line); free(l); return vk_error(cmd, VK_ERROR_OUT_OF_HOST_MEMORY); } + if (sub->g5p_log_last) sub->g5p_log_last->next = l; + else sub->g5p_log_first = l; + sub->g5p_log_last = l; + sub->g5p_log_count++; + return VK_SUCCESS; +} + /* * One dispatch: a DMA task loading the buffer addresses (from the bound * descriptor sets) and the thread count into the common store, then the @@ -190,7 +292,7 @@ emit_record(struct pvr_csb *csb, const struct g5p_cdm_record *rec) VkResult PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd_compute *sub_cmd, - uint64_t indirect_va, const uint32_t base_group[static 3], + uint64_t indirect_va, uint64_t indirect_offset, const uint32_t base_group[static 3], const uint32_t group_count[static 3]) { struct pvr_device *device = cmd_buffer->device; @@ -211,6 +313,11 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, unsigned bind_dwords, ctrl0, nc = 0; VkResult result; + if (state->a9_compute_log) { + result = g5p_record_dispatch(cmd_buffer, sub_cmd, indirect_va, indirect_offset, group_count); + if (result != VK_SUCCESS) return result; + } + if (base_group[0] || base_group[1] || base_group[2]) return vk_errorf(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY, "G5P: base dispatch not implemented yet"); if (indirect_va && ((indirect_va & 3) || indirect_va >> 40)) @@ -695,6 +802,7 @@ PVR_PER_ARCH(g5p_compute_job_submit)(struct pvr_compute_ctx *ctx, if (cs->pvr_bo != list_first_entry(&cs->pvr_bo_list, struct pvr_bo, link)) return vk_errorf(d, VK_ERROR_OUT_OF_HOST_MEMORY, "G5P: chained CDM stream not implemented yet"); struct pvr_winsys_compute_submit_info info = { .wait = wait, + .g5p_log = sub->g5p_log_first, .g5p_log_count = sub->g5p_log_count, .frame_num = d->global_queue_present_count, .job_num = d->global_cmd_buffer_submit_count }; info.fw_stream_len = pvr_arch_g5p_compute_stream(d, pvr_csb_get_start_address(cs).addr, diff --git a/src/imagination/vulkan/pvr_arch_g5p_tex.c b/src/imagination/vulkan/pvr_arch_g5p_tex.c index 7d8321db3bf..edb1debaab9 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_tex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_tex.c @@ -14,6 +14,7 @@ #include "vk_enum_to_str.h" #include "vk_format.h" +#include "pvr_g5p_shader_dump.h" #include "vk_internal_exts.h" #include "vk_log.h" #include "vk_util.h" @@ -165,7 +166,7 @@ g5p_view_state(const struct pvr_image_view *iview, bool storage, uint64_t q[3]) /* * The sampled state of a view: q0, q1 in words[], q2 in meta[0..1], - * meta[2..3] zero, the 32 bytes the backend's image upload reads. + * meta[2..3] diagnostic view pointer when logging, otherwise zero. */ static const char * g5p_pack_view(const struct pvr_image_view *iview, bool storage, struct pvr_image_descriptor *desc) @@ -191,6 +192,14 @@ g5p_pack_view(const struct pvr_image_view *iview, bool storage, struct pvr_image desc->words[1] = q[1]; desc->meta[0] = (uint32_t)q[2]; desc->meta[1] = (uint32_t)(q[2] >> 32); + /* q3 is CPU metadata, not one of the three A9 texture-state qwords. + * Descriptor copies/templates preserve it. Dispatch logging snapshots the + * view immediately, never retaining this pointer in a recorded command. */ + if (pvr_g5p_compute_log()) { + const uintptr_t ptr = (uintptr_t)iview; + static_assert(sizeof(ptr) == 8, "A9 userspace pointers"); + memcpy(&desc->meta[2], &ptr, sizeof(ptr)); + } return NULL; } diff --git a/src/imagination/vulkan/pvr_cmd_buffer.h b/src/imagination/vulkan/pvr_cmd_buffer.h index 682a1e64d03..622d85ea9b9 100644 --- a/src/imagination/vulkan/pvr_cmd_buffer.h +++ b/src/imagination/vulkan/pvr_cmd_buffer.h @@ -276,6 +276,8 @@ struct pvr_sub_cmd_compute { /* A kernel record was emitted since the last end of pass: the next one * starts with iOS's barrier task (pvr_arch_g5p_compute.c). */ bool g5p_kernel_since_end; + struct pvr_g5p_dispatch_log *g5p_log_first, *g5p_log_last; + unsigned g5p_log_count; }; struct pvr_sub_cmd_transfer { diff --git a/src/imagination/vulkan/pvr_g5p_compute.h b/src/imagination/vulkan/pvr_g5p_compute.h index 9cf9bb40d54..a2f54cf550f 100644 --- a/src/imagination/vulkan/pvr_g5p_compute.h +++ b/src/imagination/vulkan/pvr_g5p_compute.h @@ -2,6 +2,7 @@ #ifndef PVR_G5P_COMPUTE_H #define PVR_G5P_COMPUTE_H #include +#include #include #include "pvr_macros.h" #include "pvr_winsys.h" @@ -11,9 +12,16 @@ struct pvr_cmd_buffer; struct pvr_sub_cmd_compute; struct pvr_compute_ctx; struct vk_sync; +/* Owned by its command-buffer sub-command; immutable across submits. */ +struct pvr_g5p_dispatch_log { + struct pvr_g5p_dispatch_log *next; + char *line; + char pipeline[13]; + bool stub; +}; #ifdef PVR_PER_ARCH VkResult PVR_PER_ARCH(g5p_compute_pipeline_compile)(struct pvr_device *, const VkComputePipelineCreateInfo *, struct pvr_compute_pipeline *); -VkResult PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *, struct pvr_sub_cmd_compute *, uint64_t, const uint32_t[3], const uint32_t[3]); +VkResult PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *, struct pvr_sub_cmd_compute *, uint64_t, uint64_t, const uint32_t[3], const uint32_t[3]); VkResult PVR_PER_ARCH(g5p_compute_end_pass)(struct pvr_cmd_buffer *, struct pvr_sub_cmd_compute *); VkResult PVR_PER_ARCH(g5p_compute_job_submit)(struct pvr_compute_ctx *, struct pvr_sub_cmd_compute *, struct vk_sync *, struct vk_sync *); VkResult PVR_PER_ARCH(g5p_compute_ctx_create)(struct pvr_device *, enum pvr_winsys_ctx_priority, struct pvr_compute_ctx **); diff --git a/src/imagination/vulkan/pvr_g5p_compute_hooks.h b/src/imagination/vulkan/pvr_g5p_compute_hooks.h index ffad1f784df..8081d307603 100644 --- a/src/imagination/vulkan/pvr_g5p_compute_hooks.h +++ b/src/imagination/vulkan/pvr_g5p_compute_hooks.h @@ -24,7 +24,7 @@ } #define PVR_G5P_HOOK_compute_dispatch() \ if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g5p)) { \ - result = pvr_arch_g5p_cmd_dispatch(cmd_buffer, sub_cmd, indirect_addr.addr, base_group, workgroup_size); \ + result = pvr_arch_g5p_cmd_dispatch(cmd_buffer, sub_cmd, indirect_addr.addr, indirect_offset, base_group, workgroup_size); \ if (result != VK_SUCCESS) pvr_cmd_buffer_set_error_unwarned(cmd_buffer, result); \ goto end_cmd_dispatch; \ } diff --git a/src/imagination/vulkan/pvr_g5p_compute_state.h b/src/imagination/vulkan/pvr_g5p_compute_state.h index 5ee4be651cc..e82ad2e32b7 100644 --- a/src/imagination/vulkan/pvr_g5p_compute_state.h +++ b/src/imagination/vulkan/pvr_g5p_compute_state.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: MIT */ #ifndef PVR_G5P_COMPUTE_STATE_H #define PVR_G5P_COMPUTE_STATE_H -#define PVR_G5P_COMPUTE_STATE_FIELDS unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ +#define PVR_G5P_COMPUTE_STATE_FIELDS char a9_pipeline_id[13]; bool a9_capture_stub; bool a9_compute_log; unsigned a9_log_local[3], a9_log_shared; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ unsigned a9_num_uploads; struct { unsigned char kind, set, binding, element, combined, level, common, dwords; uint32_t value; } a9_uploads[10]; #define PVR_G5P_NATIVE_STORAGE(d) PVR_HAS_FEATURE((d), apple_g5p) #endif diff --git a/src/imagination/vulkan/pvr_g5p_shader_dump.c b/src/imagination/vulkan/pvr_g5p_shader_dump.c index 2432ce40fcb..1565566f1f1 100644 --- a/src/imagination/vulkan/pvr_g5p_shader_dump.c +++ b/src/imagination/vulkan/pvr_g5p_shader_dump.c @@ -30,6 +30,7 @@ DEBUG_GET_ONCE_OPTION(pvr_shader_dump_dir, "PVR_SHADER_DUMP_DIR", "/tmp/pvr-shad static const char *dump_dir; static bool stub_on; +static bool compute_log; static util_once_flag env_once = UTIL_ONCE_FLAG_INIT; static void @@ -45,6 +46,7 @@ read_env(void) else mesa_logi("G5P shader dump: %s (pid %ld)", d, (long)getpid()); } + compute_log = PVR_IS_DEBUG_SET(COMPUTE_LOG); stub_on = PVR_IS_DEBUG_SET(CAPTURE_STUB); if (stub_on) mesa_logw("G5P capture stub: ON (refused shaders are replaced; pictures will be wrong)"); @@ -64,6 +66,13 @@ pvr_g5p_capture_stub(void) return stub_on; } +bool +pvr_g5p_compute_log(void) +{ + util_call_once(&env_once, read_env); + return compute_log; +} + static bool stage_spirv(const VkPipelineShaderStageCreateInfo *info, const uint32_t **words, size_t *bytes) { @@ -82,6 +91,22 @@ stage_spirv(const VkPipelineShaderStageCreateInfo *info, const uint32_t **words, return false; /* a driver-internal NIR module */ } +void +pvr_g5p_shader_id(const VkPipelineShaderStageCreateInfo *info, char id[13]) +{ + const uint32_t *words; + size_t bytes; + memcpy(id, "internal-nir", 13); + if (stage_spirv(info, &words, &bytes)) { + blake3_hash hash; + char hex[BLAKE3_OUT_LEN * 2 + 1]; + _mesa_blake3_compute(words, bytes, hash); + _mesa_blake3_format(hex, hash); + memcpy(id, hex, 12); + id[12] = 0; + } +} + /* write @size bytes to @path through a temporary name and a rename */ static void write_atomic(const char *path, const void *data, size_t size, bool keep_existing) diff --git a/src/imagination/vulkan/pvr_g5p_shader_dump.h b/src/imagination/vulkan/pvr_g5p_shader_dump.h index 609139a0e3d..3d3ddab56c0 100644 --- a/src/imagination/vulkan/pvr_g5p_shader_dump.h +++ b/src/imagination/vulkan/pvr_g5p_shader_dump.h @@ -42,6 +42,8 @@ struct pvr_g5p_dump_blob { /* NULL when PVR_DEBUG=shader_dump is unset (read once) */ const char *pvr_g5p_shader_dump_dir(void); bool pvr_g5p_capture_stub(void); +bool pvr_g5p_compute_log(void); +void pvr_g5p_shader_id(const VkPipelineShaderStageCreateInfo *info, char id[13]); /* Record one compile of @stage_info (no-op without PVR_DEBUG=shader_dump). * @result: "ADMIT", "REFUSE" or "STUB" (refused, then stubbed). */ diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c index 77656c5f443..98990b7cf18 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c @@ -22,6 +22,9 @@ */ #include +#include +#include +#include "pvr_g5p_compute.h" #include #include #include @@ -45,6 +48,7 @@ struct pvr_drm_winsys_compute_ctx { /* Handle to kernel context. */ uint32_t handle; + uint64_t submit_seq; }; #define to_pvr_drm_winsys_compute_ctx(ctx) \ @@ -122,6 +126,7 @@ VkResult pvr_drm_winsys_compute_ctx_create( drm_ctx->base.ws = ws; drm_ctx->handle = ctx_args.handle; + drm_ctx->submit_seq = 0; *ctx_out = &drm_ctx->base; @@ -138,7 +143,7 @@ void pvr_drm_winsys_compute_ctx_destroy(struct pvr_winsys_compute_ctx *ctx) { struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); struct pvr_drm_winsys_compute_ctx *drm_ctx = - to_pvr_drm_winsys_compute_ctx(ctx); + to_pvr_drm_winsys_compute_ctx((struct pvr_winsys_compute_ctx *)ctx); struct drm_pvr_ioctl_destroy_context_args args = { .handle = drm_ctx->handle, }; @@ -174,7 +179,7 @@ VkResult pvr_drm_winsys_compute_submit( { struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); struct pvr_drm_winsys_compute_ctx *drm_ctx = - to_pvr_drm_winsys_compute_ctx(ctx); + to_pvr_drm_winsys_compute_ctx((struct pvr_winsys_compute_ctx *)ctx); struct drm_pvr_sync_op sync_ops[2]; struct drm_pvr_job job_args = { @@ -218,5 +223,22 @@ VkResult pvr_drm_winsys_compute_submit( const VkResult result = pvr_drm_submit_jobs(drm_ws, &args, "compute"); if (result == VK_SUCCESS) pvr_drm_tag_job_sync(drm_ws, signal_sync, "compute"); + if (drm_ws->is_g5p && result == VK_SUCCESS) { + const uint64_t seq = ++drm_ctx->submit_seq; + if (submit_info->g5p_log) { + flockfile(stderr); + fprintf(stderr, "G5P compute ctx=%u submit=%" PRIu64 " out_syncobj=%u dispatches=%u\n", + drm_ctx->handle, seq, + signal_sync ? vk_sync_as_drm_syncobj(signal_sync)->syncobj : 0, + submit_info->g5p_log_count); + unsigned i = 0; + for (const struct pvr_g5p_dispatch_log *l = submit_info->g5p_log; l; l = l->next, i++) { + fprintf(stderr, "G5P compute #%u %s\n", i, l->line); + if (l->stub) + fprintf(stderr, "G5P compute omitted pipe=%s\n", l->pipeline); + } + funlockfile(stderr); + } + } return result; } diff --git a/src/imagination/vulkan/winsys/pvr_winsys.h b/src/imagination/vulkan/winsys/pvr_winsys.h index a5e8d866055..07933c35691 100644 --- a/src/imagination/vulkan/winsys/pvr_winsys.h +++ b/src/imagination/vulkan/winsys/pvr_winsys.h @@ -292,7 +292,11 @@ struct pvr_winsys_transfer_submit_info { struct pvr_winsys_transfer_cmd cmds[PVR_TRANSFER_MAX_PREPARES_PER_SUBMIT]; }; +struct pvr_g5p_dispatch_log; + struct pvr_winsys_compute_submit_info { + const struct pvr_g5p_dispatch_log *g5p_log; + unsigned g5p_log_count; uint32_t frame_num; uint32_t job_num; From 1db492754a9e7c14f6cbc3b794be6e9e26e9e31b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:05:00 +0100 Subject: [PATCH 070/109] g9p: register LOD for A9 compute arrays and cubes, as Apple Array and cube samples at an explicit level now take the level from a register in Apple's form, as Apple's A9 kernels encode it: the f16 conversion reads the level from source slot A with its last read marked, and the sample is the coupled temporary-level form with the level register (g5p_usc_sample_register_lod()), instead of the plain-2D form, which reads the source from slot B. The plain-2D path keeps its form. Assisted-by: LLM --- src/imagination/g5p/g5p_usc.c | 16 +++++++++++ src/imagination/g5p/g5p_usc.h | 3 ++ src/imagination/g9p/g9p_a9_tex.h | 48 ++++++++++++++++++++++++++++---- 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/imagination/g5p/g5p_usc.c b/src/imagination/g5p/g5p_usc.c index d461f02a7ad..ef9c7c6c372 100644 --- a/src/imagination/g5p/g5p_usc.c +++ b/src/imagination/g5p/g5p_usc.c @@ -1180,6 +1180,22 @@ unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t) return finish(out, raw, 10); } +/* A9 compute's array or cube sample with the level in a register (Apple's A9 compiler's form): the coupled + * temporary-level tuple of g5p_usc_sample_ext(). Its fetch form (read) keeps the coordinate operand live: it + * clears the coordinate's last-read bit, which every other sample form sets (texture_base()'s coordinate byte). */ +#define G5P_TEX_COORD_BYTE 5 +#define G5P_TEX_COORD_LAST_READ 0x80 +unsigned g5p_usc_sample_register_lod(uint8_t *out, const struct g5p_tex_ext *t) +{ + CHECK(t->tuple == G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL && t->lod == G5P_TEX_LOD_LEVEL && !t->lod_shared && + (t->dim == G5P_TEX_2D_ARRAY || t->dim == G5P_TEX_CUBE), + "the register-LOD form is an array or cube sample at a level"); + const unsigned n = g5p_usc_sample_ext(out, t); + if (n && t->read) + out[G5P_TEX_COORD_BYTE] &= ~G5P_TEX_COORD_LAST_READ; + return n; +} + unsigned g5p_usc_half_immediate(uint8_t *out, unsigned dst_half, uint16_t value) { CHECK(dst_half <= 63, "dst_half beyond r31"); diff --git a/src/imagination/g5p/g5p_usc.h b/src/imagination/g5p/g5p_usc.h index 7e19c7f7f65..8c05bb9e149 100644 --- a/src/imagination/g5p/g5p_usc.h +++ b/src/imagination/g5p/g5p_usc.h @@ -202,6 +202,9 @@ struct g5p_tex_ext { enum g5p_tex_tuple tuple; bool lod_shared; /* prefix level source: common store instead of register */ }; +/* A9 compute's register-level array or cube sample and fetch: the coupled temporary-level tuple, the fetch's + * coordinate kept live (fields at the encoder). */ +unsigned g5p_usc_sample_register_lod(uint8_t *out, const struct g5p_tex_ext *t); unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t); /* 16-bit immediate into one half (class 0x88, the 32-bit move's split map; * Apple's texel-offset probes witness bits 0-9). */ diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index b5261db812e..8ef0f4f7a0e 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -74,10 +74,7 @@ static int a9_tex_shared_lod(struct g9p_ctx *ctx, nir_tex_instr *tex, bool read) { int li = nir_tex_instr_src_index(tex, nir_tex_src_lod); - if (li >= 0 && !nir_src_is_const(tex->src[li].src)) { - a9_tex_refuse(ctx, "dynamic LOD on a compute 2D array or cube (no A9 witness yet)"); - return -1; - } + assert(li < 0 || nir_src_is_const(tex->src[li].src)); uint32_t value = li < 0 ? 0 : read ? nir_src_as_uint(tex->src[li].src) : _mesa_float_to_half(nir_src_as_float(tex->src[li].src)); unsigned used = 0; @@ -256,7 +253,28 @@ a9_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) /* level (f16 for samples, u32 for reads) or gradients */ int lod; - if (layered) { + const int li = nir_tex_instr_src_index(tex, nir_tex_src_lod); + const bool register_lod = layered && li >= 0 && !nir_src_is_const(tex->src[li].src); + if (register_lod) { + /* Apple's A9 witnesses sla_d/slc_d convert the level to the low f16 of + * a register with the source in slot A, its last read marked; the + * older plain-2D g9p_f16_of() reads slot B; leave that proven path + * alone. Copy into a private low temporary: the witness marks the + * source's last read, so it cannot be a still-live NIR value. */ + int src = src_reg(ctx, nir_get_scalar(tex->src[li].src.ssa, 0)); + if (src < 0) return false; + int copy = alloc_low_temp(ctx); + if (copy < 0) return false; + g9p_alu2(ctx->c, G9P_OR, copy, src, src, false); + lod = read ? copy : alloc_low_temp(ctx); + if (lod < 0) return false; + if (!read) { + const unsigned at = ctx->c->n; + g9p_f16_of(ctx->c, lod, 0); + ctx->c->w[at + 2] |= copy << G9P_SRC_A_SHIFT | G9P_SRC_A_LAST_READ; + } + /* ra_d loads its uint LOD directly into r0: no f16 conversion. */ + } else if (layered) { lod = a9_tex_shared_lod(ctx, tex, read); if (lod < 0) return false; } else if (grad) { @@ -301,6 +319,25 @@ a9_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) if (dst < 0 || dst > 28) return false; + if (register_lod) { + /* Apple's A9 register-LOD array or cube sample (slc_d/sla_d) and fetch + * (ra_d, coordinate r3; its preceding ops produce x,y,layer in r3,r4,r5): + * g5p_usc_sample_register_lod(), the coupled temporary-level tuple + * with the LOD register. Keep this local to these witnessed compute + * cases; g5p_usc_sample_ext() and all graphics encodings are unchanged. */ + const struct g5p_tex_ext x = { + .dst_half = 2 * dst, .image_qword = image / 2, .sampler_qword = sampler / 2, .coord_dword = block, + .dim = cube ? G5P_TEX_CUBE : G5P_TEX_2D_ARRAY, .mask = 0xf, .read = read, + .lod = G5P_TEX_LOD_LEVEL, .lod_dword = lod, .tuple = G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL, + }; + uint8_t raw[G5P_USC_MAX]; + const unsigned n = g5p_usc_sample_register_lod(raw, &x); + if (!n) { + a9_tex_refuse(ctx, "the register-LOD sample was refused by the encoder"); + return false; + } + a9_tex_emit_raw(ctx, raw, n); + } else { struct g5p_tex_ext x = { .dst_half = 2 * dst, .image_qword = image / 2, .sampler_qword = sampler / 2, .coord_dword = block, .dim = cube ? G5P_TEX_CUBE : array ? G5P_TEX_2D_ARRAY : G5P_TEX_2D, .mask = 0xf, .read = read, @@ -316,6 +353,7 @@ a9_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) return false; } a9_tex_emit_raw(ctx, raw, n); + } g9p_wait(ctx->c); for (unsigned c = 0; c < 4; c++) { ctx->reg[key(&tex->def, c)] = dst + c; From ab93615c0918fa4657f08f764d842269cc886c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:05:00 +0100 Subject: [PATCH 071/109] pvr/g5p: spill workgroup memory above 16 KiB to a per-dispatch buffer Four GravityMark kernels declare 28 KiB of workgroup memory, above the A9's 16 KiB. A9 compute now turns the shared accesses of such a kernel into storage-buffer accesses on a driver binding. The address is the workgroup's linear index times the 64-byte-aligned shared size, plus the offset. Its barriers also order buffer memory. The dispatch allocates one uninitialised region for the whole grid and keeps it with the command buffer. The region is capped at 64 MiB. A larger dispatch is omitted, logged once per pipeline, and fails the command buffer with OUT_OF_DEVICE_MEMORY; an indirect spilling dispatch is omitted with FEATURE_NOT_PRESENT. maxComputeSharedMemorySize stays 16384. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_compute.h | 70 ++++++++++++++++++- src/imagination/g9p/g9p_compile.h | 3 + src/imagination/vulkan/pvr_arch_g5p_compute.c | 60 +++++++++++++++- .../vulkan/pvr_g5p_compute_state.h | 2 +- 4 files changed, 130 insertions(+), 5 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index 38ecb33a0b1..cd36fdc46d4 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -157,6 +157,67 @@ a9_lower_array_layer(nir_builder *b, nir_tex_instr *tex, void *data) return true; } +/* The grid owns disjoint stride-sized regions. Keep this before shared + * vectorisation: the ordinary SSBO lowering handles vectors and atomics. */ +static bool +a9_spill_shared(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + if (in->intrinsic == nir_intrinsic_barrier) { + nir_variable_mode modes = nir_intrinsic_memory_modes(in); + if (!(modes & nir_var_mem_shared)) return false; + nir_intrinsic_set_memory_modes(in, modes | nir_var_mem_ssbo); + return true; + } + nir_intrinsic_op op; + unsigned off_src = 0; + switch (in->intrinsic) { + case nir_intrinsic_load_shared: op = nir_intrinsic_load_ssbo; break; + case nir_intrinsic_store_shared: op = nir_intrinsic_store_ssbo; off_src = 1; break; + case nir_intrinsic_shared_atomic: op = nir_intrinsic_ssbo_atomic; break; + case nir_intrinsic_shared_atomic_swap: op = nir_intrinsic_ssbo_atomic_swap; break; + default: return false; + } + b->cursor = nir_before_instr(&in->instr); + nir_def *wg = nir_load_workgroup_id(b), *nwg = nir_load_num_workgroups(b); + nir_def *linear = nir_iadd(b, nir_channel(b, wg, 0), + nir_imul(b, nir_channel(b, nwg, 0), nir_iadd(b, nir_channel(b, wg, 1), + nir_imul(b, nir_channel(b, nwg, 1), nir_channel(b, wg, 2))))); + nir_def *offset = nir_iadd(b, nir_imul_imm(b, linear, *(unsigned *)data), + nir_iadd_imm(b, in->src[off_src].ssa, nir_intrinsic_base(in))); + nir_def *index = nir_vulkan_resource_index(b, 2, 32, nir_imm_int(b, 0), + .desc_set = G9P_SHARED_SPILL, .binding = 0, + .desc_type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER); + nir_intrinsic_instr *spill = nir_intrinsic_instr_create(b->shader, op); + spill->num_components = in->num_components; + nir_intrinsic_set_access(spill, nir_intrinsic_access(in)); + if (op == nir_intrinsic_store_ssbo) { + spill->src[0] = nir_src_for_ssa(in->src[0].ssa); + spill->src[1] = nir_src_for_ssa(index); + spill->src[2] = nir_src_for_ssa(offset); + nir_intrinsic_set_write_mask(spill, nir_intrinsic_write_mask(in)); + } else { + spill->src[0] = nir_src_for_ssa(index); + spill->src[1] = nir_src_for_ssa(offset); + if (op != nir_intrinsic_load_ssbo) { + nir_intrinsic_set_atomic_op(spill, nir_intrinsic_atomic_op(in)); + spill->src[2] = nir_src_for_ssa(in->src[1].ssa); + if (op == nir_intrinsic_ssbo_atomic_swap) + spill->src[3] = nir_src_for_ssa(in->src[2].ssa); + } + nir_def_init(&spill->instr, &spill->def, in->def.num_components, in->def.bit_size); + } + if (op == nir_intrinsic_load_ssbo || op == nir_intrinsic_store_ssbo) { + nir_intrinsic_set_align_mul(spill, nir_intrinsic_align_mul(in)); + nir_intrinsic_set_align_offset(spill, + (nir_intrinsic_align_offset(in) + nir_intrinsic_base(in)) % nir_intrinsic_align_mul(in)); + } + nir_builder_instr_insert(b, &spill->instr); + if (op != nir_intrinsic_store_ssbo) + nir_def_rewrite_uses(&in->def, &spill->def); + nir_instr_remove(&in->instr); + return true; +} + bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_ctx) { nir->options = &nir_options_a9; @@ -174,13 +235,16 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c NIR_PASS(_, nir, nir_lower_alu); /* lower_front() already allocated and lowered shared variables. Doing * it again double-counts the allocation after addresses became explicit. */ + if (nir->info.shared_size > 16384) { + out->shared_spill_stride = ALIGN_POT(nir->info.shared_size, 64); + NIR_PASS(_, nir, nir_shader_intrinsics_pass, a9_spill_shared, + nir_metadata_control_flow, &out->shared_spill_stride); + nir->info.shared_size = 0; + } const nir_lower_mem_access_bit_sizes_options shared_opts = { .modes = nir_var_mem_shared, .callback = a9_shared_size_align, }; NIR_PASS(_, nir, nir_lower_mem_access_bit_sizes, &shared_opts); - if (nir->info.shared_size > 16384) { - return g9p_fail(out, "shared memory %u bytes, above the A9's 16384", nir->info.shared_size); - } out->a9_local_dwords = 4 + DIV_ROUND_UP(nir->info.shared_size, 4); NIR_PASS(_, nir, nir_shader_tex_pass, a9_lower_array_layer, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_lower_compute_system_values, &csv); diff --git a/src/imagination/g9p/g9p_compile.h b/src/imagination/g9p/g9p_compile.h index 062a0f2abbb..e97bc6583dc 100644 --- a/src/imagination/g9p/g9p_compile.h +++ b/src/imagination/g9p/g9p_compile.h @@ -89,6 +89,8 @@ g9p_fs_buf_base(unsigned n) #define G9P_SHARED_BASE 16 /* Compute: gl_NumWorkGroups, x and y in binding 0's slot, z in binding 1's (filled per dispatch) */ #define G9P_NUM_WORKGROUPS (~22u) +/* Device-backed workgroup memory, binding 0. */ +#define G9P_SHARED_SPILL (~24u) /* A draw's parameters: binding 0's slot holds firstInstance (filled per draw) */ #define G9P_DRAW_PARAMS (~23u) /* The set of the push-constant slot (its address is loaded like a buffer's) */ @@ -193,6 +195,7 @@ struct g9p_shader { unsigned local_size[3]; /* compute: bytes of shared variables (after G9P_SHARED_BASE) */ unsigned shared_size; + unsigned shared_spill_stride; /* 0: hardware threadgroup memory */ /* * Vertex: varyings written (VARn, each 4 components after the position); diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index a7483427ceb..58ea064c830 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -8,6 +8,10 @@ #include "vk_pipeline.h" #include "vk_alloc.h" #include "util/ralloc.h" +#include "util/u_atomic.h" +#include "util/u_debug.h" +#include "util/u_call_once.h" +#include "util/log.h" #include "g9p_cdm.h" #include "g9p_compile.h" #include "g5p_cdm.h" @@ -175,6 +179,7 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( state->a9_uploads[i].dwords = u->dwords; state->a9_uploads[i].value = u->value; } + state->shared_spill_stride = shader.shared_spill_stride; state->uses_workgroup_id = shader.uses_workgroup_id; memcpy(state->local_size, shader.local_size, sizeof(state->local_size)); @@ -283,6 +288,10 @@ g5p_record_dispatch(struct pvr_cmd_buffer *cmd, struct pvr_sub_cmd_compute *sub, return VK_SUCCESS; } +/* This translation unit is A9-only. Read the override once, off the hot path. */ +/* Bound per-dispatch shared-memory spill allocations to 64 MiB. */ +#define PVR_G5P_SHARED_SPILL_MAX_BYTES (64u << 20) + /* * One dispatch: a DMA task loading the buffer addresses (from the bound * descriptor sets) and the thread count into the common store, then the @@ -341,6 +350,50 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, return VK_SUCCESS; } + struct pvr_bo *shared_spill = NULL; + if (state->shared_spill_stride) { + if (indirect_va) { + if (p_atomic_cmpxchg(&state->shared_spill_logged, 0, 1) == 0) + mesa_logw("G5P refuses: indirect dispatch of a spilling workgroup-memory pipeline"); + vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + + uint64_t groups = (uint64_t)group_count[0] * group_count[1] * group_count[2]; + /* Divide before multiplying: even a valid grid can overflow bytes. */ + if (groups > PVR_G5P_SHARED_SPILL_MAX_BYTES / state->shared_spill_stride) { + if (p_atomic_cmpxchg(&state->shared_spill_logged, 0, 1) == 0) + mesa_logw("G5P refuses: a dispatch of %" PRIu64 " groups × %u bytes of spilled workgroup memory " + "(%.2f MB, above the cap)", groups, state->shared_spill_stride, + (double)groups * state->shared_spill_stride / (1024 * 1024)); + vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + /* bo_list is freed as pvr_suballoc_bo, despite its pvr_bo comment. + * Give the dedicated generic BO a normal ownership wrapper. Reset + * may cache it in suballoc_general, so use that allocator's mapping + * flags too; a later ordinary upload must find a mapped BO. */ + struct pvr_suballoc_bo *owner = vk_zalloc(&device->vk.alloc, sizeof(*owner), 8, + VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + if (!owner) { + vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + result = pvr_bo_alloc(device, device->heaps.general_heap, + groups * state->shared_spill_stride, 64, + PVR_BO_ALLOC_FLAG_CPU_MAPPED, &shared_spill); + if (result != VK_SUCCESS) { + vk_free(&device->vk.alloc, owner); + vk_command_buffer_set_error(&cmd_buffer->vk, result); + return result; + } + owner->allocator = &device->suballoc_general; + owner->bo = shared_spill; + owner->dev_addr = shared_spill->vma->dev_addr; + owner->size = groups * state->shared_spill_stride; + list_add(&owner->link, &cmd_buffer->bo_list); + } + /* * Values the DMA copies that are not in descriptor memory: the * push-constant buffer's address and the thread count. @@ -355,6 +408,10 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uniforms[2 * k] = group_count[state->buffers[k].binding ? 2 : 0]; uniforms[2 * k + 1] = state->buffers[k].binding ? 0 : group_count[1]; } + if (state->buffers[k].set == G9P_SHARED_SPILL) { + uniforms[2 * k] = shared_spill->vma->dev_addr.addr; + uniforms[2 * k + 1] = shared_spill->vma->dev_addr.addr >> 32; + } if (state->buffers[k].set == G9P_PUSH_CONSTANTS) { const struct pvr_push_constants *pc = &cmd_buffer->state.push_consts[PVR_STAGE_ALLOCATION_COMPUTE]; @@ -449,7 +506,8 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, dest = state->a9_uploads[i].common; dwords = state->a9_uploads[i].dwords; } else if (k < n && state->buffers[k].set != G9P_PUSH_CONSTANTS && - state->buffers[k].set != G9P_NUM_WORKGROUPS) { + state->buffers[k].set != G9P_NUM_WORKGROUPS && + state->buffers[k].set != G9P_SHARED_SPILL) { const struct pvr_descriptor_set *set = desc->sets[state->buffers[k].set]; /* an array's element (Zink keeps every storage buffer in one array) and the size flag (the * descriptor's size word at byte 8, arrayLength()): G9P's 496e44739dc ("pvr/g9p: gl_NumWorkGroups, workgroup ID z, storage buffer array elements in compute"), 1d51ff5cec6 */ diff --git a/src/imagination/vulkan/pvr_g5p_compute_state.h b/src/imagination/vulkan/pvr_g5p_compute_state.h index e82ad2e32b7..727502925bb 100644 --- a/src/imagination/vulkan/pvr_g5p_compute_state.h +++ b/src/imagination/vulkan/pvr_g5p_compute_state.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: MIT */ #ifndef PVR_G5P_COMPUTE_STATE_H #define PVR_G5P_COMPUTE_STATE_H -#define PVR_G5P_COMPUTE_STATE_FIELDS char a9_pipeline_id[13]; bool a9_capture_stub; bool a9_compute_log; unsigned a9_log_local[3], a9_log_shared; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ +#define PVR_G5P_COMPUTE_STATE_FIELDS unsigned shared_spill_stride; int shared_spill_logged; char a9_pipeline_id[13]; bool a9_capture_stub; bool a9_compute_log; unsigned a9_log_local[3], a9_log_shared; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ unsigned a9_num_uploads; struct { unsigned char kind, set, binding, element, combined, level, common, dwords; uint32_t value; } a9_uploads[10]; #define PVR_G5P_NATIVE_STORAGE(d) PVR_HAS_FEATURE((d), apple_g5p) #endif From da1a0a5d7f4afd4523c6f10919337bb9a33f12e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:05:01 +0100 Subject: [PATCH 072/109] g9p: grow loop, exit and code storage; size loop records to the program The G9P compiler kept liveness records for at most 64 loops in a fixed array (a limit spelled per backend since the storage-buffer change), refused programs with more than 256 exits, and asserted past 65536 code words. Loop records are now counted before liveness and allocated to size. Exit jumps double their storage as needed, and code storage starts at 65536 words and doubles. Every back-patch reacquires the buffer by word index. The 64-loop check is gone. If-else nesting deeper than 60 is refused by name: the break count is a 6-bit immediate (W2[6:1]) holding if_depth + 3, so 60 is the deepest nesting this form encodes. Before, deeper nesting was mis-encoded. This reaches the A10 too (compile-checked only). Assisted-by: LLM --- src/imagination/g9p/g9p_a9_alu.h | 2 +- src/imagination/g9p/g9p_a9_intrinsics.h | 3 +- src/imagination/g9p/g9p_a9_tex.h | 2 +- src/imagination/g9p/g9p_compile.c | 123 +++++++++++++++++------- src/imagination/g9p/g9p_isa.h | 81 +++++++++++++--- 5 files changed, 161 insertions(+), 50 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_alu.h b/src/imagination/g9p/g9p_a9_alu.h index 3d676ff76c8..3d653bb8ff9 100644 --- a/src/imagination/g9p/g9p_a9_alu.h +++ b/src/imagination/g9p/g9p_a9_alu.h @@ -83,7 +83,7 @@ a9_alu_pack_half(struct g9p_ctx *ctx, nir_alu_instr *alu, nir_scalar s[3]) if (r < 0 || (r = one_src(ctx, r)) < 0) return false; unsigned at = ctx->c->n; g9p_f16_of(ctx->c, dst, r); - ctx->c->w[at + 1] |= i; /* the high half: bit 0 of the destination half index, as Apple's half packs */ + (*g9p_code_word(ctx->c, at + 1)) |= i; /* the high half: bit 0 of the destination half index, as Apple's half packs */ } release(ctx, &alu->instr); ctx->busy |= BITFIELD64_BIT(dst); diff --git a/src/imagination/g9p/g9p_a9_intrinsics.h b/src/imagination/g9p/g9p_a9_intrinsics.h index 58efcaf6d50..f925e9d5a11 100644 --- a/src/imagination/g9p/g9p_a9_intrinsics.h +++ b/src/imagination/g9p/g9p_a9_intrinsics.h @@ -109,7 +109,8 @@ static bool a9_barrier(struct g9p_ctx *ctx, nir_intrinsic_instr *in) } if (scope == SCOPE_WORKGROUP) { unsigned N = ctx->a9_barriers_remaining--; - if (!N || N > 31) return false; + if (!N || N > 31) + return g9p_fail(ctx->out, "more than 31 workgroup barrier sites in a compute shader"); if (n > 32) { int r = alloc_temp(ctx), t = alloc_temp(ctx); if (r < 0 || t < 0) return false; diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index 8ef0f4f7a0e..0069a620322 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -271,7 +271,7 @@ a9_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) if (!read) { const unsigned at = ctx->c->n; g9p_f16_of(ctx->c, lod, 0); - ctx->c->w[at + 2] |= copy << G9P_SRC_A_SHIFT | G9P_SRC_A_LAST_READ; + (*g9p_code_word(ctx->c, at + 2)) |= copy << G9P_SRC_A_SHIFT | G9P_SRC_A_LAST_READ; } /* ra_d loads its uint LOD directly into r0: no f16 conversion. */ } else if (layered) { diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index 5ea5442f678..5473105d160 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -140,7 +140,9 @@ g9p_a9_bank1_limit(unsigned threads) #define VALUE_REGS(ctx) ((ctx)->stage == MESA_SHADER_VERTEX && (ctx)->st == 15 && !getenv("G9P_REGS") \ ? 32u : (ctx)->value_cap ? MIN2(G9P_NUM_REGS, (ctx)->value_cap) : G9P_NUM_REGS) #define LOW_REGS(ctx) MIN2(16u, VALUE_REGS(ctx)) -#define G9P_MAX_LOOPS 64 +/* The immediate source of a break's stack word is six bits (W2[6:1]); + * bit 7 is last-read, bit 8 continuation. It encodes if_depth + 3. */ +#define G9P_MAX_IF_DEPTH (63 - 3) /* nir_tex_instr.backend_flags: a seamful cube's gradients, for the hardware */ #define G9P_TEX_NATIVE_GRADIENTS 1 @@ -349,7 +351,7 @@ struct g9p_ctx { struct { unsigned start, end; - } loops[G9P_MAX_LOOPS]; + } *loops; unsigned num_loops, loop_index; /* Jumps to the innermost loop's exit (after a break), patched at its end */ @@ -362,6 +364,11 @@ struct g9p_ctx { static bool backend_failure(struct g9p_shader *out, const struct g9p_ctx *ctx) { + if (ctx->c && debug_get_option_g9p_pressure()) + mesa_logi("g9p: failed attempt reached %u bytes of code, %u loop records, instruction %u\n", + 2 * ctx->c->n, ctx->num_loops, ctx->ip); + if (ctx->c && ctx->c->error) + return g9p_fail(out, "%s", ctx->c->error); if (out->a9) { if (out->a9_refusal[0]) { char why[sizeof(out->a9_refusal)]; @@ -409,9 +416,9 @@ helper_samples_end(struct g9p_ctx *ctx) const unsigned at = ctx->frag_samples[i]; /* Apple's word pair for a sample before the last: W3 0600, W4 044d */ - if (at != last && ctx->c->w[at + 4] == 0x0200) { - ctx->c->w[at + 3] &= ~0x0100; - ctx->c->w[at + 4] = 0x044d; + if (at != last && (*g9p_code_word(ctx->c, at + 4)) == 0x0200) { + (*g9p_code_word(ctx->c, at + 3)) &= ~0x0100; + (*g9p_code_word(ctx->c, at + 4)) = 0x044d; } } } @@ -1089,12 +1096,11 @@ walk_liveness(struct g9p_ctx *ctx, struct exec_list *list, bool uses) nir_loop *loop = nir_cf_node_as_loop(node); unsigned l = ctx->loop_index++; - if (!uses && l < G9P_MAX_LOOPS) { + if (!uses && l < ctx->num_loops) { ctx->loops[l].start = ctx->ip + 1; - ctx->num_loops = MAX2(ctx->num_loops, l + 1); } walk_liveness(ctx, &loop->body, uses); - if (!uses && l < G9P_MAX_LOOPS) + if (!uses && l < ctx->num_loops) ctx->loops[l].end = ctx->ip; break; } @@ -1104,6 +1110,23 @@ walk_liveness(struct g9p_ctx *ctx, struct exec_list *list, bool uses) } } +/* Exactly the loop records visited by walk_liveness(), including nested + * loops in either arm. Allocate before its two traversals. */ +static unsigned +count_loop_records(struct exec_list *list) +{ + unsigned count = 0; + foreach_list_typed(nir_cf_node, node, node, list) { + if (node->type == nir_cf_node_loop) { + count += 1 + count_loop_records(&nir_cf_node_as_loop(node)->body); + } else if (node->type == nir_cf_node_if) { + nir_if *nif = nir_cf_node_as_if(node); + count += count_loop_records(&nif->then_list) + count_loop_records(&nif->else_list); + } + } + return count; +} + /* --- Code ---------------------------------------------------------------- */ /* The common-store slot of the storage buffer @index refers to (@flags on the binding) */ @@ -2628,9 +2651,9 @@ sample_lod(struct g9p_ctx *ctx, nir_tex_instr *tex, unsigned at, int lod) { if (lod < 0) return; - ctx->c->w[at + 3] = (ctx->c->w[at + 3] & 0xff00) | 0x80 | + (*g9p_code_word(ctx->c, at + 3)) = ((*g9p_code_word(ctx->c, at + 3)) & 0xff00) | 0x80 | (tex->op == nir_texop_txl ? 0x40 : 0x20) | lod; - ctx->c->w[at + 4] = (ctx->c->w[at + 4] & 0xff00) | 0x80; + (*g9p_code_word(ctx->c, at + 4)) = ((*g9p_code_word(ctx->c, at + 4)) & 0xff00) | 0x80; } /* @@ -2774,11 +2797,11 @@ extras_patch(struct g9p_ctx *ctx, unsigned at, const struct tex_extras *e) { if (e->half < 0) return; - ctx->c->w[at + 4] |= 0x40 | e->half; + (*g9p_code_word(ctx->c, at + 4)) |= 0x40 | e->half; if (e->compare) - ctx->c->w[at + 3] |= 0x8000; + (*g9p_code_word(ctx->c, at + 3)) |= 0x8000; if (e->offset) - ctx->c->w[at + 2] |= 0x4000; + (*g9p_code_word(ctx->c, at + 2)) |= 0x4000; } /* @@ -2836,7 +2859,7 @@ emit_tex_layered(struct g9p_ctx *ctx, nir_tex_instr *tex) return false; at = ctx->c->n; g9p_sample_regs(ctx->c, dst, tex_state_dword(ctx, slot), tex_sampler_dword(ctx, slot)); - ctx->c->w[at] = (ctx->c->w[at] & ~0x0f00) | + (*g9p_code_word(ctx->c, at)) = ((*g9p_code_word(ctx->c, at)) & ~0x0f00) | (cube ? G9P_TEX_CUBE : is3d ? G9P_TEX_3D : G9P_TEX_2D_ARRAY) << 8; sample_lod(ctx, tex, at, lod); extras_patch(ctx, at, &extras); @@ -2952,18 +2975,18 @@ emit_tex_vertex(struct g9p_ctx *ctx, nir_tex_instr *tex) g9p_sample_vertex(ctx->c, dst, tex_state_dword(ctx, slot), tex_sampler_dword(ctx, slot), coord, lod); if (cube) { - ctx->c->w[at] = (ctx->c->w[at] & ~0x0f00) | G9P_TEX_CUBE << 8; - ctx->c->w[at + 2] |= 0x8000; + (*g9p_code_word(ctx->c, at)) = ((*g9p_code_word(ctx->c, at)) & ~0x0f00) | G9P_TEX_CUBE << 8; + (*g9p_code_word(ctx->c, at + 2)) |= 0x8000; } else if (is3d) { - ctx->c->w[at] = (ctx->c->w[at] & ~0x0f00) | G9P_TEX_3D << 8; + (*g9p_code_word(ctx->c, at)) = ((*g9p_code_word(ctx->c, at)) & ~0x0f00) | G9P_TEX_3D << 8; } else if (tex->is_array) { - ctx->c->w[at] = (ctx->c->w[at] & ~0x0f00) | G9P_TEX_2D_ARRAY << 8; - ctx->c->w[at + 2] |= 0x8000; + (*g9p_code_word(ctx->c, at)) = ((*g9p_code_word(ctx->c, at)) & ~0x0f00) | G9P_TEX_2D_ARRAY << 8; + (*g9p_code_word(ctx->c, at + 2)) |= 0x8000; } extras_patch(ctx, at, &extras); /* the vertex form's compare also sets bit 15 of W6 (Apple's sample_compare at a level) */ if (extras.compare) - ctx->c->w[at + 6] |= 0x8000; + (*g9p_code_word(ctx->c, at + 6)) |= 0x8000; g9p_wait(ctx->c); release(ctx, &tex->instr); for (unsigned c = 0; c < 4; c++) { @@ -3640,8 +3663,13 @@ emit_break(struct g9p_ctx *ctx, bool when_true) } g9p_loop_break(ctx->c, ctx->st, when_true); if (ctx->num_exit_jumps == ctx->max_exit_jumps) { - fprintf(stderr, "g9p: too many breaks\n"); - return false; + unsigned capacity = ctx->max_exit_jumps * 2; + unsigned *jumps = capacity > ctx->max_exit_jumps ? + reralloc(ctx->c, ctx->exit_jumps, unsigned, capacity) : NULL; + if (!jumps) + return g9p_fail(ctx->out, "allocation failure growing loop exits"); + ctx->exit_jumps = jumps; + ctx->max_exit_jumps = capacity; } ctx->exit_jumps[ctx->num_exit_jumps++] = g9p_jump(ctx->c, G9P_JMP_NONE, 0); return true; @@ -3657,6 +3685,8 @@ emit_if_else(struct g9p_ctx *ctx, nir_if *nif, bool sense) { unsigned to_else, to_pop; + if (ctx->if_depth >= G9P_MAX_IF_DEPTH) + return g9p_fail(ctx->out, "control flow nested deeper than %u", G9P_MAX_IF_DEPTH); /* keep the lanes for which the condition holds */ g9p_push_if(ctx->c, ctx->st, !sense); ctx->if_depth++; @@ -3755,6 +3785,7 @@ emit_cf_list(struct g9p_ctx *ctx, struct exec_list *list) switch (node->type) { case nir_cf_node_block: nir_foreach_instr(instr, nir_cf_node_as_block(node)) { + if (ctx->c->error) return false; ctx->ip++; /* NIR registers whose live range starts here (a loop's start) */ for (unsigned d = 0; d < ctx->num_decls; d++) { @@ -3910,11 +3941,12 @@ generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) * write is the end. */ stack = ctx->stage == MESA_SHADER_COMPUTE || ctx->c->uses_stack; + if (ctx->c->error) return g9p_fail(ctx->out, "%s", ctx->c->error); body_n = ctx->c->n; G9P_A9_SET_BODYLESS(ctx, body_n); body = malloc(body_n * sizeof(uint16_t) + 1); if (!body) - return false; + return g9p_fail(ctx->out, "allocation failure framing program code"); memcpy(body, ctx->c->w, body_n * sizeof(uint16_t)); ctx->c->n = 0; if (ctx->stage == MESA_SHADER_COMPUTE) @@ -4027,7 +4059,7 @@ generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) g9p_mov_from_high(ctx->c, data, st->data); } g9p_mem_at(ctx->c, true, G9P_SIZE_U32, data, addrs[k].pair); - ctx->c->w[ctx->c->n - 1] |= st->dword; + (*g9p_code_word(ctx->c, ctx->c->n - 1)) |= st->dword; if (++batch == 8 && i + 1 < ctx->num_vs_stores) { g9p_wait(ctx->c); batch = 0; @@ -4079,7 +4111,7 @@ generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) G9P_A9_END_PAD(ctx); g9p_end(ctx->c); break; } - return true; + return !ctx->c->error || g9p_fail(ctx->out, "%s", ctx->c->error); } int @@ -5000,6 +5032,10 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.attribs = attribs; ctx.frag_coord_slot = frag_coord_slot; ctx.c = rzalloc(mem_ctx, struct g9p_code); + if (!ctx.c) return g9p_fail(out, "allocation failure allocating program code"); + ctx.c->alloc_ctx = ctx.c; + ctx.c->w = ctx.c->initial_w; + ctx.c->capacity = G9P_INITIAL_CODE_WORDS; ctx.c->fragment = stage == MESA_SHADER_FRAGMENT; ctx.c->high_bank0 = stage == MESA_SHADER_COMPUTE && out->a9; ctx.c->a9_stack = stage == MESA_SHADER_COMPUTE && out->a9; @@ -5018,14 +5054,14 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, memset(ctx.reg_first, 0xff, impl->ssa_alloc * sizeof(unsigned)); ctx.decls = ralloc_array(mem_ctx, nir_def *, impl->ssa_alloc); ctx.max_exit_jumps = 256; - ctx.exit_jumps = ralloc_array(mem_ctx, unsigned, ctx.max_exit_jumps); + ctx.exit_jumps = ralloc_array(ctx.c, unsigned, ctx.max_exit_jumps); + ctx.num_loops = count_loop_records(&impl->body); + ctx.loops = ralloc_array(ctx.c, __typeof__(*ctx.loops), MAX2(ctx.num_loops, 1)); + if (!ctx.loops || !ctx.exit_jumps) + return g9p_fail(out, "allocation failure allocating loop records or exits"); ctx.ip = 0; walk_liveness(&ctx, &impl->body, false); - if (ctx.loop_index > (out->a9 ? G9P_MAX_LOOPS : 64)) { - g9p_fail(out, "too many loops (%u, maximum %u)", ctx.loop_index, out->a9 ? G9P_MAX_LOOPS : 64); - return backend_failure(out, &ctx); - } ctx.ip = 0; ctx.loop_index = 0; walk_liveness(&ctx, &impl->body, true); @@ -5064,6 +5100,16 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, } mesa_logi( "g9p: pressure peak %u at instruction %u of %u (%u NIR register scalars)\n", peak, peak_ip, ctx.ip, decls); + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + nir_def *def = nir_instr_def(instr); + if (def && ctx.def_ip[def->index] == peak_ip) { + mesa_logi( "g9p: pressure peak instruction: "); + nir_print_instr(instr, stderr); + fputc('\n', stderr); + } + } + } } /* @@ -5080,8 +5126,10 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, struct { unsigned start, end; int8_t home; - } *homed = ralloc_array(mem_ctx, __typeof__(*homed), 1024); - unsigned num_homed = 0; + } *homed = NULL; + unsigned num_homed = 0, homed_capacity = 1024; + homed = ralloc_array(mem_ctx, __typeof__(*homed), homed_capacity); + if (!homed) return g9p_fail(out, "allocation failure allocating parked live ranges"); /* retries that ran out at the same instruction: parking more does not help there */ unsigned stuck = 0, last_oom_ip = UINT_MAX; @@ -5105,7 +5153,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, continue; } /* A9 compute parks values above its bank-1 limit in bank 0. */ - if (!ctx.oom || (stage == MESA_SHADER_COMPUTE && !(out->a9)) || num_homed >= 1024 || + if (!ctx.oom || (stage == MESA_SHADER_COMPUTE && !(out->a9)) || g9p_no_homes) return backend_failure(out, &ctx); stuck = ctx.oom_ip == last_oom_ip ? stuck + 1 : 0; @@ -5176,6 +5224,15 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, g9p_fail(out, "out of registers above r31 at instruction %u (bank-0 homes exhausted)", ctx.oom_ip); return backend_failure(out, &ctx); } + if (num_homed == homed_capacity) { + unsigned capacity = homed_capacity * 2; + __typeof__(homed) ranges = capacity > homed_capacity ? + reralloc(mem_ctx, homed, __typeof__(*homed), capacity) : NULL; + if (!ranges) + return g9p_fail(out, "allocation failure growing parked live ranges"); + homed = ranges; + homed_capacity = capacity; + } homed[num_homed++] = (__typeof__(*homed)){ start, end, (int8_t)home }; ctx.home[victim] = home; ctx.num_homes = MAX2(ctx.num_homes, (unsigned)home - 31); diff --git a/src/imagination/g9p/g9p_isa.h b/src/imagination/g9p/g9p_isa.h index 19bf8632b78..1d2aa517ac7 100644 --- a/src/imagination/g9p/g9p_isa.h +++ b/src/imagination/g9p/g9p_isa.h @@ -16,11 +16,22 @@ #include #include #include +#include +#include +#include "util/ralloc.h" -#define G9P_MAX_CODE_WORDS 65536 +#define G9P_INITIAL_CODE_WORDS 65536 struct g9p_code { - uint16_t w[G9P_MAX_CODE_WORDS]; + uint16_t *w; + unsigned capacity; + void *alloc_ctx; + const char *error; + /* Small generated graphics helpers are calloc/free objects, not ralloc + * objects. Preserve their inline storage and lifetime. Compiler programs + * grow beyond this initial storage under alloc_ctx. */ + uint16_t initial_w[G9P_INITIAL_CODE_WORDS]; + uint16_t failed_patch; unsigned n; /* a fragment shader: registers in the other bank, see g9p_w0() */ bool fragment; @@ -57,9 +68,51 @@ g9p_dst(const struct g9p_code *c, unsigned r) static inline void g9p_emit(struct g9p_code *c, const uint16_t *w, unsigned count) { - assert(c->n + count <= G9P_MAX_CODE_WORDS); - for (unsigned i = 0; i < count; i++) - c->w[c->n++] = w[i]; + if (c->error) return; + if (!c->w) { + c->w = c->initial_w; + c->capacity = G9P_INITIAL_CODE_WORDS; + } + /* Signed 32-bit byte-relative jumps and unsigned byte code_size. */ + if (count > INT_MAX / 2u - c->n) { + c->error = "program exceeds the 32-bit jump-offset range"; + return; + } + if (count > c->capacity - c->n) { + unsigned capacity = c->capacity; + while (capacity < c->n + count) + capacity = capacity > INT_MAX / 4u ? INT_MAX / 2u : capacity * 2; + if (!c->alloc_ctx) { + c->error = "generated helper exceeds inline code storage"; + return; + } + uint16_t *w = reralloc(c->alloc_ctx, c->w == c->initial_w ? NULL : c->w, + uint16_t, capacity); + if (!w) { + c->error = "allocation failure growing program code"; + return; + } + if (c->w == c->initial_w) + memcpy(w, c->w, c->n * sizeof(*w)); + c->w = w; + c->capacity = capacity; + } + memcpy(c->w + c->n, w, count * sizeof(*w)); + c->n += count; +} + +/* Patches retain word indices, never pointers across an emit/reallocation. + * A failed append must not turn its following patch into an out-of-bounds + * write. The compile reports error before exporting code. */ +static inline uint16_t * +g9p_code_word(struct g9p_code *c, unsigned at) +{ + if (c->error) return &c->failed_patch; + if (at >= c->n) { + c->error = "program back-patch outside emitted code"; + return &c->failed_patch; + } + return &c->w[at]; } #define G9P_EMIT(c, ...) \ @@ -136,10 +189,10 @@ g9p_end(struct g9p_code *c) unsigned end = (jump + 6 + 63) / 64 * 64; G9P_EMIT(c, 0x0100, 0x0100 | (end - jump), 0x0000); - while (2 * c->n < end) + while (!c->error && 2 * c->n < end) G9P_EMIT(c, 0xc000); G9P_EMIT(c, 0x8000); - while (c->n % 32) + while (!c->error && c->n % 32) G9P_EMIT(c, 0xc000); } @@ -568,8 +621,8 @@ g9p_jump_patch(struct g9p_code *c, unsigned at, int offset) { uint32_t off = (uint32_t)offset; - c->w[at + 1] = (off & 0xfefe) | 0x0100 | ((off >> 8) & 1); - c->w[at + 2] = ((off >> 16) & 0xfefe) | ((off >> 24) & 1); + *g9p_code_word(c, at + 1) = (off & 0xfefe) | 0x0100 | ((off >> 8) & 1); + *g9p_code_word(c, at + 2) = ((off >> 16) & 0xfefe) | ((off >> 24) & 1); } static inline unsigned @@ -602,10 +655,10 @@ g9p_end_vertex(struct g9p_code *c, unsigned reg, unsigned slot) unsigned end = (jump + 6 + 63) / 64 * 64; G9P_EMIT(c, 0x0100, 0x0100 | (end - jump), 0x0000); - while (2 * c->n < end) + while (!c->error && 2 * c->n < end) G9P_EMIT(c, 0xc000); G9P_EMIT(c, 0x155e, 0xc100 | 2 * reg, slot); - while (c->n % 32) + while (!c->error && c->n % 32) G9P_EMIT(c, 0x0000); } @@ -617,10 +670,10 @@ g9p_end_vertex_stores(struct g9p_code *c) unsigned end = (jump + 6 + 63) / 64 * 64; G9P_EMIT(c, 0x0100, 0x0100 | (end - jump), 0x0000); - while (2 * c->n < end) + while (!c->error && 2 * c->n < end) G9P_EMIT(c, 0xc000); G9P_EMIT(c, 0x015e, 0xa000); - while (c->n % 32) + while (!c->error && c->n % 32) G9P_EMIT(c, 0x0000); } @@ -835,7 +888,7 @@ g9p_end_fragment(struct g9p_code *c, unsigned words) if (words) G9P_EMIT(c, (words & 7) << 12 | 0x011c, 0x0800); G9P_EMIT(c, 0x8000); - while (c->n % 32) + while (!c->error && c->n % 32) G9P_EMIT(c, 0x0000); } From 9a09e5223bc035304c68f448df79a9478a4b25d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:05:01 +0100 Subject: [PATCH 073/109] g9p: A9 barrier ids wrap every 15 sites, as Apple Apple's A9 compiler numbers a kernel's barriers with a countdown that wraps every 15 sites, N = ((rem - 1) % 15) + 1, and puts the same N in both barrier words: kernels with 30, 32, 40 and 70 barriers match the formula at every site. Ours counted down without wrapping. A9 kernels with 16 or more barrier sites change; others are byte-identical. Assisted-by: LLM --- src/imagination/g5p/g5p_barrier.h | 2 +- src/imagination/g9p/g9p_a9_intrinsics.h | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/imagination/g5p/g5p_barrier.h b/src/imagination/g5p/g5p_barrier.h index 9403f0fe0e0..a9b01688410 100644 --- a/src/imagination/g5p/g5p_barrier.h +++ b/src/imagination/g5p/g5p_barrier.h @@ -10,7 +10,7 @@ static inline void g5p_shared_barrier_fenced(struct g9p_code *c, unsigned r, unsigned T, unsigned k, unsigned st, unsigned N, int fence_pair) { - assert(!c->fragment && k < 128 && N && N <= 31); + assert(!c->fragment && k < 128 && N && N <= 15); g9p_wait(c); G9P_EMIT(c, 0x1181, 0x0101 | T << 9, 0x0120, 0xa5bd, 0x0070); unsigned skip = g9p_jump(c, 0x0120, 0); diff --git a/src/imagination/g9p/g9p_a9_intrinsics.h b/src/imagination/g9p/g9p_a9_intrinsics.h index f925e9d5a11..9c1388baee9 100644 --- a/src/imagination/g9p/g9p_a9_intrinsics.h +++ b/src/imagination/g9p/g9p_a9_intrinsics.h @@ -108,9 +108,10 @@ static bool a9_barrier(struct g9p_ctx *ctx, nir_intrinsic_instr *in) g9p_mov_uniform(ctx->c,fence_pair+1,57,zero); } if (scope == SCOPE_WORKGROUP) { - unsigned N = ctx->a9_barriers_remaining--; - if (!N || N > 31) - return g9p_fail(ctx->out, "more than 31 workgroup barrier sites in a compute shader"); + /* Apple's A9 b30/b32/b40/b70 (disassembled barrier kernels): both + * X2f and X0f count down and wrap in 1..15, not 1..31. */ + unsigned rem = ctx->a9_barriers_remaining--; + unsigned N = ((rem - 1) % 15) + 1; if (n > 32) { int r = alloc_temp(ctx), t = alloc_temp(ctx); if (r < 0 || t < 0) return false; From 455ffc17320fabe663cfd972b60ad3ed90972253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:05:01 +0100 Subject: [PATCH 074/109] g9p: A9 compute pressure retry A kernel that runs out of registers is compiled again in pressure mode. Pressure mode is A9 compute only and is reached only after an "out of registers" refusal, so every kernel that compiled before is unchanged: - shared offsets fold into the access (up to 16336 bytes); - register liveness comes from a block-level dataflow (gen/kill backward liveness and forward reaching writes), and a register freed at a loop's end is reused after the loop is popped; - a failed allocation homes up to four held values per attempt, low registers first, before the usual fallbacks. The ASTC encoder kernel of GravityMark (129 values live at its peak) now compiles in 93 registers. This path has not run on a device yet. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_compute.h | 14 +- src/imagination/g9p/g9p_compile.c | 407 ++++++++++++++++++++++----- 2 files changed, 348 insertions(+), 73 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index cd36fdc46d4..c661a4be3f3 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -218,6 +218,7 @@ a9_spill_shared(nir_builder *b, nir_intrinsic_instr *in, void *data) return true; } + bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_ctx) { nir->options = &nir_options_a9; @@ -249,8 +250,17 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c NIR_PASS(_, nir, nir_shader_tex_pass, a9_lower_array_layer, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_lower_compute_system_values, &csv); NIR_PASS(_, nir, nir_lower_undef_to_zero, NULL); - if (!compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx)) - return false; + /* A kernel that ran out of registers is compiled again in pressure mode + * (g9p_compile.c "pressure"). Clone before compile_back mutates nir. */ + nir_shader *again = nir_shader_clone(mem_ctx, nir); + const struct g9p_shader saved = *out; + if (!compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx, false)) { + if (!again || strncmp(out->a9_refusal, "out of registers", 16)) + return false; + *out = saved; + if (!compile_back(again, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx, true)) + return false; + } /* the fixed common-store places of texture words (g9p_a9_tex.h) */ bool sizes = false, lods = false; for (unsigned i = 0; i < out->a9_num_uploads; i++) { diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index 5473105d160..3de0abebd56 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -313,6 +313,11 @@ struct g9p_ctx { /* The decl_reg defs */ nir_def **decls; unsigned num_decls; + bool pressure; + /* Keys grouped by last use, and ownership released in this generation. */ + unsigned *dies_at, *dies_keys; + uint8_t *released, *is_decl; + unsigned emit_loop_index; /* * Fragment NIR registers parked above r31 (per decl_reg key, -1: none), @@ -320,6 +325,7 @@ struct g9p_ctx { */ int8_t *home; unsigned num_homes; + bool oom_low; /* the failed allocation needs r0-r15 */ bool oom; /* the last generate() ran out of registers */ unsigned oom_ip; @@ -498,6 +504,7 @@ alloc_reg(struct g9p_ctx *ctx) flush_loads(ctx); return alloc_reg(ctx); } + ctx->oom_low = false; dump_busy(ctx); return -1; } @@ -634,6 +641,7 @@ alloc_low_temp(struct g9p_ctx *ctx) return r; } } + ctx->oom_low = true; dump_busy(ctx); return -1; } @@ -662,6 +670,7 @@ alloc_low_range(struct g9p_ctx *ctx, unsigned n) return r; } } + ctx->oom_low = true; dump_busy(ctx); return -1; } @@ -718,6 +727,7 @@ alloc_pair(struct g9p_ctx *ctx) return r; } } + ctx->oom_low = true; dump_busy(ctx); return -1; } @@ -733,6 +743,7 @@ alloc_range(struct g9p_ctx *ctx, unsigned n) return r; } } + ctx->oom_low = false; dump_busy(ctx); return -1; } @@ -835,6 +846,8 @@ release_scalar(struct g9p_ctx *ctx, nir_def *def, unsigned comp) if (ctx->last_use[k] > ctx->ip || ctx->reg[k] < 0) return; + if (ctx->released) + ctx->released[k] = 1; /* a NIR register's last use: unpinned and free */ if (nir_def_is_intrinsic(def) && nir_def_as_intrinsic(def)->intrinsic == nir_intrinsic_decl_reg) { @@ -1110,6 +1123,160 @@ walk_liveness(struct g9p_ctx *ctx, struct exec_list *list, bool uses) } } +/* Register ranges from CFG liveness, intersected with reaching writes. SSA + * ranges stay as recorded by walk_liveness(). */ +struct reg_block_range { unsigned first, last; }; + +static void +number_reg_blocks(struct exec_list *list, unsigned *ip, struct reg_block_range *blk, unsigned *instr_ip) +{ + foreach_list_typed(nir_cf_node, node, node, list) { + switch (node->type) { + case nir_cf_node_block: { + nir_block *b = nir_cf_node_as_block(node); + blk[b->index].first = *ip + 1; + nir_foreach_instr(instr, b) { + (*ip)++; + instr_ip[instr->index] = *ip; + } + blk[b->index].last = *ip; + if (blk[b->index].first > blk[b->index].last) + blk[b->index].first = blk[b->index].last; /* empty block */ + break; + } + case nir_cf_node_if: { + nir_if *nif = nir_cf_node_as_if(node); + (*ip)++; + number_reg_blocks(&nif->then_list, ip, blk, instr_ip); + number_reg_blocks(&nif->else_list, ip, blk, instr_ip); + break; + } + case nir_cf_node_loop: + number_reg_blocks(&nir_cf_node_as_loop(node)->body, ip, blk, instr_ip); + break; + default: + break; + } + } +} + +static void +reg_liveness(struct g9p_ctx *ctx, nir_function_impl *impl, void *mem_ctx) +{ + const unsigned nd = ctx->num_decls, words = BITSET_WORDS(MAX2(nd, 1)); + unsigned *decl_of = rzalloc_array(mem_ctx, unsigned, impl->ssa_alloc); /* decl def index -> d + 1 */ + + for (unsigned d = 0; d < nd; d++) + decl_of[ctx->decls[d]->index] = d + 1; + nir_metadata_require(impl, nir_metadata_block_index); + const unsigned ni = nir_index_instrs(impl); + const unsigned nb = impl->num_blocks; + struct reg_block_range *blk = rzalloc_array(mem_ctx, struct reg_block_range, nb); + unsigned *instr_ip = rzalloc_array(mem_ctx, unsigned, MAX2(ni, 1)); /* by instr index */ + BITSET_WORD *gen = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + BITSET_WORD *kill = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + BITSET_WORD *in = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + BITSET_WORD *out = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + unsigned ip = 0; + + number_reg_blocks(&impl->body, &ip, blk, instr_ip); + nir_foreach_block(b, impl) { + BITSET_WORD *g = gen + b->index * words, *k = kill + b->index * words; + nir_foreach_instr(instr, b) { + if (instr->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + if (intr->intrinsic == nir_intrinsic_load_reg) { + const unsigned d = decl_of[intr->src[0].ssa->index] - 1; + if (!BITSET_TEST(k, d)) + BITSET_SET(g, d); + } else if (intr->intrinsic == nir_intrinsic_store_reg) { + BITSET_SET(k, decl_of[intr->src[1].ssa->index] - 1); + } + } + } + for (bool progress = true; progress;) { + progress = false; + nir_foreach_block_reverse(b, impl) { + BITSET_WORD *o = out + b->index * words, *i = in + b->index * words; + for (unsigned s = 0; s < 2; s++) { + if (!b->successors[s] || b->successors[s] == impl->end_block) + continue; + const BITSET_WORD *si = in + b->successors[s]->index * words; + for (unsigned w = 0; w < words; w++) + o[w] |= si[w]; + } + for (unsigned w = 0; w < words; w++) { + const BITSET_WORD n = gen[b->index * words + w] | (o[w] & ~kill[b->index * words + w]); + if (n != i[w]) { + i[w] = n; + progress = true; + } + } + } + } + /* forward: a write reaches the block (an undefined value needs no register: uninitialised paths) */ + BITSET_WORD *din = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + BITSET_WORD *dout = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + for (bool progress = true; progress;) { + progress = false; + nir_foreach_block(b, impl) { + BITSET_WORD *di = din + b->index * words, *dd = dout + b->index * words; + nir_foreach_pred(p, b) { + if (p->index >= nb) + continue; + for (unsigned w = 0; w < words; w++) + di[w] |= dout[p->index * words + w]; + } + for (unsigned w = 0; w < words; w++) { + const BITSET_WORD n = di[w] | kill[b->index * words + w]; + if (n != dd[w]) { + dd[w] = n; + progress = true; + } + } + } + } + for (unsigned i = 0; i < nb * words; i++) { + in[i] &= din[i]; + out[i] &= dout[i]; + } + unsigned *lo = ralloc_array(mem_ctx, unsigned, MAX2(nd, 1)), *hi = rzalloc_array(mem_ctx, unsigned, MAX2(nd, 1)); + memset(lo, 0xff, MAX2(nd, 1) * sizeof(unsigned)); +#define REG_SPAN(d, a, z) do { lo[d] = MIN2(lo[d], (a)); hi[d] = MAX2(hi[d], (z)); } while (0) + nir_foreach_block(b, impl) { + for (unsigned d = 0; d < nd; d++) { + if (BITSET_TEST(in + b->index * words, d)) + REG_SPAN(d, blk[b->index].first, blk[b->index].first); + if (BITSET_TEST(out + b->index * words, d)) + REG_SPAN(d, blk[b->index].last, blk[b->index].last); + } + nir_foreach_instr(instr, b) { + if (instr->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + const unsigned at = instr_ip[instr->index]; + if (intr->intrinsic == nir_intrinsic_load_reg) { + const unsigned d = decl_of[intr->src[0].ssa->index] - 1; + REG_SPAN(d, at, MAX2(at, ctx->last_use[key(&intr->def, 0)])); + } else if (intr->intrinsic == nir_intrinsic_store_reg) { + const unsigned d = decl_of[intr->src[1].ssa->index] - 1; + REG_SPAN(d, at, at); + } + } + } + /* in+out over a loop: a register live into a block of a loop is live over the whole block range + * between; the hull covers it. Replace the walk's stretched ranges. */ + for (unsigned d = 0; d < nd; d++) { + const unsigned k = key(ctx->decls[d], 0); + if (lo[d] == UINT_MAX) + continue; + ctx->reg_first[ctx->decls[d]->index] = lo[d]; + ctx->last_use[k] = hi[d]; + } +} +#undef REG_SPAN + /* Exactly the loop records visited by walk_liveness(), including nested * loops in either arm. Allocate before its two traversals. */ static unsigned @@ -2384,6 +2551,7 @@ emit_fb_fetch(struct g9p_ctx *ctx, nir_intrinsic_instr *intr) } } if (pair < 0) { + ctx->oom_low = false; dump_busy(ctx); return false; } @@ -3754,10 +3922,33 @@ emit_if(struct g9p_ctx *ctx, nir_if *nif) return true; } +/* A stretched range may end without an instruction reading its key. */ +static void +expire_keys(struct g9p_ctx *ctx, unsigned end, bool decls_only) +{ + if (!ctx->pressure) + return; + for (unsigned i = ctx->dies_at[end]; i < ctx->dies_at[end + 1]; i++) { + const unsigned k = ctx->dies_keys[i]; + const int r = ctx->reg[k]; + if (r < 0 || ctx->released[k] || + (decls_only && (!ctx->is_decl[k] || r >= 32))) + continue; + ctx->released[k] = 1; + if (ctx->is_decl[k]) { + ctx->pinned &= ~BITFIELD64_BIT(r); + ctx->busy &= ~BITFIELD64_BIT(r); + } else if (!(ctx->pinned & BITFIELD64_BIT(r))) { + ctx->busy &= ~BITFIELD64_BIT(r); + } + } +} + static bool emit_loop(struct g9p_ctx *ctx, nir_loop *loop) { unsigned body; + const unsigned lidx = ctx->emit_loop_index++; unsigned first_exit = ctx->num_exit_jumps; const unsigned outer_depth = ctx->if_depth; @@ -3775,6 +3966,8 @@ emit_loop(struct g9p_ctx *ctx, nir_loop *loop) 2 * (int)(ctx->c->n - ctx->exit_jumps[j])); ctx->num_exit_jumps = first_exit; g9p_pop_levels(ctx->c, ctx->st, 2); + if (ctx->pressure) + expire_keys(ctx, ctx->loops[lidx].end, false); return true; } @@ -3827,6 +4020,7 @@ emit_cf_list(struct g9p_ctx *ctx, struct exec_list *list) return false; } nir_foreach_def(instr, park_def, ctx); + expire_keys(ctx, ctx->ip, true); } break; case nir_cf_node_if: @@ -3862,6 +4056,9 @@ generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) memset(ctx->reg, -1, impl->ssa_alloc * 4); memset(ctx->uni, -1, impl->ssa_alloc * 4 * sizeof(*ctx->uni)); + if (ctx->released) + memset(ctx->released, 0, ctx->num_keys); + ctx->emit_loop_index = 0; ctx->c->n = 0; ctx->c->uses_stack = false; ctx->busy = ctx->pinned = ctx->temps = 0; @@ -4920,12 +5117,13 @@ size_of_sampled_level(nir_builder *b, nir_tex_instr *tex, void *data) static bool compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, const struct g9p_vertex_attrib *attribs, unsigned frag_coord_slot, - bool repeat_loads, bool point_size, void *mem_ctx) + bool repeat_loads, bool point_size, void *mem_ctx, bool pressure) { /* G9P_LATE_OUTPUTS: outputs held to the end unless out of registers, as before */ /* not with transform feedback: its capture reads the outputs at the end */ const bool xfb = nir->xfb_info && nir->xfb_info->buffers_written; - struct g9p_ctx ctx = { .point_size = point_size, .repeat_loads = repeat_loads, + struct g9p_ctx ctx = { .pressure = pressure && stage == MESA_SHADER_COMPUTE && out->a9, + .point_size = point_size, .repeat_loads = repeat_loads, .flush_outputs = repeat_loads || (!xfb && !getenv("G9P_LATE_OUTPUTS")) }; nir_function_impl *impl; @@ -4973,6 +5171,13 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, NULL, NULL); NIR_PASS(_, nir, nir_opt_algebraic_late); optimize(nir); + if (ctx.pressure) { + /* Fold constant shared addresses into a9_shared()'s word immediate; + * CSE otherwise keeps an iadd per inlined copy live. */ + const nir_opt_offsets_options off = { .shared_max = 16336 }; + NIR_PASS(_, nir, nir_opt_offsets, &off); + optimize(nir); + } if (stage == MESA_SHADER_FRAGMENT) { NIR_PASS(_, nir, nir_shader_intrinsics_pass, lower_dynamic_ubo_instr, nir_metadata_control_flow, NULL); @@ -5066,6 +5271,29 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.loop_index = 0; walk_liveness(&ctx, &impl->body, true); + if (ctx.pressure) { + reg_liveness(&ctx, impl, mem_ctx); + const unsigned nip = ctx.ip + 2; + unsigned total = 0; + ctx.dies_at = rzalloc_array(mem_ctx, unsigned, nip + 1); + ctx.released = rzalloc_array(mem_ctx, uint8_t, ctx.num_keys); + ctx.is_decl = rzalloc_array(mem_ctx, uint8_t, ctx.num_keys); + for (unsigned d = 0; d < ctx.num_decls; d++) + ctx.is_decl[key(ctx.decls[d], 0)] = 1; + for (unsigned k = 0; k < ctx.num_keys; k++) + if (ctx.last_use[k] && ctx.last_use[k] < nip) + ctx.dies_at[ctx.last_use[k] + 1]++, total++; + for (unsigned i = 1; i <= nip; i++) + ctx.dies_at[i] += ctx.dies_at[i - 1]; + ctx.dies_keys = ralloc_array(mem_ctx, unsigned, MAX2(total, 1)); + unsigned *fill = ralloc_array(mem_ctx, unsigned, nip); + memcpy(fill, ctx.dies_at, nip * sizeof(unsigned)); + for (unsigned k = 0; k < ctx.num_keys; k++) + if (ctx.last_use[k] && ctx.last_use[k] < nip) + ctx.dies_keys[fill[ctx.last_use[k]]++] = k; + } + + /* G9P_PRESSURE: the most scalars live at once, from the liveness above */ if (debug_get_option_g9p_pressure()) { unsigned *delta = rzalloc_array(mem_ctx, unsigned, ctx.ip + 2), *dec = rzalloc_array(mem_ctx, unsigned, ctx.ip + 2); @@ -5163,79 +5391,116 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, g9p_fail(out, "out of registers at instruction %u (parking does not help)", ctx.oom_ip); return backend_failure(out, &ctx); } - for (unsigned d = 0; d < ctx.num_decls; d++) { - const unsigned k = key(ctx.decls[d], 0); - - if (ctx.home[k] < 0 && ctx.reg_first[ctx.decls[d]->index] <= ctx.oom_ip && - ctx.last_use[k] >= ctx.oom_ip && ctx.last_use[k] > best) { - best = ctx.last_use[k]; - victim = k; - victim_decl = true; + for (unsigned batch = 0; batch < (ctx.pressure ? 4 : 1); batch++) { + best = 0; + victim = -1; + victim_decl = false; + if (ctx.pressure) { + const int lim = ctx.oom_low ? 16 : 32; + for (unsigned d = 0; d < ctx.num_decls; d++) { + const unsigned k = key(ctx.decls[d], 0); + const int r = ctx.reg[k]; + if (ctx.home[k] < 0 && r >= 0 && r < lim && + (ctx.pinned & BITFIELD64_BIT(r)) && + ctx.reg_first[ctx.decls[d]->index] <= ctx.oom_ip && + ctx.last_use[k] >= ctx.oom_ip && ctx.last_use[k] > best) { + best = ctx.last_use[k]; + victim = k; + victim_decl = true; + } + } + if (victim < 0) { + for (unsigned k = 0; k < ctx.num_keys; k++) { + const int r = ctx.reg[k]; + if (!ctx.is_decl[k] && r >= 0 && r < lim && ctx.home[k] < 0 && + ctx.last_use[k] > ctx.oom_ip && ctx.last_use[k] > best && + !((ctx.pinned | ctx.temps) & BITFIELD64_BIT(r))) { + best = ctx.last_use[k]; + victim = k; + } + } + } } - } - /* else any NIR register, even past its last use: they stay pinned (dEQP tmp_array) */ - for (unsigned d = 0; victim < 0 && d < ctx.num_decls; d++) { - const unsigned k = key(ctx.decls[d], 0); + if (victim < 0) { + for (unsigned d = 0; d < ctx.num_decls; d++) { + const unsigned k = key(ctx.decls[d], 0); + + if (ctx.home[k] < 0 && ctx.reg_first[ctx.decls[d]->index] <= ctx.oom_ip && + ctx.last_use[k] >= ctx.oom_ip && ctx.last_use[k] > best) { + best = ctx.last_use[k]; + victim = k; + victim_decl = true; + } + } + /* else any NIR register, even past its last use: they stay pinned (dEQP tmp_array) */ + for (unsigned d = 0; victim < 0 && d < ctx.num_decls; d++) { + const unsigned k = key(ctx.decls[d], 0); - if (ctx.home[k] < 0 && ctx.reg_first[ctx.decls[d]->index] <= ctx.oom_ip) { - victim = k; - victim_decl = true; - } - } - /* else the live value (not a NIR register's) with the furthest last use */ - for (unsigned k = 0, ssa_best = 0; victim < 0 && k < ctx.num_keys; k++) { - const int r = ctx.reg[k]; + if (ctx.home[k] < 0 && ctx.reg_first[ctx.decls[d]->index] <= ctx.oom_ip) { + victim = k; + victim_decl = true; + } + } + /* else the live value (not a NIR register's) with the furthest last use */ + for (unsigned k = 0, ssa_best = 0; victim < 0 && k < ctx.num_keys; k++) { + const int r = ctx.reg[k]; - if (r < 0 || r >= 32 || ctx.home[k] >= 0 || ctx.last_use[k] <= ctx.oom_ip || - (ctx.pinned | ctx.temps) & BITFIELD64_BIT(r) || ctx.last_use[k] <= ssa_best) - continue; - ssa_best = ctx.last_use[k]; - best = k + 1; - } - if (victim < 0 && best && best <= ctx.num_keys && ctx.home[best - 1] < 0 && - ctx.reg[best - 1] >= 0) - victim = best - 1; - if (victim < 0) { - g9p_fail(out, "out of registers at instruction %u (no parking victim)", ctx.oom_ip); - return backend_failure(out, &ctx); - } - { - const unsigned start = victim_decl ? ctx.reg_first[victim / 4] : ctx.def_ip[victim / 4]; - /* a NIR register's range takes in the loops it is used in (use_scalar()) */ - const unsigned end = ctx.last_use[victim]; - /* compute: at most 64 homes, bank 0 up to r95 (the DOUTU temps; residency) */ - const unsigned max_homes = - stage != MESA_SHADER_COMPUTE ? g9p_max_homes - : out->a9 ? MIN2(g9p_max_homes, g9p_a9_max_homes(MAX2(out->local_size[0], 1) * - MAX2(out->local_size[1], 1) * - MAX2(out->local_size[2], 1))) - : MIN2(g9p_max_homes, 64); - int home = -1; - - for (unsigned h = 0; home < 0 && h < max_homes; h++) { - bool free = true; - - for (unsigned i = 0; free && i < num_homed; i++) - free = homed[i].home != 32 + (int)h || homed[i].end < start || end < homed[i].start; - if (free) - home = 32 + h; + if (r < 0 || r >= 32 || ctx.home[k] >= 0 || ctx.last_use[k] <= ctx.oom_ip || + (ctx.pinned | ctx.temps) & BITFIELD64_BIT(r) || ctx.last_use[k] <= ssa_best) + continue; + ssa_best = ctx.last_use[k]; + best = k + 1; + } + if (victim < 0 && best && best <= ctx.num_keys && ctx.home[best - 1] < 0 && + ctx.reg[best - 1] >= 0) + victim = best - 1; } - if (home < 0) { - g9p_fail(out, "out of registers above r31 at instruction %u (bank-0 homes exhausted)", ctx.oom_ip); + if (victim < 0 && batch) + break; + if (victim < 0) { + g9p_fail(out, "out of registers at instruction %u (no parking victim)", ctx.oom_ip); return backend_failure(out, &ctx); } - if (num_homed == homed_capacity) { - unsigned capacity = homed_capacity * 2; - __typeof__(homed) ranges = capacity > homed_capacity ? - reralloc(mem_ctx, homed, __typeof__(*homed), capacity) : NULL; - if (!ranges) - return g9p_fail(out, "allocation failure growing parked live ranges"); - homed = ranges; - homed_capacity = capacity; + { + const unsigned start = victim_decl ? ctx.reg_first[victim / 4] : ctx.def_ip[victim / 4]; + /* a NIR register's range takes in the loops it is used in (use_scalar()) */ + const unsigned end = ctx.last_use[victim]; + /* compute: at most 64 homes, bank 0 up to r95 (the DOUTU temps; residency) */ + const unsigned max_homes = + stage != MESA_SHADER_COMPUTE ? g9p_max_homes + : out->a9 ? MIN2(g9p_max_homes, g9p_a9_max_homes(MAX2(out->local_size[0], 1) * + MAX2(out->local_size[1], 1) * + MAX2(out->local_size[2], 1))) + : MIN2(g9p_max_homes, 64); + int home = -1; + + for (unsigned h = 0; home < 0 && h < max_homes; h++) { + bool free = true; + + for (unsigned i = 0; free && i < num_homed; i++) + free = homed[i].home != 32 + (int)h || homed[i].end < start || end < homed[i].start; + if (free) + home = 32 + h; + } + if (home < 0) { + if (batch) + break; + g9p_fail(out, "out of registers above r31 at instruction %u (bank-0 homes exhausted)", ctx.oom_ip); + return backend_failure(out, &ctx); + } + if (num_homed == homed_capacity) { + unsigned capacity = homed_capacity * 2; + __typeof__(homed) ranges = capacity > homed_capacity ? + reralloc(mem_ctx, homed, __typeof__(*homed), capacity) : NULL; + if (!ranges) + return g9p_fail(out, "allocation failure growing parked live ranges"); + homed = ranges; + homed_capacity = capacity; + } + homed[num_homed++] = (__typeof__(*homed)){ start, end, (int8_t)home }; + ctx.home[victim] = home; + ctx.num_homes = MAX2(ctx.num_homes, (unsigned)home - 31); } - homed[num_homed++] = (__typeof__(*homed)){ start, end, (int8_t)home }; - ctx.home[victim] = home; - ctx.num_homes = MAX2(ctx.num_homes, (unsigned)home - 31); } } st = ctx.max_reg + 1; @@ -5393,7 +5658,7 @@ g9p_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_ctx) memset(out, 0, sizeof(*out)); lower_front(nir); NIR_PASS(_, nir, nir_lower_compute_system_values, &csv); - return compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx); + return compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx, false); } #include "g9p_a9_compute.h" /* @@ -5782,7 +6047,7 @@ g9p_compile_vertex_attempts(nir_shader *nir, const struct g9p_vertex_attrib *att nir_shader *n = nir_shader_clone(mem_ctx, attempt[i]); g9p_no_schedule = i == 2; - if (compile_back(n, out, MESA_SHADER_VERTEX, attribs, 0, i > 0, point_size, mem_ctx)) { + if (compile_back(n, out, MESA_SHADER_VERTEX, attribs, 0, i > 0, point_size, mem_ctx, false)) { g9p_no_schedule = g9p_no_homes = false; g9p_max_homes = 92; return true; @@ -6174,7 +6439,7 @@ g9p_compile_fragment(nir_shader *nir, unsigned frag_coord_slot, nir_shader *n = nir_shader_clone(mem_ctx, attempt[i]); g9p_no_schedule = i == 2; - if (compile_back(n, out, MESA_SHADER_FRAGMENT, NULL, frag_coord_slot, i > 0, false, mem_ctx)) { + if (compile_back(n, out, MESA_SHADER_FRAGMENT, NULL, frag_coord_slot, i > 0, false, mem_ctx, false)) { g9p_no_schedule = g9p_no_homes = false; g9p_max_homes = 92; return true; From b7ab4a6c5a92d6ba386515c6495bc6cf15405ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 23:53:11 +0100 Subject: [PATCH 075/109] g9p: give A9 compute Apple's register budget when the pressure retry fails Keep the original allocation and the pressure retry at their existing homes cap. Only after both run out of registers, retry pressure mode with the bank-0 budget for the actual workgroup size. Existing admitted kernels, including the 93-register ASTC encoder, keep their code. Apple's A9 compiler declares 240 temps without spilling for the 64-lane loop144 kernel, and p120 reads r238. Its per-pipeline formula allows 208 homes at 64 lanes and fewer for larger groups. Use that formula, widen home indices and carry the witnessed register extensions through the parking moves. The shared workgroup residency check remains the final guard. The r128 destination bit is witnessed in Apple's class 99 instructions; its combination with both bank-1 sources is composed from those fields, with class 9d as supporting evidence. No private spill frame is added. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_compute.h | 24 +++++++++++++++--- src/imagination/g9p/g9p_compile.c | 37 +++++++++++++++------------- src/imagination/g9p/g9p_isa.h | 14 ++++++++--- 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index c661a4be3f3..a6c3f142035 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -254,12 +254,28 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c * (g9p_compile.c "pressure"). Clone before compile_back mutates nir. */ nir_shader *again = nir_shader_clone(mem_ctx, nir); const struct g9p_shader saved = *out; - if (!compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx, false)) { - if (!again || strncmp(out->a9_refusal, "out of registers", 16)) + const bool compiled = compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, + false, false, mem_ctx, false, false); + if (!compiled) { + if (!again || strncmp(out->a9_refusal, "out of registers", 16)) { return false; + } + /* Preserve the existing pressure retry and its 64-home ceiling. Only a second + * register refusal gets Apple's larger budget, using the same pressure + * mode on a fresh clone of the prepared NIR. */ + nir_shader *budget = nir_shader_clone(mem_ctx, again); *out = saved; - if (!compile_back(again, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx, true)) - return false; + const bool retried = compile_back(again, out, MESA_SHADER_COMPUTE, NULL, 0, + false, false, mem_ctx, true, false); + if (!retried) { + if (strncmp(out->a9_refusal, "out of registers", 16)) { + return false; + } + *out = saved; + if (!compile_back(budget, out, MESA_SHADER_COMPUTE, NULL, 0, + false, false, mem_ctx, true, true)) + return false; + } } /* the fixed common-store places of texture words (g9p_a9_tex.h) */ bool sizes = false, lods = false; diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index 3de0abebd56..ea12cfeead8 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -108,13 +108,14 @@ g9p_num_regs(void) /* * A9 compute: the homes (bank-0 registers above r31) a workgroup of @threads lanes may use. Apple's bank-0 * limit for t DOUTU temps is 32 * floor((3329 - a) / a) lanes, a = (4t + 28) & ~31 (the same +0xb08 value); - * the temps are 32 + homes rounded up to 8. 256 lanes allow 88 temps (56 homes); at most 64 homes as before. + * the temps are 32 + homes rounded up to 8. Apple's A9 allocator declares up to 240 temps at 64 lanes without + * spilling (loop144 240, p120 reads r238). Homes run r32..r239, fewer as the workgroup grows (256 lanes: 56). * A kernel that needs more is refused by name here, instead of at pipeline creation. */ static unsigned g9p_a9_max_homes(unsigned threads) { - unsigned homes = 64; + unsigned homes = 208; while (homes > 0) { const unsigned t = 8 * DIV_ROUND_UP(32 + homes, 8), a = (4 * t + 28) & ~31u; if (32 * ((3329 - a) / a) >= threads) @@ -323,7 +324,7 @@ struct g9p_ctx { * Fragment NIR registers parked above r31 (per decl_reg key, -1: none), * chosen across generate() attempts when registers run out */ - int8_t *home; + int16_t *home; unsigned num_homes; bool oom_low; /* the failed allocation needs r0-r15 */ bool oom; /* the last generate() ran out of registers */ @@ -5117,10 +5118,11 @@ size_of_sampled_level(nir_builder *b, nir_tex_instr *tex, void *data) static bool compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, const struct g9p_vertex_attrib *attribs, unsigned frag_coord_slot, - bool repeat_loads, bool point_size, void *mem_ctx, bool pressure) + bool repeat_loads, bool point_size, void *mem_ctx, bool pressure, bool apple_budget) { /* G9P_LATE_OUTPUTS: outputs held to the end unless out of registers, as before */ /* not with transform feedback: its capture reads the outputs at the end */ + apple_budget &= stage == MESA_SHADER_COMPUTE && out->a9; const bool xfb = nir->xfb_info && nir->xfb_info->buffers_written; struct g9p_ctx ctx = { .pressure = pressure && stage == MESA_SHADER_COMPUTE && out->a9, .point_size = point_size, .repeat_loads = repeat_loads, @@ -5345,15 +5347,15 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, * Fragment shaders that run out of registers park the NIR register live * at that point with the furthest last use above r31 and try again. */ - ctx.home = ralloc_array(mem_ctx, int8_t, impl->ssa_alloc * 4); - memset(ctx.home, -1, impl->ssa_alloc * 4); + ctx.home = ralloc_array(mem_ctx, int16_t, impl->ssa_alloc * 4); + memset(ctx.home, -1, impl->ssa_alloc * 4 * sizeof(*ctx.home)); /* * Homes are shared: a value parks in the first one no other parked value is live in. * One home each ran out at 96 in Firefox's shaders. */ struct { unsigned start, end; - int8_t home; + int16_t home; } *homed = NULL; unsigned num_homed = 0, homed_capacity = 1024; homed = ralloc_array(mem_ctx, __typeof__(*homed), homed_capacity); @@ -5387,7 +5389,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, stuck = ctx.oom_ip == last_oom_ip ? stuck + 1 : 0; last_oom_ip = ctx.oom_ip; /* each retry parks one more value; 40 live scalars need more than 16 (drawbench fs 2) */ - if (stuck >= 64) { + if (stuck >= (apple_budget ? 208 : 64)) { g9p_fail(out, "out of registers at instruction %u (parking does not help)", ctx.oom_ip); return backend_failure(out, &ctx); } @@ -5465,17 +5467,18 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, const unsigned start = victim_decl ? ctx.reg_first[victim / 4] : ctx.def_ip[victim / 4]; /* a NIR register's range takes in the loops it is used in (use_scalar()) */ const unsigned end = ctx.last_use[victim]; - /* compute: at most 64 homes, bank 0 up to r95 (the DOUTU temps; residency) */ + /* Only the third A9 compute attempt gets Apple's full per-lane budget. */ const unsigned max_homes = stage != MESA_SHADER_COMPUTE ? g9p_max_homes - : out->a9 ? MIN2(g9p_max_homes, g9p_a9_max_homes(MAX2(out->local_size[0], 1) * - MAX2(out->local_size[1], 1) * - MAX2(out->local_size[2], 1))) + : out->a9 ? MIN2(apple_budget ? 208 : 64, + g9p_a9_max_homes(MAX2(out->local_size[0], 1) * + MAX2(out->local_size[1], 1) * + MAX2(out->local_size[2], 1))) : MIN2(g9p_max_homes, 64); int home = -1; for (unsigned h = 0; home < 0 && h < max_homes; h++) { - bool free = true; + bool free = !(out->a9 && 32 + (int)h > 239); for (unsigned i = 0; free && i < num_homed; i++) free = homed[i].home != 32 + (int)h || homed[i].end < start || end < homed[i].start; @@ -5497,7 +5500,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, homed = ranges; homed_capacity = capacity; } - homed[num_homed++] = (__typeof__(*homed)){ start, end, (int8_t)home }; + homed[num_homed++] = (__typeof__(*homed)){ start, end, (int16_t)home }; ctx.home[victim] = home; ctx.num_homes = MAX2(ctx.num_homes, (unsigned)home - 31); } @@ -5658,7 +5661,7 @@ g9p_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_ctx) memset(out, 0, sizeof(*out)); lower_front(nir); NIR_PASS(_, nir, nir_lower_compute_system_values, &csv); - return compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx, false); + return compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx, false, false); } #include "g9p_a9_compute.h" /* @@ -6047,7 +6050,7 @@ g9p_compile_vertex_attempts(nir_shader *nir, const struct g9p_vertex_attrib *att nir_shader *n = nir_shader_clone(mem_ctx, attempt[i]); g9p_no_schedule = i == 2; - if (compile_back(n, out, MESA_SHADER_VERTEX, attribs, 0, i > 0, point_size, mem_ctx, false)) { + if (compile_back(n, out, MESA_SHADER_VERTEX, attribs, 0, i > 0, point_size, mem_ctx, false, false)) { g9p_no_schedule = g9p_no_homes = false; g9p_max_homes = 92; return true; @@ -6439,7 +6442,7 @@ g9p_compile_fragment(nir_shader *nir, unsigned frag_coord_slot, nir_shader *n = nir_shader_clone(mem_ctx, attempt[i]); g9p_no_schedule = i == 2; - if (compile_back(n, out, MESA_SHADER_FRAGMENT, NULL, frag_coord_slot, i > 0, false, mem_ctx, false)) { + if (compile_back(n, out, MESA_SHADER_FRAGMENT, NULL, frag_coord_slot, i > 0, false, mem_ctx, false, false)) { g9p_no_schedule = g9p_no_homes = false; g9p_max_homes = 92; return true; diff --git a/src/imagination/g9p/g9p_isa.h b/src/imagination/g9p/g9p_isa.h index 1d2aa517ac7..9eef9e1535e 100644 --- a/src/imagination/g9p/g9p_isa.h +++ b/src/imagination/g9p/g9p_isa.h @@ -272,19 +272,25 @@ g9p_alu2(struct g9p_code *c, unsigned op, unsigned dst, unsigned a, unsigned b, * its high side in bank 0, the fragment form above (validated at r40/r77/r100 on the A9): an operand's * header bit and the destination's W1 bit 9 name each side's bank. */ +/* A9 compute (high_bank0): bank-0 homes up to r239. Above r127 the destination's half-index bit 8 is W0 bit 9 + * (header bit 1), as in Apple's A9 code (moves to r127 and r128 differ in that bit and the half index; a + * move to r218 with bank-1 sources sets it too). */ +#define G9P_W0_DST_HALF_BIT8 0x0200 /* the destination half index's bit 8 */ static inline void g9p_mov_to_high(struct g9p_code *c, unsigned hi, unsigned src) { - assert(src < 32 && hi < 128); - G9P_EMIT(c, g9p_w0(c, G9P_W0_ALU2), c->high_bank0 ? G9P_CONT | 2 * hi : g9p_dst(c, hi), + assert(src < 32 && hi < (c->high_bank0 ? 240u : 128u)); + G9P_EMIT(c, g9p_w0(c, G9P_W0_ALU2) | (c->high_bank0 && hi >= 128 ? G9P_W0_DST_HALF_BIT8 : 0), + c->high_bank0 ? G9P_CONT | ((2 * hi) & 0xff) : g9p_dst(c, hi), g9p_srcs(src, src, false), G9P_OR); } static inline void g9p_mov_from_high(struct g9p_code *c, unsigned dst, unsigned hi) { - assert(dst < 32 && hi < 128); - /* on A9 compute (high_bank0) both sources are bank 0's */ + assert(dst < 32 && hi < (c->high_bank0 ? 240u : 128u)); + /* Three-bit source extensions (hi >> 5 in the fourth word), as Apple's A9 compute reads r182; + * on A9 compute (high_bank0) both sources are bank 0's. */ G9P_EMIT(c, g9p_w0(c, G9P_W0_ALU2) & ~(c->high_bank0 ? G9P_W0_SRC_A_BANK1 | G9P_W0_SRC_B_BANK1 : 0), g9p_dst(c, dst), g9p_srcs(hi & 31, hi & 31, false) | G9P_SRCS_EXT, G9P_CONT | (hi >> 5) << G9P_EXT_A_SHIFT | (hi >> 5) << G9P_EXT_B_SHIFT, G9P_OR); From a4d6841c0977a27ff3815183d04ce63cd18a7f4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:22:30 +0100 Subject: [PATCH 076/109] pvr/g5p: bound barriered compute workgroups by Apple's A9 residency limit GravityMark's 256-lane irradiance kernel uses 70 temporaries per lane. The generic Rogue residency check limits that workgroup to 64 and rejects it even though Apple's A9 allocation limit allows 320 lanes. Use Apple's per-pipeline limit as the A9 compute residency bound and share its calculation with offline pipeline replay. The limit comes from the A9 Metal compute pipeline's maximum threadgroup size and matches the 384-lane limit reported on iOS for a kernel with 64 bank-0 and 8 bank-1 registers. Keep the existing allocation refusal and shader code generation unchanged. Assisted-by: LLM --- src/imagination/g9p/g9p_a9.c | 25 +++++++++++++++ src/imagination/g9p/g9p_a9.h | 1 + src/imagination/g9p/meson.build | 2 +- src/imagination/vulkan/pvr_arch_g5p_compute.c | 32 ++----------------- 4 files changed, 29 insertions(+), 31 deletions(-) create mode 100644 src/imagination/g9p/g9p_a9.c diff --git a/src/imagination/g9p/g9p_a9.c b/src/imagination/g9p/g9p_a9.c new file mode 100644 index 00000000000..1aba87a8f5d --- /dev/null +++ b/src/imagination/g9p/g9p_a9.c @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: MIT */ +#include "g9p_compile.h" + +/* AGXMetalA9 ComputeProgramVariant -> +0xb08, returned by + * -[AGXA9FamilyComputePipeline maxTotalThreadsPerThreadgroup]. The allocation + * uses bank 0's DOUTU temps and bank 1's kernel-record register field. Both + * banks keep one task (32 lanes) free. With t = 64 and f = 8 this gives 384, + * matching k_hi48 on iOS. + */ +unsigned +g9p_a9_pipeline_max_lanes(const struct g9p_shader *shader) +{ + const unsigned t = 8 * DIV_ROUND_UP(MAX2(shader->num_regs, 1), 8); + const unsigned f = 8 * DIV_ROUND_UP(shader->a9_bank1_regs ? shader->a9_bank1_regs : shader->num_regs, 8); + const unsigned a = (4 * t + 28) & ~31u; + const unsigned b = (64 * f + 448) & ~511u; + unsigned max = MIN2(a ? 32 * ((3329 - a) / a) : 512, + b ? 393216 / b - 32 : 512); + + max = MIN2(max, 512); + if (max > 32) + max &= ~31u; + + return max; +} diff --git a/src/imagination/g9p/g9p_a9.h b/src/imagination/g9p/g9p_a9.h index d7b27d97d04..25d551a2c7f 100644 --- a/src/imagination/g9p/g9p_a9.h +++ b/src/imagination/g9p/g9p_a9.h @@ -32,5 +32,6 @@ struct g9p_a9_upload { } while (0) const char *g9p_a9_last_refusal(void); struct g9p_shader; +unsigned g9p_a9_pipeline_max_lanes(const struct g9p_shader *shader); bool g9p_a9_compile_compute(nir_shader *, struct g9p_shader *, void *); #endif diff --git a/src/imagination/g9p/meson.build b/src/imagination/g9p/meson.build index 48fbe3f8fb0..56aa7feb923 100644 --- a/src/imagination/g9p/meson.build +++ b/src/imagination/g9p/meson.build @@ -5,7 +5,7 @@ inc_powervr_g9p = include_directories(['.']) libpowervr_g9p = static_library( 'powervr_g9p', - files('g9p_compile.c'), + files('g9p_compile.c', 'g9p_a9.c'), include_directories : [inc_imagination, inc_powervr_g9p, inc_include, inc_src], c_args : [imagination_c_args, no_override_init_args], dependencies : [idep_mesautil, idep_nir], diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 58ea064c830..ad35334e7a3 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -30,7 +30,6 @@ #include "pvr_g5p.h" #include "pvr_g5p_tex_encode.h" #include "pvr_g5p_shader_dump.h" -#include "hwdef/rogue_hw_utils.h" #include "pvr_g5p_compute.h" #include "pvr_job_context.h" #include "pvr_winsys.h" @@ -97,40 +96,13 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( } else { pvr_g5p_shader_dump(&create_info->stage, create_info->layout, "comp", NULL, 0, "ADMIT", NULL); } - /* A barrier above 32 lanes waits for every task of the workgroup - * (Apple's software barrier, g5p_barrier.h), so all of them must be - * resident in one USC at once: Rogue's bound on the temporaries per lane - * for that workgroup size (rogue_max_wg_temps(), as upstream's compute - * pipelines check). A pipeline above it would spin forever: refused. */ { const unsigned threads = shader.local_size[0] * shader.local_size[1] * shader.local_size[2]; - const struct pvr_device_info *dev_info = &device->pdevice->dev_info; - if (shader.a9_has_barriers && threads > ROGUE_MAX_INSTANCES_PER_TASK && - rogue_max_wg_temps(dev_info, shader.num_regs, threads, true) < shader.num_regs) { - mesa_logw("G5P refuses: a %u-lane workgroup with barriers needs %u temporaries per lane, above " - "the %u that keep every task resident", threads, shader.num_regs, - rogue_max_wg_temps(dev_info, shader.num_regs, threads, true)); - result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, - "G5P compute: %u-lane barriered workgroup needs %u temporaries per lane", threads, - shader.num_regs); - goto out; - } - /* Apple's per-pipeline limit (AGXMetalA9 ComputeProgramVariant 0x207d2c4b0 -> +0xb08, which - * -[AGXA9FamilyComputePipeline maxTotalThreadsPerThreadgroup] returns), from the allocation the - * kernel task sends: bank 0 (the DOUTU temps, t) and bank 1 (the record's register field, f - * registers). min(32 * floor((3329 - a) / a) with a = (4t + 28) & ~31, 393216 / ((64f + 448) & ~511)), - * at most 512, a multiple of 32 above 32. k_hi48 (t 64, f 8) gives 384, the limit Metal reported for it - * on iOS. Vulkan has no per-pipeline limit, so a larger workgroup is refused by - * name: the USC cannot hold it. */ + /* Apple's limit keeps all tasks of a barriered workgroup resident. */ if (shader.a9) { const unsigned t = 8 * DIV_ROUND_UP(MAX2(shader.num_regs, 1), 8); const unsigned f = 8 * DIV_ROUND_UP(shader.a9_bank1_regs ? shader.a9_bank1_regs : shader.num_regs, 8); - /* bank 1 keeps one task (32 lanes) free, as Apple's bank-0 term does (g9p_a9_bank1_limit()) */ - const unsigned a = (4 * t + 28) & ~31u, b = (64 * f + 448) & ~511u; - unsigned max = MIN2(a ? 32 * ((3329 - a) / a) : 512, b ? 393216 / b - 32 : 512); - max = MIN2(max, 512); - if (max > 32) - max &= ~31u; + const unsigned max = g9p_a9_pipeline_max_lanes(&shader); if (threads > max) { mesa_logw("G5P refuses: a %u-lane workgroup with %u bank-0 and %u bank-1 registers per lane, above " "the %u lanes Apple's A9 driver allows for that allocation", threads, t, f, max); From f8c5f72c24f2ef6e98eb38bbcded596161578501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:34:40 +0100 Subject: [PATCH 077/109] pvr/g5p: decide compute workgroup residency by device feature Assisted-by: LLM --- src/imagination/common/device_info/apple-a9.h | 17 +++++++ src/imagination/common/pvr_compute_limits.h | 46 +++++++++++++++++++ src/imagination/common/pvr_device_info.h | 19 ++++++++ src/imagination/g9p/g9p_a9.c | 25 ---------- src/imagination/g9p/g9p_a9.h | 1 - src/imagination/g9p/meson.build | 2 +- src/imagination/vulkan/pvr_arch_g5p_compute.c | 25 +++++----- 7 files changed, 97 insertions(+), 38 deletions(-) create mode 100644 src/imagination/common/pvr_compute_limits.h delete mode 100644 src/imagination/g9p/g9p_a9.c diff --git a/src/imagination/common/device_info/apple-a9.h b/src/imagination/common/device_info/apple-a9.h index 1bd1bb2d47d..6627b624ed8 100644 --- a/src/imagination/common/device_info/apple-a9.h +++ b/src/imagination/common/device_info/apple-a9.h @@ -30,6 +30,23 @@ static const struct pvr_device_features pvr_device_features_9_V_6_2 = { .has_apple_g5p = true, + .has_apple_compute_dispatch = true, + /* AGXMetalA9 ComputeProgramVariant -> +0xb08 (maximum threadgroup size). + * Bank 0 uses DOUTU temps; bank 1 uses the kernel-record allocation. + */ + .apple_compute_dispatch = { + .bank0_budget = 3329U, + .bank1_budget = 393216U, + .register_granularity = 8U, + .bank0_scale = 4U, + .bank0_rounding_bias = 28U, + .bank0_granularity = 32U, + .bank1_scale = 64U, + .bank1_rounding_bias = 448U, + .bank1_granularity = 512U, + .task_size = 32U, + .max_workgroup_size = 512U, + }, .has_num_clusters = true, /* A9 */ .has_tile_size_x = true, /* A9 */ .has_tile_size_y = true, /* A9 */ diff --git a/src/imagination/common/pvr_compute_limits.h b/src/imagination/common/pvr_compute_limits.h new file mode 100644 index 00000000000..a9ad3b991e8 --- /dev/null +++ b/src/imagination/common/pvr_compute_limits.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef PVR_COMPUTE_LIMITS_H +#define PVR_COMPUTE_LIMITS_H + +#include "hwdef/rogue_hw_utils.h" + +/* Check residency, independently of the API's workgroup size limits. + * max_lanes reports the largest resident workgroup for this allocation. + */ +static inline bool +pvr_compute_wg_residency_ok(const struct pvr_device_info *dev_info, + unsigned regs, + unsigned bank1_regs, + unsigned wg_size, + bool has_barriers, + unsigned *max_lanes) +{ + if (PVR_HAS_FEATURE(dev_info, apple_compute_dispatch)) { + const struct pvr_apple_compute_dispatch_limits *limits = + &dev_info->features.apple_compute_dispatch; + const unsigned t = ALIGN_POT(MAX2(regs, 1), limits->register_granularity); + const unsigned f = ALIGN_POT(bank1_regs ? bank1_regs : regs, + limits->register_granularity); + const unsigned a = (limits->bank0_scale * t + limits->bank0_rounding_bias) & + ~(limits->bank0_granularity - 1); + const unsigned b = (limits->bank1_scale * f + limits->bank1_rounding_bias) & + ~(limits->bank1_granularity - 1); + unsigned max = MIN2(a ? limits->task_size * ((limits->bank0_budget - a) / a) : + limits->max_workgroup_size, + b ? limits->bank1_budget / b - limits->task_size : + limits->max_workgroup_size); + + /* Apple reserves one task in each bank. This matches Metal's + * maxTotalThreadsPerThreadgroup, including for barriered workgroups. + */ + max = MIN2(max, limits->max_workgroup_size); + if (max > limits->task_size) + max &= ~(limits->task_size - 1); + *max_lanes = max; + return wg_size <= max; + } + + UNREACHABLE("compute residency requires Apple dispatch limits"); +} + +#endif /* PVR_COMPUTE_LIMITS_H */ diff --git a/src/imagination/common/pvr_device_info.h b/src/imagination/common/pvr_device_info.h index 9731be888e4..ea6b3471f27 100644 --- a/src/imagination/common/pvr_device_info.h +++ b/src/imagination/common/pvr_device_info.h @@ -253,6 +253,21 @@ struct pvr_device_ident { const char *public_name; }; +/* Per-pipeline residency parameters for Apple's compute dispatch. */ +struct pvr_apple_compute_dispatch_limits { + uint32_t bank0_budget; + uint32_t bank1_budget; + uint32_t register_granularity; + uint32_t bank0_scale; + uint32_t bank0_rounding_bias; + uint32_t bank0_granularity; + uint32_t bank1_scale; + uint32_t bank1_rounding_bias; + uint32_t bank1_granularity; + uint32_t task_size; + uint32_t max_workgroup_size; +}; + struct pvr_device_features { /* * Apple A-chip GPUs (G9P on the A10): Rogue-like data masters, PDS and @@ -360,6 +375,10 @@ struct pvr_device_features { * other field's position. */ bool has_apple_g5p : 1; + + /* Only set where Apple's compute residency parameters are established. */ + bool has_apple_compute_dispatch : 1; + struct pvr_apple_compute_dispatch_limits apple_compute_dispatch; }; struct pvr_device_enhancements { diff --git a/src/imagination/g9p/g9p_a9.c b/src/imagination/g9p/g9p_a9.c deleted file mode 100644 index 1aba87a8f5d..00000000000 --- a/src/imagination/g9p/g9p_a9.c +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#include "g9p_compile.h" - -/* AGXMetalA9 ComputeProgramVariant -> +0xb08, returned by - * -[AGXA9FamilyComputePipeline maxTotalThreadsPerThreadgroup]. The allocation - * uses bank 0's DOUTU temps and bank 1's kernel-record register field. Both - * banks keep one task (32 lanes) free. With t = 64 and f = 8 this gives 384, - * matching k_hi48 on iOS. - */ -unsigned -g9p_a9_pipeline_max_lanes(const struct g9p_shader *shader) -{ - const unsigned t = 8 * DIV_ROUND_UP(MAX2(shader->num_regs, 1), 8); - const unsigned f = 8 * DIV_ROUND_UP(shader->a9_bank1_regs ? shader->a9_bank1_regs : shader->num_regs, 8); - const unsigned a = (4 * t + 28) & ~31u; - const unsigned b = (64 * f + 448) & ~511u; - unsigned max = MIN2(a ? 32 * ((3329 - a) / a) : 512, - b ? 393216 / b - 32 : 512); - - max = MIN2(max, 512); - if (max > 32) - max &= ~31u; - - return max; -} diff --git a/src/imagination/g9p/g9p_a9.h b/src/imagination/g9p/g9p_a9.h index 25d551a2c7f..d7b27d97d04 100644 --- a/src/imagination/g9p/g9p_a9.h +++ b/src/imagination/g9p/g9p_a9.h @@ -32,6 +32,5 @@ struct g9p_a9_upload { } while (0) const char *g9p_a9_last_refusal(void); struct g9p_shader; -unsigned g9p_a9_pipeline_max_lanes(const struct g9p_shader *shader); bool g9p_a9_compile_compute(nir_shader *, struct g9p_shader *, void *); #endif diff --git a/src/imagination/g9p/meson.build b/src/imagination/g9p/meson.build index 56aa7feb923..48fbe3f8fb0 100644 --- a/src/imagination/g9p/meson.build +++ b/src/imagination/g9p/meson.build @@ -5,7 +5,7 @@ inc_powervr_g9p = include_directories(['.']) libpowervr_g9p = static_library( 'powervr_g9p', - files('g9p_compile.c', 'g9p_a9.c'), + files('g9p_compile.c'), include_directories : [inc_imagination, inc_powervr_g9p, inc_include, inc_src], c_args : [imagination_c_args, no_override_init_args], dependencies : [idep_mesautil, idep_nir], diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index ad35334e7a3..736d51bc72a 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -18,6 +18,7 @@ #include "g5p_usc.h" #include "g5p_end_barrier.h" #include "pvr_bo.h" +#include "pvr_compute_limits.h" #include "pvr_cmd_buffer.h" #include "pvr_csb.h" #include "pds/pvr_rogue_pds_defs.h" @@ -98,19 +99,21 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( } { const unsigned threads = shader.local_size[0] * shader.local_size[1] * shader.local_size[2]; - /* Apple's limit keeps all tasks of a barriered workgroup resident. */ - if (shader.a9) { + unsigned max; + if (!pvr_compute_wg_residency_ok(&device->pdevice->dev_info, + shader.num_regs, + shader.a9_bank1_regs, + threads, + shader.a9_has_barriers, + &max)) { const unsigned t = 8 * DIV_ROUND_UP(MAX2(shader.num_regs, 1), 8); const unsigned f = 8 * DIV_ROUND_UP(shader.a9_bank1_regs ? shader.a9_bank1_regs : shader.num_regs, 8); - const unsigned max = g9p_a9_pipeline_max_lanes(&shader); - if (threads > max) { - mesa_logw("G5P refuses: a %u-lane workgroup with %u bank-0 and %u bank-1 registers per lane, above " - "the %u lanes Apple's A9 driver allows for that allocation", threads, t, f, max); - result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, - "G5P compute: %u-lane workgroup above the %u-lane limit of its allocation", threads, - max); - goto out; - } + mesa_logw("G5P refuses: a %u-lane workgroup with %u bank-0 and %u bank-1 registers per lane, above " + "the %u lanes Apple's A9 driver allows for that allocation", threads, t, f, max); + result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, + "G5P compute: %u-lane workgroup above the %u-lane limit of its allocation", threads, + max); + goto out; } } From c69640289ca54f7e38a17b40f94f9566edc4d49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:39:04 +0100 Subject: [PATCH 078/109] g9p: defer A9 compute refusal logs until the final verdict A pressure retry can admit a kernel after the first register allocation fails. Retain the diagnostic for retry decisions but log a refusal only when compilation ultimately fails. No allocation or code generation changes. Assisted-by: LLM --- src/imagination/g9p/g9p_a9_compute.h | 13 ++++++++++--- src/imagination/g9p/g9p_compile.c | 4 +--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index a6c3f142035..9444c726bcc 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -254,10 +254,14 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c * (g9p_compile.c "pressure"). Clone before compile_back mutates nir. */ nir_shader *again = nir_shader_clone(mem_ctx, nir); const struct g9p_shader saved = *out; + /* The first attempt is provisional: an admitted retry is not a refusal. */ + const bool compiled = compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx, false, false); + if (!compiled) { if (!again || strncmp(out->a9_refusal, "out of registers", 16)) { + return false; } /* Preserve the existing pressure retry and its 64-home ceiling. Only a second @@ -265,11 +269,14 @@ bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_c * mode on a fresh clone of the prepared NIR. */ nir_shader *budget = nir_shader_clone(mem_ctx, again); *out = saved; - const bool retried = compile_back(again, out, MESA_SHADER_COMPUTE, NULL, 0, + + const bool retried = compile_back(again, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx, true, false); - if (!retried) { + + if (!retried) { if (strncmp(out->a9_refusal, "out of registers", 16)) { - return false; + + return false; } *out = saved; if (!compile_back(budget, out, MESA_SHADER_COMPUTE, NULL, 0, diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index ea12cfeead8..2513097ac6c 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -69,9 +69,7 @@ g9p_fail(struct g9p_shader *out, const char *fmt, ...) snprintf(out->a9_refusal, sizeof(out->a9_refusal), "%s", why); snprintf(a9_last_refusal, sizeof(a9_last_refusal), "%s", out->a9_refusal); } - if (out->a9) { - mesa_logw("G5P refuses: %s", why); - } else { + if (!out->a9) { /* Preserve main's non-A9 diagnostics. */ if (!strcmp(why, "shader has no entry point")) return false; From 4742332444c68de10dcd197318de33bd41b49c44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 23:41:58 +0100 Subject: [PATCH 079/109] pvr: bind dynamic buffers in compute dispatches from their bound state Dynamic uniform and storage buffers have no descriptor in set memory: their layout binding offset is ~0 and the buffer lives in set->dynamic_buffers with the offset vkCmdBindDescriptorSets gave it. The A9 (G5P) and A10 (G9P) compute bind tasks DMA every buffer slot from set->dev_addr + binding->offset, so a dynamic buffer's slot was read from set->dev_addr + 0xffffffff. For a set holding only a dynamic buffer (dev_addr 0) that is the GPU MMU fault GravityMark's first compute job hit on the A9 (read at 0xffffffc0, requestor MCU_TEXAS0), and the dynamic offset was never applied. Add a shared helper that gives the two dwords such a slot would read (the address with the dynamic offset applied, or the size word for arrayLength()) and upload them with the dispatch's other per-dispatch values in both compute paths. Bindings with descriptor memory are unchanged. The compute log also lists dynamic-only sets now. Compiled code is unchanged. Assisted-by: LLM --- src/imagination/vulkan/pvr_arch_g5p_compute.c | 24 +++++- src/imagination/vulkan/pvr_arch_g9p.c | 14 +++- src/imagination/vulkan/pvr_descriptor_set.h | 74 +++++++++++++++++++ 3 files changed, 108 insertions(+), 4 deletions(-) diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 736d51bc72a..d96550fc5a6 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -198,12 +198,14 @@ g5p_record_dispatch(struct pvr_cmd_buffer *cmd, struct pvr_sub_cmd_compute *sub, const char *sep = ""; for (unsigned si = 0; si < PVR_MAX_DESCRIPTOR_SETS; si++) { const struct pvr_descriptor_set *set = desc->sets[si]; - if (!set || !set->layout || !set->mapping) continue; + /* a set of dynamic buffers only has no descriptor memory (no mapping) */ + if (!set || !set->layout) continue; for (unsigned bi = 0; bi < set->layout->binding_count; bi++) { const struct pvr_descriptor_set_layout_binding *b = &set->layout->bindings[bi]; if (!(b->stage_flags & VK_SHADER_STAGE_COMPUTE_BIT)) continue; + if (!set->mapping && !pvr_descriptor_set_binding_is_dynamic(b)) continue; for (unsigned e = 0; e < b->descriptor_count; e++) { - const uint8_t *d = (uint8_t *)set->mapping + b->offset + e * b->stride; + const uint8_t *d = set->mapping ? (uint8_t *)set->mapping + b->offset + e * b->stride : NULL; switch (b->type) { case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: @@ -293,6 +295,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uint32_t bind_data[4 * (G9P_MAX_BUFFERS + 1) + 4 + 3 * G9P_A9_MAX_UPLOADS] = { 0 }; uint32_t bind_code[G9P_MAX_BUFFERS + 5 + G9P_A9_MAX_UPLOADS]; uint32_t kernel_data[12] = { 0 }, kernel_code[8]; + bool from_uniforms[G9P_MAX_BUFFERS] = { false }; struct pvr_pds_upload bind, kernel; unsigned bind_dwords, ctrl0, nc = 0; VkResult result; @@ -394,6 +397,21 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uniforms[2 * k] = pc->dev_addr.addr; uniforms[2 * k + 1] = pc->dev_addr.addr >> 32; } + if (state->buffers[k].set < PVR_MAX_DESCRIPTOR_SETS) { + /* A dynamic buffer has no descriptor in set memory (its binding + * offset is ~0: a DMA from there reads set->dev_addr + 0xffffffff). + * Its address with the bound dynamic offset, or its range for + * arrayLength(), goes through this upload. */ + const unsigned element = (state->buffers[k].binding & ~G9P_BINDING_SIZE) >> G9P_BINDING_ELEMENT_SHIFT; + const unsigned b = state->buffers[k].binding & BITFIELD_MASK(G9P_BINDING_ELEMENT_SHIFT); + + if (!pvr_descriptor_set_dynamic_slot(desc->sets[state->buffers[k].set], b, element, + state->buffers[k].binding & G9P_BINDING_SIZE, + &uniforms[2 * k], &from_uniforms[k])) + return vk_errorf(cmd_buffer, VK_ERROR_UNKNOWN, + "G5P compute: set %u binding %u element %u holds no bound buffer", + state->buffers[k].set, b, element); + } } unsigned images=state->a9_num_images; if(images && n>15)return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P image LOD common31 overlaps the16th buffer pointer"); @@ -480,7 +498,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, src = uniforms_bo->dev_addr.addr + 4 * (tex0 + 8 * i); dest = state->a9_uploads[i].common; dwords = state->a9_uploads[i].dwords; - } else if (k < n && state->buffers[k].set != G9P_PUSH_CONSTANTS && + } else if (k < n && !from_uniforms[k] && state->buffers[k].set != G9P_PUSH_CONSTANTS && state->buffers[k].set != G9P_NUM_WORKGROUPS && state->buffers[k].set != G9P_SHARED_SPILL) { const struct pvr_descriptor_set *set = desc->sets[state->buffers[k].set]; diff --git a/src/imagination/vulkan/pvr_arch_g9p.c b/src/imagination/vulkan/pvr_arch_g9p.c index b8b22d8b7fd..dd9f04ea3da 100644 --- a/src/imagination/vulkan/pvr_arch_g9p.c +++ b/src/imagination/vulkan/pvr_arch_g9p.c @@ -197,6 +197,7 @@ PVR_PER_ARCH(g9p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uint32_t bind_data[4 * (G9P_MAX_BUFFERS + 1) + 4] = { 0 }; uint32_t bind_code[G9P_MAX_BUFFERS + 2]; uint32_t kernel_data[G9P_KERNEL_DATA_DWORDS] = { 0 }, kernel_code[8]; + bool from_uniforms[G9P_MAX_BUFFERS] = { false }; struct pvr_pds_upload bind, kernel; unsigned bind_dwords, ctrl0, nc = 0; VkResult result; @@ -218,6 +219,17 @@ PVR_PER_ARCH(g9p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uniforms[2 * k] = pc->dev_addr.addr; uniforms[2 * k + 1] = pc->dev_addr.addr >> 32; } + if (state->buffers[k].set < PVR_MAX_DESCRIPTOR_SETS) { + /* a dynamic buffer has no descriptor in set memory (binding offset ~0) */ + const unsigned element = + (state->buffers[k].binding & ~G9P_BINDING_SIZE) >> G9P_BINDING_ELEMENT_SHIFT; + const unsigned b = state->buffers[k].binding & BITFIELD_MASK(G9P_BINDING_ELEMENT_SHIFT); + + if (!pvr_descriptor_set_dynamic_slot(desc->sets[state->buffers[k].set], b, element, + state->buffers[k].binding & G9P_BINDING_SIZE, + &uniforms[2 * k], &from_uniforms[k])) + return vk_error(cmd_buffer, VK_ERROR_UNKNOWN); + } } uniforms[2 * n] = count; result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, uniforms, @@ -236,7 +248,7 @@ PVR_PER_ARCH(g9p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uint64_t src; uint32_t dest = 2 * k, dwords = 2; - if (k < n && state->buffers[k].set != G9P_PUSH_CONSTANTS && + if (k < n && !from_uniforms[k] && state->buffers[k].set != G9P_PUSH_CONSTANTS && state->buffers[k].set != G9P_NUM_WORKGROUPS) { const struct pvr_descriptor_set *set = desc->sets[state->buffers[k].set]; /* an array's element (Zink keeps all storage buffers in one array) */ diff --git a/src/imagination/vulkan/pvr_descriptor_set.h b/src/imagination/vulkan/pvr_descriptor_set.h index 255eeb03be6..4bb2cf56ef7 100644 --- a/src/imagination/vulkan/pvr_descriptor_set.h +++ b/src/imagination/vulkan/pvr_descriptor_set.h @@ -18,6 +18,7 @@ #include "util/vma.h" #include "vk_descriptor_set_layout.h" +#include "vk_util.h" #include "pvr_common.h" #include "pvr_macros.h" @@ -85,6 +86,79 @@ struct pvr_descriptor_set { struct pvr_buffer_descriptor dynamic_buffers[]; }; +/* Dynamic uniform and storage buffers have no descriptor memory in the set: + * their binding offset is ~0, and the buffer lives in set->dynamic_buffers[] + * with the offset vkCmdBindDescriptorSets gave it. A path that reads buffer + * descriptors from the set's device memory (a DMA from set->dev_addr) must + * take these from here instead. + */ +static inline bool +pvr_descriptor_set_binding_is_dynamic( + const struct pvr_descriptor_set_layout_binding *binding) +{ + return vk_descriptor_type_is_dynamic(binding->type); +} + +/* The bound buffer of a dynamic binding's element: its address with the + * dynamic offset applied, and its range from there. Returns false when the + * element was never written (no buffer). + */ +static inline bool +pvr_descriptor_set_dynamic_buffer( + const struct pvr_descriptor_set *set, + const struct pvr_descriptor_set_layout_binding *binding, + unsigned element, + uint64_t *addr, + uint32_t *range) +{ + const struct pvr_buffer_descriptor *d = + &set->dynamic_buffers[binding->dynamic_buffer_idx + element]; + + if (!d->addr) + return false; + + *addr = d->addr + d->offset; + *range = d->size; + return true; +} + +/* For a dispatch that DMAs a buffer slot's two dwords from descriptor + * memory (the A9 and A10 compute bind tasks): when the slot's binding is + * dynamic, the dwords it would read (the address, or the size word and the + * zero after it), to be uploaded instead. *dynamic is false for a binding + * that has descriptor memory. Fails for an unbound set, an element past the + * binding or an unwritten dynamic buffer. + */ +static inline bool +pvr_descriptor_set_dynamic_slot(const struct pvr_descriptor_set *set, + unsigned binding, + unsigned element, + bool size, + uint32_t out[static 2], + bool *dynamic) +{ + *dynamic = false; + if (!set || !set->layout || binding >= set->layout->binding_count) + return false; + + const struct pvr_descriptor_set_layout_binding *lb = + &set->layout->bindings[binding]; + if (!pvr_descriptor_set_binding_is_dynamic(lb)) + return true; + if (element >= MAX2(lb->descriptor_count, 1)) + return false; + + uint64_t addr; + uint32_t range; + if (!pvr_descriptor_set_dynamic_buffer(set, lb, element, &addr, &range)) + return false; + + out[0] = size ? range : addr; + out[1] = size ? 0 : addr >> 32; + *dynamic = true; + return true; +} + struct pvr_push_descriptor_set { struct pvr_descriptor_set_layout *layout; uint8_t data[PVR_MAX_PUSH_DESCRIPTORS * PVR_MAX_DESCRIPTOR_SIZE]; /* CPU-side descriptor blob */ From 2a5bf5c492ac2cf931e0dcf5ebd0f59723a58835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 12:31:46 +0100 Subject: [PATCH 080/109] pvr/g5p: keep 4x for input-attachment usage pvr_g5p_msaa_image_samples() dropped an image format to 1x whenever the usage included VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT. Vulkan bounds sampleCounts by the framebuffer, sampled-image and storage-image limits of the usage; input attachments have no limit of their own. The 4x layout of an input attachment is the same as that of any 4x colour or depth/stencil attachment, and multisample subpass reads stay refused by the compiler. Zink requests INPUT_ATTACHMENT for every render target, GL renderbuffers included, so even a usage candidate without SAMPLED got no 4x for a depth/stencil renderbuffer. Assisted-by: LLM --- src/imagination/vulkan/pvr_g5p_image.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/imagination/vulkan/pvr_g5p_image.c b/src/imagination/vulkan/pvr_g5p_image.c index 7b162f12c26..78a04852c15 100644 --- a/src/imagination/vulkan/pvr_g5p_image.c +++ b/src/imagination/vulkan/pvr_g5p_image.c @@ -227,21 +227,23 @@ pvr_g5p_msaa_limits(struct pvr_physical_device *pdevice, struct vk_properties *p /* * An image format's sample counts: 4 as well only for what a 4x pass * renders (our ICD's rule): a 2D optimal colour attachment of 4-byte - * samples, or one of the depth/stencil formats the ZLS keeps on chip, used - * neither as a sampled, storage nor input attachment image. Upstream's Rogue answer is every count up to 8. + * samples, or one of the depth/stencil formats the ZLS keeps on chip, not + * used as a storage image, nor as a sampled depth/stencil image. Upstream's Rogue answer is every count up to 8. */ void pvr_g5p_msaa_image_samples(struct pvr_physical_device *pdevice, const VkPhysicalDeviceImageFormatInfo2 *info, VkImageTiling tiling, VkImageFormatProperties *props) { const VkImageUsageFlags attach = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT; - const VkImageUsageFlags read = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT | - VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT; + /* Vulkan bounds sampleCounts by the framebuffer, sampled and storage limits of the usage; input + * attachments have no limit of their own and do not change the 4x layout. Multisample subpass + * reads stay refused by the compiler (tex_shape_refusal()). */ + const VkImageUsageFlags read = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT; const bool format_ok = pvr_g5p_ds_format(info->format) || (vk_format_is_color(info->format) && !vk_format_is_compressed(info->format) && !vk_format_is_int(info->format) && vk_format_get_blocksize(info->format) == 4); /* a 4-sample colour attachment may also be sampled (its view is the 2Wx2H twiddled texture of - * its samples, pvr_arch_g5p_tex.c); depth/stencil, storage and input attachments stay 1x */ + * its samples, pvr_arch_g5p_tex.c); sampled depth/stencil and storage images stay 1x */ const VkImageUsageFlags colour_read = vk_format_is_color(info->format) ? VK_IMAGE_USAGE_SAMPLED_BIT : 0; const bool four = tiling == VK_IMAGE_TILING_OPTIMAL && info->type == VK_IMAGE_TYPE_2D && !(info->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) && From 8fa588f0db39d5e29797f0ed3834652a2f11f2d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 00:01:03 +0100 Subject: [PATCH 081/109] pvr: log buffer object allocations with PVR_DEBUG=bo_log Track application and driver allocations, their bindings and image padding in the shared driver. Report the live set on allocation failures, peak steps, device teardown and process exit. Keep process-wide winsys totals separate for created and display buffers so kernel charges can be compared. Assisted-by: LLM --- src/imagination/common/pvr_debug.c | 2 + src/imagination/common/pvr_debug.h | 1 + src/imagination/vulkan/meson.build | 1 + src/imagination/vulkan/pvr_arch_device.c | 11 + src/imagination/vulkan/pvr_bo.c | 27 +- src/imagination/vulkan/pvr_bo.h | 15 +- src/imagination/vulkan/pvr_bo_log.c | 624 ++++++++++++++++++ src/imagination/vulkan/pvr_bo_log.h | 40 ++ src/imagination/vulkan/pvr_device.c | 38 +- src/imagination/vulkan/pvr_device.h | 1 + src/imagination/vulkan/pvr_image.c | 8 + .../vulkan/winsys/powervr/pvr_drm_bo.c | 12 + .../vulkan/winsys/powervr/pvr_drm_bo.h | 2 + src/imagination/vulkan/winsys/pvr_winsys.c | 16 + src/imagination/vulkan/winsys/pvr_winsys.h | 3 + 15 files changed, 786 insertions(+), 15 deletions(-) create mode 100644 src/imagination/vulkan/pvr_bo_log.c create mode 100644 src/imagination/vulkan/pvr_bo_log.h diff --git a/src/imagination/common/pvr_debug.c b/src/imagination/common/pvr_debug.c index ecdb55e777f..5e97f7cda14 100644 --- a/src/imagination/common/pvr_debug.c +++ b/src/imagination/common/pvr_debug.c @@ -42,6 +42,8 @@ static const struct debug_named_value debug_control[] = { { "capture_stub", PVR_DEBUG_CAPTURE_STUB, "Replace refused shaders with diagnostic stubs; renders incorrect pixels." }, { "errors", PVR_DEBUG_ERRORS, "Log every Vulkan error the driver returns, with its origin." }, { "compute_log", PVR_DEBUG_COMPUTE_LOG, "Log compute pipelines and dispatches." }, + { "bo_log", PVR_DEBUG_BO_LOG, + "Log device-memory and driver BO allocations with size, heap and purpose; live-set summary on any failed allocation and at device destroy." }, DEBUG_NAMED_VALUE_END }; /* clang-format on */ diff --git a/src/imagination/common/pvr_debug.h b/src/imagination/common/pvr_debug.h index 578c3ef7b6b..86198a951f5 100644 --- a/src/imagination/common/pvr_debug.h +++ b/src/imagination/common/pvr_debug.h @@ -38,6 +38,7 @@ extern uint32_t PVR_DEBUG; #define PVR_DEBUG_TRACK_BOS BITFIELD_BIT(1) #define PVR_DEBUG_VK_DUMP_DESCRIPTOR_SET_LAYOUT BITFIELD_BIT(2) #define PVR_DEBUG_INFO BITFIELD_BIT(3) +#define PVR_DEBUG_BO_LOG BITFIELD_BIT(4) #define PVR_DEBUG_SHADER_DUMP BITFIELD_BIT(6) diff --git a/src/imagination/vulkan/meson.build b/src/imagination/vulkan/meson.build index 540d3ab098a..1b49d9a70b0 100644 --- a/src/imagination/vulkan/meson.build +++ b/src/imagination/vulkan/meson.build @@ -47,6 +47,7 @@ pvr_files = files( 'winsys/pvr_winsys.c', 'winsys/pvr_winsys_helper.c', 'pvr_bo.c', + 'pvr_bo_log.c', 'pvr_csb.c', 'pvr_descriptor_set.c', 'pvr_device.c', diff --git a/src/imagination/vulkan/pvr_arch_device.c b/src/imagination/vulkan/pvr_arch_device.c index 5c1e3a4f185..9f8a933db92 100644 --- a/src/imagination/vulkan/pvr_arch_device.c +++ b/src/imagination/vulkan/pvr_arch_device.c @@ -10,6 +10,7 @@ * SPDX-License-Identifier: MIT */ +#include "pvr_bo_log.h" #include "pvr_device.h" #include "vk_log.h" @@ -858,6 +859,8 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, if (result != VK_SUCCESS) goto err_vk_device_finish; + pvr_bo_log_device_init(device); + pvr_bo_suballocator_init(&device->suballoc_general, device->heaps.general_heap, device, @@ -1057,6 +1060,10 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, err_dec_device_count: p_atomic_dec(&device->instance->active_device_count); + if (unlikely(device->bo_log)) { + pvr_bo_log_summary(device, "destroy"); + pvr_bo_log_device_finish(device); + } pvr_bo_suballocator_fini(&device->suballoc_vis_test); pvr_bo_suballocator_fini(&device->suballoc_usc); pvr_bo_suballocator_fini(&device->suballoc_transfer); @@ -1129,6 +1136,10 @@ void PVR_PER_ARCH(destroy_device)(struct pvr_device *device, pvr_bo_suballoc_free(device->nop_program.usc); pvr_free_list_destroy(device->global_free_list); pvr_finish_null_pages(device); + if (unlikely(device->bo_log)) { + pvr_bo_log_summary(device, "destroy"); + pvr_bo_log_device_finish(device); + } pvr_bo_suballocator_fini(&device->suballoc_vis_test); pvr_bo_suballocator_fini(&device->suballoc_usc); pvr_bo_suballocator_fini(&device->suballoc_transfer); diff --git a/src/imagination/vulkan/pvr_bo.c b/src/imagination/vulkan/pvr_bo.c index 629c4f81fde..6c2b40c93e1 100644 --- a/src/imagination/vulkan/pvr_bo.c +++ b/src/imagination/vulkan/pvr_bo.c @@ -37,6 +37,7 @@ #endif #include "hwdef/rogue_hw_utils.h" +#include "pvr_bo_log.h" #include "pvr_bo.h" #include "pvr_debug.h" #include "pvr_device.h" @@ -338,12 +339,13 @@ static inline void pvr_bo_free_bo(const struct pvr_device *const device, * * \sa #pvr_bo_free() */ -VkResult pvr_bo_alloc(struct pvr_device *device, - struct pvr_winsys_heap *heap, - uint64_t size, - uint64_t alignment, - uint64_t flags, - struct pvr_bo **const pvr_bo_out) +VkResult pvr_bo_alloc_tagged(const char *tag, + struct pvr_device *device, + struct pvr_winsys_heap *heap, + uint64_t size, + uint64_t alignment, + uint64_t flags, + struct pvr_bo **const pvr_bo_out) { struct pvr_bo *pvr_bo; VkResult result; @@ -383,6 +385,14 @@ VkResult pvr_bo_alloc(struct pvr_device *device, pvr_bo_store_insert(device->bo_store, pvr_bo); *pvr_bo_out = pvr_bo; + if (unlikely(device->bo_log)) + pvr_bo_log_driver_alloc(device, + tag, + heap, + size, + flags, + pvr_bo, + VK_SUCCESS); return VK_SUCCESS; @@ -400,6 +410,8 @@ VkResult pvr_bo_alloc(struct pvr_device *device, pvr_bo_free_bo(device, pvr_bo); err_out: + if (unlikely(device->bo_log)) + pvr_bo_log_driver_alloc(device, tag, heap, size, flags, NULL, result); return result; } @@ -479,6 +491,9 @@ void pvr_bo_free(struct pvr_device *device, struct pvr_bo *pvr_bo) vk_free(&device->vk.alloc, pvr_bo->bo->vbits); #endif /* defined(HAVE_VALGRIND) */ + if (unlikely(device->bo_log)) + pvr_bo_log_free(device, pvr_bo); + pvr_bo_store_remove(device->bo_store, pvr_bo); device->ws->ops->vma_unmap(pvr_bo->vma); diff --git a/src/imagination/vulkan/pvr_bo.h b/src/imagination/vulkan/pvr_bo.h index bf9c22a5882..941f98b9e2b 100644 --- a/src/imagination/vulkan/pvr_bo.h +++ b/src/imagination/vulkan/pvr_bo.h @@ -117,12 +117,15 @@ struct pvr_suballoc_bo { */ #define PVR_BO_ALLOC_FLAG_PM_FW_PROTECT BITFIELD_BIT(3U) -VkResult pvr_bo_alloc(struct pvr_device *device, - struct pvr_winsys_heap *heap, - uint64_t size, - uint64_t alignment, - uint64_t flags, - struct pvr_bo **const bo_out); +VkResult pvr_bo_alloc_tagged(const char *tag, + struct pvr_device *device, + struct pvr_winsys_heap *heap, + uint64_t size, + uint64_t alignment, + uint64_t flags, + struct pvr_bo **const bo_out); +#define pvr_bo_alloc(...) pvr_bo_alloc_tagged(__func__, __VA_ARGS__) + VkResult pvr_bo_cpu_map(struct pvr_device *device, struct pvr_bo *bo); void pvr_bo_cpu_unmap(struct pvr_device *device, struct pvr_bo *bo); void pvr_bo_free(struct pvr_device *device, struct pvr_bo *bo); diff --git a/src/imagination/vulkan/pvr_bo_log.c b/src/imagination/vulkan/pvr_bo_log.c new file mode 100644 index 00000000000..3875cbb2f4f --- /dev/null +++ b/src/imagination/vulkan/pvr_bo_log.c @@ -0,0 +1,624 @@ +/* SPDX-License-Identifier: MIT */ +#include "pvr_bo_log.h" + +#include +#include +#include +#include +#include +#if defined(__linux__) +# include +#endif + +#include "pvr_bo.h" +#include "pvr_buffer.h" +#include "pvr_debug.h" +#include "pvr_device.h" +#include "pvr_image.h" +#include "util/hash_table.h" +#include "util/list.h" +#include "util/simple_mtx.h" +#include "util/u_math.h" +#include "util/log.h" +#include "util/u_debug.h" +#include "vk_enum_to_str.h" +#include "vk_format.h" +#include "vk_util.h" +#include "winsys/pvr_winsys.h" + +struct pvr_bo_log_entry { + const char *kind; + const char *tag; + const char *heap; + uint64_t size; + bool imported; + bool display; + char thread[17]; +}; + +/* Keep object records separately so padding totals include every live binding, + * while each allocation's live line shows only its four largest objects. + * Descriptions never dereference Vulkan objects at exit. + */ +struct pvr_bo_log_binding { + const void *memory; + uint64_t size; + uint64_t tight; + bool image; + char description[512]; +}; + +struct pvr_bo_log { + struct list_head link; + simple_mtx_t mutex; + struct hash_table *entries; + struct hash_table *bindings; + uint64_t min_size; + uint64_t app_live; + uint64_t app_count; + uint64_t import_live; + uint64_t display_live; + uint64_t drv_live; + uint64_t drv_count; + uint64_t peak; + uint64_t failed; + uint64_t tracking_failed; +}; + +static struct list_head logs = { &logs, &logs }; +static simple_mtx_t logs_mutex = SIMPLE_MTX_INITIALIZER; +DEBUG_GET_ONCE_NUM_OPTION(pvr_bo_log_min_kib, "PVR_BO_LOG_MIN_KIB", 64) + +static const char *thread_name(char buf[17]) +{ + buf[0] = 0; +#if defined(PR_GET_NAME) + if (prctl(PR_GET_NAME, buf) != 0) + buf[0] = 0; + buf[16] = 0; +#endif + for (char *c = buf; *c; c++) { + if (*c == ' ') + *c = '_'; + } + return buf[0] ? buf : "?"; +} + +static void +dedicated_word(const VkMemoryAllocateInfo *info, char *buf, size_t n) +{ + const VkMemoryDedicatedAllocateInfo *ded = + vk_find_struct_const(info->pNext, MEMORY_DEDICATED_ALLOCATE_INFO); + + if (ded && ded->image != VK_NULL_HANDLE) + snprintf(buf, n, "image:%#" PRIx64, (uint64_t)(uintptr_t)ded->image); + else if (ded && ded->buffer != VK_NULL_HANDLE) + snprintf(buf, n, "buffer:%#" PRIx64, (uint64_t)(uintptr_t)ded->buffer); + else + snprintf(buf, n, "none"); +} + +static const char *heap_name(const struct pvr_device *device, + const struct pvr_winsys_heap *heap) +{ + if (heap == device->heaps.general_heap) + return "general"; + if (heap == device->heaps.pds_heap) + return "pds"; + if (heap == device->heaps.rgn_hdr_heap) + return "rgn_hdr"; + if (heap == device->heaps.transfer_frag_heap) + return "transfer_frag"; + if (heap == device->heaps.usc_heap) + return "usc"; + if (heap == device->heaps.vis_test_heap) + return "vis_test"; + return "unknown"; +} + +static uint64_t live_bytes(const struct pvr_bo_log *log) +{ + return log->app_live - log->import_live + log->drv_live; +} + +struct pvr_bo_log_tag { + const char *kind; + const char *tag; + const char *heap; + uint64_t size; + uint64_t count; +}; + +static int compare_tags(const void *a, const void *b) +{ + const struct pvr_bo_log_tag *ta = a, *tb = b; + return (ta->size < tb->size) - (ta->size > tb->size); +} + +static void summary_locked(struct pvr_bo_log *log, const char *why, bool tables) +{ + const uint64_t created = live_bytes(log) - log->display_live; + const uint64_t ws_created = pvr_winsys_counted_bytes(0); + + mesa_logi("pvr bo summary why=%s pid=%ld app_live_kib=%" PRIu64 + " app_count=%" PRIu64 " import_kib=%" PRIu64 " drv_live_kib=%" PRIu64 + " drv_count=%" PRIu64 " created_live_kib=%" PRIu64 + " ws_created_kib=%" PRIu64 " ws_display_kib=%" PRIu64 + " untagged_kib=%" PRId64 " peak_kib=%" PRIu64 " failed=%" PRIu64 + " tracking_failed=%" PRIu64 "\n", + why, + (long)getpid(), + log->app_live >> 10, + log->app_count, + log->import_live >> 10, + log->drv_live >> 10, + log->drv_count, + created >> 10, + ws_created >> 10, + pvr_winsys_counted_bytes(1) >> 10, + ((int64_t)ws_created - (int64_t)created) / 1024, + log->peak >> 10, + log->failed, + log->tracking_failed); + if (!tables) { + + return; + } + + uint64_t images = 0, total = 0, tight = 0; + hash_table_foreach (log->bindings, h) { + const struct pvr_bo_log_binding *b = h->data; + if (b->image) { + images++; + total += b->size; + tight += b->tight; + } + } + mesa_logi("pvr bo summary-pad images=%" PRIu64 " total_kib=%" PRIu64 + " tight_kib=%" PRIu64 " pad_kib=%" PRId64 "\n", + images, + total >> 10, + tight >> 10, + ((int64_t)total - (int64_t)tight) / 1024); + + struct pvr_bo_log_tag *tags = + calloc(MAX2(log->entries->entries, 1), sizeof(*tags)); + unsigned num_tags = 0; + const struct pvr_bo_log_entry *largest[40] = { 0 }; + const void *keys[40] = { 0 }; + + hash_table_foreach (log->entries, h) { + const struct pvr_bo_log_entry *e = h->data; + if (tags) { + unsigned t; + for (t = 0; t < num_tags; t++) { + if (!strcmp(tags[t].kind, e->kind) && + !strcmp(tags[t].tag, e->tag) && !strcmp(tags[t].heap, e->heap)) + break; + } + if (t == num_tags) { + tags[t].kind = e->kind; + tags[t].tag = e->tag; + tags[t].heap = e->heap; + num_tags++; + } + tags[t].size += e->size; + tags[t].count++; + } + for (unsigned i = 0; i < ARRAY_SIZE(largest); i++) { + if (!largest[i] || e->size > largest[i]->size) { + for (unsigned j = ARRAY_SIZE(largest) - 1; j > i; j--) { + largest[j] = largest[j - 1]; + keys[j] = keys[j - 1]; + } + largest[i] = e; + keys[i] = h->key; + break; + } + } + } + if (tags) { + qsort(tags, num_tags, sizeof(*tags), compare_tags); + for (unsigned i = 0; i < num_tags; i++) { + mesa_logi("pvr bo summary-tag kind=%s tag=%s heap=%s count=%" PRIu64 + " kib=%" PRIu64 "\n", + tags[i].kind, + tags[i].tag, + tags[i].heap, + tags[i].count, + tags[i].size >> 10); + } + free(tags); + } + for (unsigned i = 0; i < ARRAY_SIZE(largest) && largest[i]; i++) { + const struct pvr_bo_log_entry *e = largest[i]; + const struct pvr_bo_log_binding *bound[4] = { 0 }; + unsigned count = 0; + hash_table_foreach (log->bindings, h) { + const struct pvr_bo_log_binding *b = h->data; + if (b->memory != keys[i]) + continue; + count++; + for (unsigned j = 0; j < ARRAY_SIZE(bound); j++) { + if (!bound[j] || b->size > bound[j]->size) { + for (unsigned k = ARRAY_SIZE(bound) - 1; k > j; k--) + bound[k] = bound[k - 1]; + bound[j] = b; + break; + } + } + } + char descriptions[4096] = { 0 }; + size_t used = 0; + for (unsigned j = 0; j < ARRAY_SIZE(bound) && bound[j]; j++) { + int n = snprintf(descriptions + used, sizeof(descriptions) - used, + "%s%s", j ? ";" : "", bound[j]->description); + used += MIN2(MAX2(n, 0), sizeof(descriptions) - used - 1); + } + if (count > ARRAY_SIZE(bound)) + snprintf(descriptions + used, sizeof(descriptions) - used, + ";+%u", count - (unsigned)ARRAY_SIZE(bound)); + mesa_logi("pvr bo live kind=%s size_kib=%" PRIu64 + " tag=%s heap=%s thread=%s bound=%s", + e->kind, e->size >> 10, e->tag, e->heap, e->thread, descriptions); + } + +} + +void pvr_bo_log_summary(struct pvr_device *device, const char *why) +{ + struct pvr_bo_log *log = device->bo_log; + simple_mtx_lock(&log->mutex); + summary_locked(log, why, true); + simple_mtx_unlock(&log->mutex); +} + +void pvr_bo_log_device_init(struct pvr_device *device) +{ + if (!PVR_IS_DEBUG_SET(BO_LOG)) + return; + + struct pvr_bo_log *log = calloc(1, sizeof(*log)); + if (!log) + return; + log->entries = _mesa_pointer_hash_table_create(NULL); + log->bindings = _mesa_pointer_hash_table_create(NULL); + if (!log->entries || !log->bindings) { + _mesa_hash_table_destroy(log->entries, NULL); + _mesa_hash_table_destroy(log->bindings, NULL); + free(log); + return; + } + log->min_size = MAX2(0, debug_get_option_pvr_bo_log_min_kib()); + log->min_size = MIN2(log->min_size, UINT64_MAX / 1024) * 1024; + simple_mtx_init(&log->mutex, mtx_plain); + simple_mtx_lock(&logs_mutex); + list_addtail(&log->link, &logs); + device->bo_log = log; + simple_mtx_unlock(&logs_mutex); +} + +static void free_record(struct hash_entry *entry) +{ + free(entry->data); +} + +void pvr_bo_log_device_finish(struct pvr_device *device) +{ + struct pvr_bo_log *log = device->bo_log; + simple_mtx_lock(&logs_mutex); + list_del(&log->link); + device->bo_log = NULL; + simple_mtx_unlock(&logs_mutex); + _mesa_hash_table_destroy(log->bindings, free_record); + _mesa_hash_table_destroy(log->entries, free_record); + simple_mtx_destroy(&log->mutex); + free(log); +} + +static void record_alloc(struct pvr_bo_log *log, + const void *key, + const struct pvr_bo_log_entry *entry, + const char *detail, + VkResult result) +{ + simple_mtx_lock(&log->mutex); + if (result == VK_SUCCESS) { + struct pvr_bo_log_entry *copy = malloc(sizeof(*copy)); + if (copy) { + *copy = *entry; + if (!_mesa_hash_table_insert(log->entries, key, copy)) { + free(copy); + log->tracking_failed++; + } + } else { + log->tracking_failed++; + } + if (!strcmp(entry->kind, "app")) { + log->app_live += entry->size; + log->app_count++; + if (entry->imported) + log->import_live += entry->size; + else if (entry->display) + log->display_live += entry->size; + } else { + log->drv_live += entry->size; + log->drv_count++; + } + } else { + log->failed++; + } + if (entry->size >= log->min_size || result != VK_SUCCESS) { + mesa_logi("pvr bo %s kind=%s %s=%p size_kib=%" PRIu64 + " tag=%s heap=%s %s result=%s live_kib=%" PRIu64 + " thread=%s pid=%ld\n", + result == VK_SUCCESS ? "alloc" : "alloc-failed", + entry->kind, + !strcmp(entry->kind, "app") ? "mem" : "bo", + key, + entry->size >> 10, + entry->tag, + entry->heap, + detail, + vk_Result_to_str(result), + live_bytes(log) >> 10, + entry->thread, + (long)getpid()); + + } + uint64_t old_peak = log->peak; + log->peak = MAX2(log->peak, live_bytes(log)); + if ((log->peak >> 26) > (old_peak >> 26)) + summary_locked(log, "peak", false); + if (result != VK_SUCCESS && log->failed <= 3) + summary_locked(log, "alloc-failed", true); + simple_mtx_unlock(&log->mutex); +} + +void pvr_bo_log_driver_alloc(struct pvr_device *device, + const char *tag, + struct pvr_winsys_heap *heap, + uint64_t size, + uint64_t flags, + const struct pvr_bo *bo, + VkResult result) +{ + struct pvr_bo_log_entry entry = { + .kind = "drv", + .tag = tag, + .heap = heap_name(device, heap), + .size = bo ? bo->bo->size : size, + }; + char detail[48]; + snprintf(detail, sizeof(detail), "flags=%#" PRIx64, flags); + const char *name = thread_name(entry.thread); + if (name != entry.thread) + snprintf(entry.thread, sizeof(entry.thread), "%s", name); + record_alloc(device->bo_log, bo, &entry, detail, result); +} + +void pvr_bo_log_memory_alloc(struct pvr_device *device, + const VkMemoryAllocateInfo *info, + const struct pvr_device_memory *mem, + VkResult result) +{ + const VkImportMemoryFdInfoKHR *imp = + vk_find_struct_const(info->pNext, IMPORT_MEMORY_FD_INFO_KHR); + const VkMemoryDedicatedAllocateInfo *ded = + vk_find_struct_const(info->pNext, MEMORY_DEDICATED_ALLOCATE_INFO); + const bool wsi = + vk_find_struct_const(info->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA) != NULL; + struct pvr_bo_log_entry entry = { + .kind = "app", + .tag = "app", + .heap = "general", + .size = mem ? mem->bo->size : info->allocationSize, + .imported = imp && imp->handleType, + .display = wsi && device->ws->display_fd >= 0, + }; + if (wsi) + entry.tag = "wsi"; + else if (entry.imported) + entry.tag = "import"; + else if (vk_find_struct_const(info->pNext, EXPORT_MEMORY_ALLOCATE_INFO)) + entry.tag = "export"; + else if (ded && ded->image != VK_NULL_HANDLE) + entry.tag = "dedicated-image"; + else if (ded && ded->buffer != VK_NULL_HANDLE) + entry.tag = "dedicated-buffer"; + + char dedicated[48], detail[96]; + dedicated_word(info, dedicated, sizeof(dedicated)); + snprintf(detail, + sizeof(detail), + "type=%u dedicated=%s display=%u", + info->memoryTypeIndex, + dedicated, + entry.display); + const char *name = thread_name(entry.thread); + if (name != entry.thread) + snprintf(entry.thread, sizeof(entry.thread), "%s", name); + record_alloc(device->bo_log, mem, &entry, detail, result); +} + +void pvr_bo_log_free(struct pvr_device *device, const void *key) +{ + struct pvr_bo_log *log = device->bo_log; + simple_mtx_lock(&log->mutex); + struct hash_entry *h = _mesa_hash_table_search(log->entries, key); + if (h) { + struct pvr_bo_log_entry *e = h->data; + if (!strcmp(e->kind, "app")) { + log->app_live -= e->size; + log->app_count--; + if (e->imported) + log->import_live -= e->size; + else if (e->display) + log->display_live -= e->size; + } else { + log->drv_live -= e->size; + log->drv_count--; + } + if (e->size >= log->min_size) { + mesa_logi("pvr bo free kind=%s %s=%p size_kib=%" PRIu64 + " live_kib=%" PRIu64 "\n", + e->kind, + !strcmp(e->kind, "app") ? "mem" : "bo", + key, + e->size >> 10, + live_bytes(log) >> 10); + + } + hash_table_foreach (log->bindings, b) { + struct pvr_bo_log_binding *binding = b->data; + if (binding->memory == key) { + free(binding); + _mesa_hash_table_remove(log->bindings, b); + } + } + free(e); + _mesa_hash_table_remove(log->entries, h); + } + simple_mtx_unlock(&log->mutex); +} + +static void record_binding(struct pvr_bo_log *log, + const void *object, + const struct pvr_bo_log_binding *binding) +{ + struct hash_entry *old = _mesa_hash_table_search(log->bindings, object); + if (old) { + free(old->data); + _mesa_hash_table_remove(log->bindings, old); + } + struct pvr_bo_log_binding *copy = malloc(sizeof(*copy)); + if (copy) { + *copy = *binding; + if (!_mesa_hash_table_insert(log->bindings, object, copy)) { + free(copy); + log->tracking_failed++; + } + } else { + log->tracking_failed++; + } +} + +void pvr_bo_log_unbind(struct pvr_device *device, const void *object) +{ + struct pvr_bo_log *log = device->bo_log; + simple_mtx_lock(&log->mutex); + struct hash_entry *h = _mesa_hash_table_search(log->bindings, object); + if (h) { + free(h->data); + _mesa_hash_table_remove(log->bindings, h); + } + simple_mtx_unlock(&log->mutex); +} + +static const char *layout_name(enum pvr_memlayout layout) +{ + switch (layout) { + case PVR_MEMLAYOUT_LINEAR: + return "LINEAR"; + case PVR_MEMLAYOUT_TWIDDLED: + return "TWIDDLED"; + case PVR_MEMLAYOUT_3DTWIDDLED: + return "3DTWIDDLED"; + default: + return "UNKNOWN"; + } +} + +void pvr_bo_log_image(struct pvr_device *device, + const struct pvr_image *image, + const struct pvr_device_memory *mem, + uint64_t offset) +{ + struct pvr_bo_log *log = device->bo_log; + struct pvr_bo_log_binding binding = { + .memory = mem, + .size = image->total_size, + .image = true, + }; + const uint32_t bw = vk_format_get_blockwidth(image->vk.format); + const uint32_t bh = vk_format_get_blockheight(image->vk.format); + const uint32_t bs = vk_format_get_blocksize(image->vk.format); + for (unsigned l = 0; l < image->vk.mip_levels; l++) { + binding.tight += + (uint64_t)DIV_ROUND_UP(u_minify(image->vk.extent.width, l), bw) * + DIV_ROUND_UP(u_minify(image->vk.extent.height, l), bh) * bs * + u_minify(image->vk.extent.depth, l) * image->vk.array_layers * + image->vk.samples; + } + snprintf(binding.description, + sizeof(binding.description), + "image=%p off=%" PRIu64 + " fmt=%s extent=%ux%ux%u levels=%u layers=%u" + " samples=%u usage=%#" PRIx64 " flags=%#" PRIx64 + " tiling=%s layout=%s phys=%ux%u" + " size_kib=%" PRIu64 " tight_kib=%" PRIu64 " pad_kib=%" PRId64, + (const void *)image, + offset, + vk_Format_to_str(image->vk.format), + image->vk.extent.width, + image->vk.extent.height, + image->vk.extent.depth, + image->vk.mip_levels, + image->vk.array_layers, + image->vk.samples, + (uint64_t)image->vk.usage, + image->vk.create_flags, + image->vk.tiling == VK_IMAGE_TILING_OPTIMAL ? "optimal" : "linear", + layout_name(image->memlayout), + image->planes[0].physical_extent.width, + image->planes[0].physical_extent.height, + binding.size >> 10, + binding.tight >> 10, + ((int64_t)binding.size - (int64_t)binding.tight) / 1024); + simple_mtx_lock(&log->mutex); + record_binding(log, image, &binding); + if (binding.size >= log->min_size) { + mesa_logi("pvr bo bind-image mem=%p %s\n", + (const void *)mem, + binding.description); + + } + simple_mtx_unlock(&log->mutex); +} + +void pvr_bo_log_buffer(struct pvr_device *device, + const struct pvr_buffer *buffer, + const struct pvr_device_memory *mem, + uint64_t offset) +{ + struct pvr_bo_log *log = device->bo_log; + struct pvr_bo_log_binding binding = { + .memory = mem, + .size = buffer->vk.size, + }; + /* Match GetBufferMemoryRequirements2, including its read-ahead padding. */ + uint64_t req = buffer->vk.size; + if (req % device->ws->page_size == 0 || + req % device->ws->page_size > + device->ws->page_size - PVR_BUFFER_MEMORY_PADDING_SIZE) + req += PVR_BUFFER_MEMORY_PADDING_SIZE; + req = align64(req, buffer->alignment); + snprintf(binding.description, + sizeof(binding.description), + "buffer=%p off=%" PRIu64 " size_kib=%" PRIu64 " req_kib=%" PRIu64 + " usage=%#" PRIx64, + (const void *)buffer, + offset, + binding.size >> 10, + req >> 10, + (uint64_t)buffer->vk.usage); + simple_mtx_lock(&log->mutex); + record_binding(log, buffer, &binding); + if (binding.size >= log->min_size) { + mesa_logi("pvr bo bind-buffer mem=%p %s\n", + (const void *)mem, + binding.description); + + } + simple_mtx_unlock(&log->mutex); +} diff --git a/src/imagination/vulkan/pvr_bo_log.h b/src/imagination/vulkan/pvr_bo_log.h new file mode 100644 index 00000000000..28606747031 --- /dev/null +++ b/src/imagination/vulkan/pvr_bo_log.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef PVR_BO_LOG_H +#define PVR_BO_LOG_H + +#include +#include + +struct pvr_bo; +struct pvr_buffer; +struct pvr_device; +struct pvr_device_memory; +struct pvr_image; +struct pvr_winsys_heap; + +void pvr_bo_log_device_init(struct pvr_device *device); +void pvr_bo_log_device_finish(struct pvr_device *device); +void pvr_bo_log_summary(struct pvr_device *device, const char *why); +void pvr_bo_log_driver_alloc(struct pvr_device *device, + const char *tag, + struct pvr_winsys_heap *heap, + uint64_t size, + uint64_t flags, + const struct pvr_bo *bo, + VkResult result); +void pvr_bo_log_memory_alloc(struct pvr_device *device, + const VkMemoryAllocateInfo *info, + const struct pvr_device_memory *mem, + VkResult result); +void pvr_bo_log_free(struct pvr_device *device, const void *key); +void pvr_bo_log_image(struct pvr_device *device, + const struct pvr_image *image, + const struct pvr_device_memory *mem, + uint64_t offset); +void pvr_bo_log_buffer(struct pvr_device *device, + const struct pvr_buffer *buffer, + const struct pvr_device_memory *mem, + uint64_t offset); +void pvr_bo_log_unbind(struct pvr_device *device, const void *object); + +#endif /* PVR_BO_LOG_H */ diff --git a/src/imagination/vulkan/pvr_device.c b/src/imagination/vulkan/pvr_device.c index 7d88fa6f397..e3bfacd1222 100644 --- a/src/imagination/vulkan/pvr_device.c +++ b/src/imagination/vulkan/pvr_device.c @@ -27,6 +27,7 @@ * SOFTWARE. */ +#include "pvr_bo_log.h" #include "pvr_device.h" #include @@ -215,8 +216,14 @@ VkResult pvr_AllocateMemory(VkDevice _device, const VkImportMemoryFdInfoKHR *fd_info = NULL; VK_FROM_HANDLE(pvr_device, device, _device); - if (vk_device_is_lost_no_report(&device->vk)) + if (vk_device_is_lost_no_report(&device->vk)) { + if (unlikely(device->bo_log)) + pvr_bo_log_memory_alloc(device, + pAllocateInfo, + NULL, + VK_ERROR_OUT_OF_DEVICE_MEMORY); return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); + } enum pvr_winsys_bo_type type = PVR_WINSYS_BO_TYPE_GPU; struct pvr_device_memory *mem; VkResult result; @@ -232,15 +239,27 @@ VkResult pvr_AllocateMemory(VkDevice _device, VkDeviceSize aligned_alloc_size = ALIGN_POT(pAllocateInfo->allocationSize, device->ws->page_size); - if (aligned_alloc_size > mem_heap->size) + if (aligned_alloc_size > mem_heap->size) { + if (unlikely(device->bo_log)) + pvr_bo_log_memory_alloc(device, + pAllocateInfo, + NULL, + VK_ERROR_OUT_OF_DEVICE_MEMORY); return VK_ERROR_OUT_OF_DEVICE_MEMORY; + } mem = vk_device_memory_create(&device->vk, pAllocateInfo, pAllocator, sizeof(*mem)); - if (!mem) + if (!mem) { + if (unlikely(device->bo_log)) + pvr_bo_log_memory_alloc(device, + pAllocateInfo, + NULL, + VK_ERROR_OUT_OF_HOST_MEMORY); return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + } vk_foreach_struct_const (sType, ext, pAllocateInfo->pNext) { switch ((unsigned)sType) { @@ -335,6 +354,8 @@ VkResult pvr_AllocateMemory(VkDevice _device, } *pMem = pvr_device_memory_to_handle(mem); + if (unlikely(device->bo_log)) + pvr_bo_log_memory_alloc(device, pAllocateInfo, mem, VK_SUCCESS); vk_device_memory_report_emit(&device->vk, VK_SUCCESS, /* is_alloc */ true, mem->vk.import_handle_type != 0, @@ -346,6 +367,8 @@ VkResult pvr_AllocateMemory(VkDevice _device, return VK_SUCCESS; err_vk_device_memory_destroy: + if (unlikely(device->bo_log)) + pvr_bo_log_memory_alloc(device, pAllocateInfo, NULL, result); vk_device_memory_report_emit(&device->vk, result, /* is_alloc */ true, mem->vk.import_handle_type != 0, mem->bo ? (uintptr_t)mem->bo : 0, @@ -428,6 +451,8 @@ void pvr_FreeMemory(VkDevice _device, if (mem->bo->map) device->ws->ops->buffer_unmap(mem->bo, false); + if (unlikely(device->bo_log)) + pvr_bo_log_free(device, mem); device->ws->ops->buffer_destroy(mem->bo); vk_device_memory_destroy(&device->vk, pAllocator, &mem->vk); @@ -663,11 +688,15 @@ VkResult pvr_BindBufferMemory2(VkDevice _device, if (result != VK_SUCCESS) { while (i--) { VK_FROM_HANDLE(pvr_buffer, buffer, pBindInfos[i].buffer); + if (unlikely(device->bo_log)) + pvr_bo_log_unbind(device, buffer); pvr_unbind_memory(device, buffer->vma); } return result; } + if (unlikely(device->bo_log)) + pvr_bo_log_buffer(device, buffer, mem, pBindInfos[i].memoryOffset); } return VK_SUCCESS; @@ -852,6 +881,9 @@ void pvr_DestroyBuffer(VkDevice _device, if (!buffer) return; + if (unlikely(device->bo_log)) + pvr_bo_log_unbind(device, buffer); + if (buffer->vma) pvr_unbind_memory(device, buffer->vma); diff --git a/src/imagination/vulkan/pvr_device.h b/src/imagination/vulkan/pvr_device.h index ca2d5b760f1..022b999dc3a 100644 --- a/src/imagination/vulkan/pvr_device.h +++ b/src/imagination/vulkan/pvr_device.h @@ -217,6 +217,7 @@ struct pvr_device { struct pvr_spm_scratch_buffer_store spm_scratch_buffer_store; struct pvr_bo_store *bo_store; + struct pvr_bo_log *bo_log; struct pvr_bo *robustness_buffer; diff --git a/src/imagination/vulkan/pvr_image.c b/src/imagination/vulkan/pvr_image.c index cef6854870c..55693cdd2d4 100644 --- a/src/imagination/vulkan/pvr_image.c +++ b/src/imagination/vulkan/pvr_image.c @@ -21,6 +21,7 @@ * SOFTWARE. */ +#include "pvr_bo_log.h" #include "pvr_image.h" #include @@ -430,6 +431,9 @@ VkResult pvr_image_init(struct pvr_device *device, void pvr_image_fini(struct pvr_device *device, struct pvr_image *image) { + if (unlikely(device->bo_log)) + pvr_bo_log_unbind(device, image); + if (image->vma) pvr_unbind_memory(device, image->vma); } @@ -509,11 +513,15 @@ VkResult pvr_BindImageMemory2(VkDevice _device, while (i--) { VK_FROM_HANDLE(pvr_image, image, pBindInfos[i].image); + if (unlikely(device->bo_log)) + pvr_bo_log_unbind(device, image); pvr_unbind_memory(device, image->vma); } return result; } + if (unlikely(device->bo_log)) + pvr_bo_log_image(device, image, mem, offset); } return VK_SUCCESS; diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c index f1cefc60b67..75ad551a814 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c @@ -36,6 +36,7 @@ #include "drm-uapi/pvr_drm.h" #include "pvr_drm.h" +#include "pvr_debug.h" #include "pvr_drm_bo.h" #include "pvr_macros.h" #include "pvr_winsys_helper.h" @@ -150,6 +151,9 @@ static void pvr_drm_buffer_release(struct pvr_drm_winsys_bo *drm_bo) if (p_atomic_dec_return(&drm_bo->ref_count) == 0) { uint32_t handle = drm_bo->handle; + if (PVR_IS_DEBUG_SET(BO_LOG) && drm_bo->count_kind < 2) + pvr_winsys_count_bo(drm_bo->count_kind, -(int64_t)drm_bo->base.size); + /* Our BO structs are stored in a sparse array in the winsys structure, * so we don't want to free the BO pointer, instead we want to reset it * to 0, to signal that array entry as being free. @@ -194,6 +198,10 @@ pvr_drm_display_buffer_create(struct pvr_drm_winsys *drm_ws, assert((*bo_out)->size >= size); + to_pvr_drm_winsys_bo(*bo_out)->count_kind = 1; + if (PVR_IS_DEBUG_SET(BO_LOG)) + pvr_winsys_count_bo(1, (*bo_out)->size); + return VK_SUCCESS; } @@ -251,6 +259,9 @@ VkResult pvr_drm_winsys_buffer_create(struct pvr_winsys *ws, drm_bo->base.ws = ws; drm_bo->handle = handle; drm_bo->flags = drm_flags; + drm_bo->count_kind = 0; + if (PVR_IS_DEBUG_SET(BO_LOG)) + pvr_winsys_count_bo(0, size); p_atomic_set(&drm_bo->ref_count, 1); g9p_bolog('+', handle, size); @@ -295,6 +306,7 @@ pvr_drm_winsys_buffer_create_from_fd(struct pvr_winsys *ws, drm_bo->base.ws = ws; drm_bo->base.size = (uint64_t)size; drm_bo->base.is_imported = true; + drm_bo->count_kind = 2; drm_bo->handle = handle; p_atomic_set(&drm_bo->ref_count, 1); diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h index b7879b17354..f102325df8a 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h @@ -40,6 +40,8 @@ struct pvr_drm_winsys_bo { struct pvr_winsys_bo base; uint32_t ref_count; + /* 0: CREATE_BO, 1: display, 2: plain import (not counted). */ + uint8_t count_kind; uint32_t handle; uint64_t flags; }; diff --git a/src/imagination/vulkan/winsys/pvr_winsys.c b/src/imagination/vulkan/winsys/pvr_winsys.c index 64c5c76b175..fb4387576e7 100644 --- a/src/imagination/vulkan/winsys/pvr_winsys.c +++ b/src/imagination/vulkan/winsys/pvr_winsys.c @@ -29,11 +29,27 @@ #include "powervr/pvr_drm_public.h" #include "pvr_winsys.h" #include "vk_log.h" +#include "util/u_atomic.h" #if defined(PVR_SUPPORT_SERVICES_DRIVER) # include "pvrsrvkm/pvr_srv_public.h" #endif +/* The kernel charges CREATE_BO memory per process, across winsys instances. */ +static int64_t counted_bytes[2]; + +void pvr_winsys_count_bo(int kind, int64_t delta) +{ + assert(kind >= 0 && kind < ARRAY_SIZE(counted_bytes)); + p_atomic_add(&counted_bytes[kind], delta); +} + +uint64_t pvr_winsys_counted_bytes(int kind) +{ + assert(kind >= 0 && kind < ARRAY_SIZE(counted_bytes)); + return p_atomic_read(&counted_bytes[kind]); +} + void pvr_winsys_destroy(struct pvr_winsys *ws) { const int display_fd = ws->display_fd; diff --git a/src/imagination/vulkan/winsys/pvr_winsys.h b/src/imagination/vulkan/winsys/pvr_winsys.h index 07933c35691..5202e8d4630 100644 --- a/src/imagination/vulkan/winsys/pvr_winsys.h +++ b/src/imagination/vulkan/winsys/pvr_winsys.h @@ -489,6 +489,9 @@ struct pvr_winsys { const struct pvr_winsys_ops *ops; }; +void pvr_winsys_count_bo(int kind, int64_t delta); +uint64_t pvr_winsys_counted_bytes(int kind); + void pvr_winsys_destroy(struct pvr_winsys *ws); VkResult pvr_winsys_create(const char *render_path, const char *display_path, From ba991a3f2bf0791bedfa4fabddd66855c93edf5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 00:01:40 +0100 Subject: [PATCH 082/109] pvr: do not keep oversize buffer objects in allocator caches Allocate requests above twice the default suballocator size in standalone BOs without replacing the current BO. Release BOs above that limit instead of caching them, including dedicated shared spill buffers freed through the suballocator. Assisted-by: LLM --- src/imagination/vulkan/pvr_bo.c | 39 ++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/imagination/vulkan/pvr_bo.c b/src/imagination/vulkan/pvr_bo.c index 6c2b40c93e1..e4bf38f2f6a 100644 --- a/src/imagination/vulkan/pvr_bo.c +++ b/src/imagination/vulkan/pvr_bo.c @@ -553,6 +553,11 @@ static inline struct pvr_bo *pvr_bo_get_ref(struct pvr_bo *bo) return bo; } +static inline uint64_t pvr_suballoc_cache_max(const struct pvr_suballocator *a) +{ + return 2 * (uint64_t)a->default_size; +} + /** * \brief Interface to sub-allocate buffer objects. * @@ -597,6 +602,37 @@ VkResult pvr_bo_suballoc(struct pvr_suballocator *allocator, simple_mtx_lock(&allocator->mtx); + /* Large allocations must not replace the current BO or enter the cache. */ + if (aligned_size > pvr_suballoc_cache_max(allocator)) { + struct pvr_bo *bo; + + result = pvr_bo_alloc(allocator->device, + allocator->heap, + aligned_size, + align, + PVR_BO_ALLOC_FLAG_CPU_MAPPED, + &bo); + if (result != VK_SUCCESS) { + vk_free(&allocator->device->vk.alloc, suballoc_bo); + simple_mtx_unlock(&allocator->mtx); + return result; + } + + suballoc_bo->allocator = allocator; + suballoc_bo->bo = bo; + suballoc_bo->dev_addr = bo->vma->dev_addr; + suballoc_bo->offset = 0; + suballoc_bo->size = aligned_size; + + if (zero_on_alloc) + memset(pvr_bo_suballoc_get_map_addr(suballoc_bo), 0, aligned_size); + + *suballoc_bo_out = suballoc_bo; + simple_mtx_unlock(&allocator->mtx); + return VK_SUCCESS; + } + + if (allocator->bo) { uint32_t aligned_offset = ALIGN_POT(allocator->next_offset, align); @@ -682,7 +718,8 @@ void pvr_bo_suballoc_free(struct pvr_suballoc_bo *suballoc_bo) simple_mtx_lock(&suballoc_bo->allocator->mtx); if (p_atomic_read(&suballoc_bo->bo->ref_count) == 1 && - !suballoc_bo->allocator->bo_cached) { + !suballoc_bo->allocator->bo_cached && + suballoc_bo->bo->bo->size <= pvr_suballoc_cache_max(suballoc_bo->allocator)) { suballoc_bo->allocator->bo_cached = suballoc_bo->bo; } else { pvr_bo_free(suballoc_bo->allocator->device, suballoc_bo->bo); From f048c571fdd2f68ed3ad53976463764612268759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 03:48:11 +0100 Subject: [PATCH 083/109] pvr: drop the 1 GiB minimum heap size Affects: all PowerVR devices. Report the calculated heap size even when it is smaller than 1 GiB. maxMemoryAllocationSize stays MAX2(heap, 1 GiB), so an allocation above the heap passes the driver check and fails in the kernel. Assisted-by: LLM --- src/imagination/vulkan/pvr_device.c | 4 +--- src/imagination/vulkan/pvr_physical_device.c | 10 ++++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/imagination/vulkan/pvr_device.c b/src/imagination/vulkan/pvr_device.c index e3bfacd1222..edf5913296e 100644 --- a/src/imagination/vulkan/pvr_device.c +++ b/src/imagination/vulkan/pvr_device.c @@ -233,13 +233,11 @@ VkResult pvr_AllocateMemory(VkDevice _device, const VkMemoryType *mem_type = &device->pdevice->memory.memoryTypes[pAllocateInfo->memoryTypeIndex]; - const VkMemoryHeap *mem_heap = - &device->pdevice->memory.memoryHeaps[mem_type->heapIndex]; VkDeviceSize aligned_alloc_size = ALIGN_POT(pAllocateInfo->allocationSize, device->ws->page_size); - if (aligned_alloc_size > mem_heap->size) { + if (aligned_alloc_size > device->pdevice->vk.properties.maxMemoryAllocationSize) { if (unlikely(device->bo_log)) pvr_bo_log_memory_alloc(device, pAllocateInfo, diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index 4770f588e3f..f7f0aa46ec9 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -609,6 +609,9 @@ static uint32_t get_api_version(void) static unsigned get_custom_border_color_samplers(const struct pvr_device_info *dev_info); +/* Minimum required by the Vulkan spec Limits (maxMemoryAllocationSize) */ +#define PVR_MAX_MEMORY_ALLOCATION_SIZE (1ull << 30) + static bool pvr_physical_device_get_properties( const struct pvr_physical_device *const pdevice, struct vk_properties *const properties) @@ -660,7 +663,7 @@ static bool pvr_physical_device_get_properties( assert(pdevice->memory.memoryHeapCount == 1); const VkDeviceSize max_memory_alloc_size = - pdevice->memory.memoryHeaps[0].size; + MAX2(pdevice->memory.memoryHeaps[0].size, PVR_MAX_MEMORY_ALLOCATION_SIZE); const uint32_t line_sub_pixel_precision_bits = PVR_HAS_FEATURE(dev_info, simple_internal_parameter_format) ? 4U : 8U; @@ -1169,14 +1172,13 @@ static bool pvr_device_is_conformant(const struct pvr_device_info *info) return false; } -/* Minimum required by the Vulkan spec Limits (maxMemoryAllocationSize) */ -#define PVR_MAX_MEMORY_ALLOCATION_SIZE (1ull << 30) + static inline uint64_t pvr_compute_heap_size(struct pvr_instance *instance) { uint64_t available_ram = os_get_gpu_heap_size(instance->drirc.misc.heap_memory_percent, NULL); - return MAX2(available_ram, PVR_MAX_MEMORY_ALLOCATION_SIZE); + return available_ram; } static void From 05f7e3fed30fd07203687d40102c4027ae0aefe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 03:48:11 +0100 Subject: [PATCH 084/109] pvr/g5p: size the A9 heap at two thirds of RAM Apple's driver limits the GPU working set to two thirds of RAM on this device class (observed through Metal's recommendedMaxWorkingSetSize). Round the default heap to the CPU page size. Explicit heap percentages and other devices keep their existing calculation. maxMemoryAllocationSize stays MAX2(heap, 1 GiB), so an allocation above the heap passes the driver check and fails in the kernel. Assisted-by: LLM --- src/imagination/vulkan/pvr_physical_device.c | 22 +++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index f7f0aa46ec9..d845603f6b1 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -1174,11 +1174,23 @@ static bool pvr_device_is_conformant(const struct pvr_device_info *info) -static inline uint64_t pvr_compute_heap_size(struct pvr_instance *instance) +static inline uint64_t +pvr_compute_heap_size(const struct pvr_physical_device *pdevice, + struct pvr_instance *instance) { - uint64_t available_ram = - os_get_gpu_heap_size(instance->drirc.misc.heap_memory_percent, NULL); - return available_ram; + uint64_t total; + if (!os_get_total_physical_memory(&total)) + return PVR_MAX_MEMORY_ALLOCATION_SIZE; + const float percent = instance->drirc.misc.heap_memory_percent; + if (percent == OS_GPU_HEAP_SIZE_HEURISTIC && + PVR_HAS_FEATURE(&pdevice->dev_info, apple_g5p)) { + uint64_t page_size; + if (!os_get_page_size(&page_size)) + return PVR_MAX_MEMORY_ALLOCATION_SIZE; + /* This device class limits the GPU working set to two thirds of RAM. */ + return ALIGN_POT(total * 2 / 3, page_size); + } + return os_gpu_heap_size_calculate(total, percent, NULL); } static void @@ -1271,7 +1283,7 @@ VkResult pvr_physical_device_init(struct pvr_physical_device *pdevice, /* Setup available memory heaps and types */ pdevice->memory.memoryHeapCount = 1; - pdevice->memory.memoryHeaps[0].size = pvr_compute_heap_size(instance); + pdevice->memory.memoryHeaps[0].size = pvr_compute_heap_size(pdevice, instance); pdevice->memory.memoryHeaps[0].flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; pdevice->memory.memoryTypeCount = 1; From b70ae51dfc35881465cfc94ae261d44dbc4bb348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:02:23 +0100 Subject: [PATCH 085/109] glapi: EXT_semaphore entry points from GLES 2.0, EXT_memory_object from 3.0 The EXT_external_objects and _fd entry points were put in the GLES dispatch only from ES 3.2 (es2="3.2"), while extensions_table.h advertises GL_EXT_semaphore(_fd) and GL_EXT_memory_object(_fd) on any ES 2 context. On a GLES 3.1 context the extensions were listed, but glGenSemaphoresEXT, glImportSemaphoreFdEXT and glWaitSemaphoreEXT reached the no-op stub and raised GL_INVALID_OPERATION: Firefox's WebRender could not wait for WebGPU canvas frames ("glImportSemaphoreFdEXT failed: 1282"). The specifications don't ask for ES 3.2: GL_EXT_semaphore requires OpenGL 1.0, and GL_EXT_memory_object requires texture storage (ES 3.0). Install the semaphore entry points (and GetUnsignedBytev*) from ES 2.0 and the memory object ones from ES 3.0. Assisted-by: LLM --- .../glapi/glapi/gen/EXT_external_objects.xml | 48 +++++++++---------- .../glapi/gen/EXT_external_objects_fd.xml | 4 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/mesa/glapi/glapi/gen/EXT_external_objects.xml b/src/mesa/glapi/glapi/gen/EXT_external_objects.xml index 297b5f4e7b4..c93bc5bb244 100644 --- a/src/mesa/glapi/glapi/gen/EXT_external_objects.xml +++ b/src/mesa/glapi/glapi/gen/EXT_external_objects.xml @@ -26,45 +26,45 @@ - + - + - + - + - + - + - + - + @@ -74,7 +74,7 @@ - + @@ -85,7 +85,7 @@ - + @@ -96,7 +96,7 @@ - + @@ -108,14 +108,14 @@ - + - + @@ -125,7 +125,7 @@ - + @@ -136,7 +136,7 @@ - + @@ -147,7 +147,7 @@ - + @@ -159,7 +159,7 @@ - + @@ -184,34 +184,34 @@ - + - + - + - + - + - + @@ -220,7 +220,7 @@ - + diff --git a/src/mesa/glapi/glapi/gen/EXT_external_objects_fd.xml b/src/mesa/glapi/glapi/gen/EXT_external_objects_fd.xml index 6eaa3f5b0cc..2bb1ed03fff 100644 --- a/src/mesa/glapi/glapi/gen/EXT_external_objects_fd.xml +++ b/src/mesa/glapi/glapi/gen/EXT_external_objects_fd.xml @@ -10,14 +10,14 @@ - + - + From 7c3a0520dc77c9019ad89503a88c7362597d3262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:02:23 +0100 Subject: [PATCH 086/109] egl/wayland: driconf cap on window-surface colour buffers A wallpaper that stops drawing can retain an extra surface-sized buffer while its compositor holds two. Allow such clients to request two to four colour buffers, defaulting to four. Driconf options can also be set from the environment under the option name. Assisted-by: LLM --- src/egl/drivers/dri2/egl_dri2.h | 3 +++ src/egl/drivers/dri2/platform_wayland.c | 17 ++++++++++++++++- src/gallium/frontends/dri/dri_util.c | 2 ++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/egl/drivers/dri2/egl_dri2.h b/src/egl/drivers/dri2/egl_dri2.h index 31916f825e7..0be08c9329b 100644 --- a/src/egl/drivers/dri2/egl_dri2.h +++ b/src/egl/drivers/dri2/egl_dri2.h @@ -353,6 +353,9 @@ struct dri2_egl_surface { bool locked; int age; } color_buffers[4], *back, *current; + /* Wayland: how many of color_buffers this surface may use (2..4, + * MESA_EGL_WAYLAND_MAX_BUFFERS). */ + unsigned max_color_buffers; #endif #ifdef HAVE_ANDROID_PLATFORM diff --git a/src/egl/drivers/dri2/platform_wayland.c b/src/egl/drivers/dri2/platform_wayland.c index 381ee9bfa54..5e5b2f186f0 100644 --- a/src/egl/drivers/dri2/platform_wayland.c +++ b/src/egl/drivers/dri2/platform_wayland.c @@ -42,6 +42,7 @@ #include #include "util/anon_file.h" +#include "util/u_debug.h" #include "util/perf/cpu_trace.h" #include "util/u_vector.h" #include "util/format/u_formats.h" @@ -758,6 +759,18 @@ dri2_wl_create_window_surface(_EGLDisplay *disp, _EGLConfig *conf, dri2_surf->base.Width = window->width; dri2_surf->base.Height = window->height; + /* A surface takes a third (and fourth) color buffer while the compositor + * holds two, and frees a spare only after it has gone unused for + * BUFFER_TRIM_AGE_HYSTERESIS frames. A surface that stops drawing, such as + * a desktop shell's wallpaper after its first frames, keeps the spare for + * good: a whole surface-sized buffer. A client that redraws rarely can + * stay double buffered instead and wait for a release when it draws. */ + int max_buffers = ARRAY_SIZE(dri2_surf->color_buffers); + dri2GalliumConfigQueryi(dri2_dpy->dri_screen_render_gpu, + "egl_wayland_max_buffers", &max_buffers); + dri2_surf->max_color_buffers = + CLAMP(max_buffers, 2, ARRAY_SIZE(dri2_surf->color_buffers)); + visual_idx = dri2_wl_visual_idx_from_config(config); assert(visual_idx != -1); assert(dri2_wl_visuals[visual_idx].pipe_format != PIPE_FORMAT_NONE); @@ -1292,8 +1305,10 @@ wait_for_free_buffer(struct dri2_egl_display *dri2_dpy, /* There might be a buffer release already queued that wasn't processed */ wl_display_dispatch_queue_pending(dri2_dpy->wl_dpy, dri2_surf->wl_queue); + const unsigned max_buffers = dri2_surf->max_color_buffers; + while (dri2_surf->back == NULL) { - for (int i = 0; i < ARRAY_SIZE(dri2_surf->color_buffers); i++) { + for (int i = 0; i < max_buffers; i++) { /* Get an unlocked buffer, preferably one with a dri_buffer * already allocated and with minimum age. */ diff --git a/src/gallium/frontends/dri/dri_util.c b/src/gallium/frontends/dri/dri_util.c index d64aca22854..9acedef54c1 100644 --- a/src/gallium/frontends/dri/dri_util.c +++ b/src/gallium/frontends/dri/dri_util.c @@ -66,6 +66,8 @@ driOptionDescription __dri2ConfigOptions[] = { DRI_CONF_SECTION_PERFORMANCE DRI_CONF_VBLANK_MODE(DRI_CONF_VBLANK_DEF_INTERVAL_1) + DRI_CONF_OPT_I(egl_wayland_max_buffers, 4, 2, 4, + "Cap on EGL Wayland window-surface colour buffers") DRI_CONF_BLOCK_ON_DEPLETED_BUFFERS(false) DRI_CONF_SECTION_END }; From 3015106fdcf891f5837cb31d2017826d309ad5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:02:23 +0100 Subject: [PATCH 087/109] util/astc: a void-extent block without its reserved bits set is the error colour The Khronos Data Format specification requires the reserved bits [11:10] of an ASTC void-extent block to be 11. Reject other values as an error block, matching astcenc. Found on Apple A9 hardware. Assisted-by: LLM --- src/util/texcompress_astc.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/util/texcompress_astc.cpp b/src/util/texcompress_astc.cpp index a413f5743e3..2e31574cf58 100644 --- a/src/util/texcompress_astc.cpp +++ b/src/util/texcompress_astc.cpp @@ -54,6 +54,7 @@ class decode_error reserved_block_mode_2, dual_plane_and_too_many_partitions, invalid_range_in_void_extent, + reserved_void_extent_bits, weight_grid_exceeds_block_size, invalid_colour_endpoints_size, invalid_colour_endpoints_count, @@ -692,6 +693,13 @@ decode_error::type Block::decode_void_extent(InputBitVector block) return decode_error::unsupported_hdr_void_extent; } + /* The Khronos Data Format specification's ASTC void-extent block requires + * reserved bits [11:10] to be 11. Other values decode to the error colour, + * matching astcenc's error-block handling. */ + if (block.get_bits(10, 2) != 0x3) { + return decode_error::reserved_void_extent_bits; + } + if (void_extent_min_s == 0x1fff && void_extent_max_s == 0x1fff && void_extent_min_t == 0x1fff && void_extent_max_t == 0x1fff) { From ddaabb4d3a150f107f4819ed3671030397f76f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 03:46:35 +0100 Subject: [PATCH 088/109] zink: 16 varyings on Imagination between 64 and 120 output components The A9 reports 112 output components: 112/8 gives 14 varyings, below the ES 3.0 minimum of 15. Limit this exception to Imagination without geometry or tessellation shaders and strictly between 64 and 120 components. The A10 at 120 and Turnip keep their existing values. Zink inlines each transform-feedback output into an existing output in update_so_info(). The GLSL linker checks all vertex outputs against maxVertexOutputComponents in check_against_output_limit(), so a program that exceeds the device's output limit fails to link. Assisted-by: LLM --- src/gallium/drivers/zink/zink_screen.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/gallium/drivers/zink/zink_screen.c b/src/gallium/drivers/zink/zink_screen.c index 50f6a86b5c8..fed6f56a379 100644 --- a/src/gallium/drivers/zink/zink_screen.c +++ b/src/gallium/drivers/zink/zink_screen.c @@ -1197,6 +1197,19 @@ zink_init_screen_caps(struct zink_screen *screen) assert(screen->info.feats.features.tessellationShader == VK_FALSE); caps->max_varyings = 16; } + /* Between 64 and 120 output components, halving for streamout falls below + * the ES 3.0 minimum of 15 varyings. Zink inlines every transform-feedback + * output into an existing output (update_so_info()), and the GLSL linker + * limits the vertex stage's outputs to maxVertexOutputComponents + * (check_against_output_limit()), so 16 varyings cannot push the vertex + * shader past the device limit: such a program fails to link instead. + */ + else if (zink_driverid(screen) == VK_DRIVER_ID_IMAGINATION_OPEN_SOURCE_MESA && + !screen->info.feats.features.geometryShader && + !screen->info.feats.features.tessellationShader && + screen->info.props.limits.maxVertexOutputComponents > 64 && + screen->info.props.limits.maxVertexOutputComponents < 120) + caps->max_varyings = 16; caps->dmabuf = #if defined(HAVE_LIBDRM) && (DETECT_OS_LINUX || DETECT_OS_BSD) From 7211300723935a50f776dca1137092daa2348a7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 22:05:43 +0100 Subject: [PATCH 089/109] pvr/g5p: report 112 vertex output and fragment input components WebGPU core wants 16 inter-stage variables, and wgpu derives them as components / 4 - 1, so the A9's 64 gave 15. The fragment compiler takes 28 locations, and the A9 interpolates 124 components exactly. maxVertexOutputComponents and maxFragmentInputComponents go from 64 to 112 on the A9: 4 + 112 + 1 + 8 = 125 of the TA's 128-dword vertex, with the position, a point size and 8 clip distances. GL on Zink keeps 16 varyings. Assisted-by: LLM --- src/imagination/vulkan/pvr_physical_device.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index d845603f6b1..d3c25d5fd1c 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -716,7 +716,8 @@ static bool pvr_physical_device_get_properties( .maxVertexInputAttributeOffset = 2048U - 1U, .maxVertexInputBindingStride = 2048U, /* G9P: its 30 varyings (Zink exposes an eighth of this as GL varyings) */ - .maxVertexOutputComponents = PVR_HAS_FEATURE(dev_info, apple_g9p) ? 120U : 64U, + .maxVertexOutputComponents = PVR_HAS_FEATURE(dev_info, apple_g9p) ? 120U : + PVR_HAS_FEATURE(dev_info, apple_g5p) ? 112U : 64U, /* G5P: 4 * PCO_G5P_MAX_FS_VARYINGS */ /* Tessellation Limits */ /* Requires tessellationShader */ @@ -738,7 +739,8 @@ static bool pvr_physical_device_get_properties( .maxGeometryTotalOutputComponents = 0U, /* Fragment Shader Limits */ - .maxFragmentInputComponents = PVR_HAS_FEATURE(dev_info, apple_g9p) ? 120U : 64U, + .maxFragmentInputComponents = PVR_HAS_FEATURE(dev_info, apple_g9p) ? 120U : + PVR_HAS_FEATURE(dev_info, apple_g5p) ? 112U : 64U, /* G5P: 4 * PCO_G5P_MAX_FS_VARYINGS */ .maxFragmentOutputAttachments = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8U : 4U, /* G5P: 5..8 compact UNORM8 targets */ .maxFragmentDualSrcAttachments = 0U, /* Requires dualSrcBlend */ .maxFragmentCombinedOutputResources = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8U : 4U, From 6b5fa91854a83ff6946ea360219e9281e05c52ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 01:37:42 +0100 Subject: [PATCH 090/109] zink: order buffer copies after earlier batches' reads A reordered command buffer follows earlier batches in submission order, but submission order alone does not synchronize their reads. Check unfinished earlier reads before reordering a buffer update. Carry current reads into ordered history for the next batch, and preserve unfinished unordered history before the first barrier of a new batch, ordered or unordered. The image path applies to images taking zink_resource_memory_barrier(), i.e. GENERAL-layout images (driver_workarounds.general_layout). Host sync-validation and conformance results are recorded in the PR body. A9 GPU-time A/B evidence remains pending a hardware run. Assisted-by: LLM --- src/gallium/drivers/zink/zink_clear.c | 2 +- src/gallium/drivers/zink/zink_context.c | 4 ++-- src/gallium/drivers/zink/zink_context.h | 2 +- src/gallium/drivers/zink/zink_query.c | 2 +- .../drivers/zink/zink_synchronization.cpp | 21 +++++++++++++++++-- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/gallium/drivers/zink/zink_clear.c b/src/gallium/drivers/zink/zink_clear.c index 156175f7695..fefd29e503f 100644 --- a/src/gallium/drivers/zink/zink_clear.c +++ b/src/gallium/drivers/zink/zink_clear.c @@ -574,7 +574,7 @@ zink_clear_buffer(struct pipe_context *pctx, - size is the number of bytes to fill, and must be either a multiple of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of the buffer */ - zink_resource_buffer_transfer_dst_barrier(ctx, res, offset, size); + zink_resource_buffer_transfer_dst_barrier(ctx, res, offset, size, false); VkCommandBuffer cmdbuf = zink_get_cmdbuf(ctx, NULL, res); zink_batch_reference_resource_rw(ctx, res, true); VKCTX(CmdFillBuffer)(cmdbuf, res->obj->buffer, offset, size, *(uint32_t*)clear_value); diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c index 57efe18739d..129a742d98b 100644 --- a/src/gallium/drivers/zink/zink_context.c +++ b/src/gallium/drivers/zink/zink_context.c @@ -5446,7 +5446,7 @@ zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zin /* unsync should only occur during subdata with staging resource */ if (!unsync) zink_screen(ctx->base.screen)->buffer_barrier(ctx, src, VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); - bool unordered_dst = zink_resource_buffer_transfer_dst_barrier(ctx, dst, dst_offset, size); + bool unordered_dst = zink_resource_buffer_transfer_dst_barrier(ctx, dst, dst_offset, size, unsync); bool can_unorder = unordered_dst && unordered_src && !ctx->no_reorder; VkCommandBuffer cmdbuf = unsync ? ctx->bs->unsynchronized_cmdbuf : can_unorder ? ctx->bs->reordered_cmdbuf : zink_get_cmdbuf(ctx, src, dst); @@ -5524,7 +5524,7 @@ zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, stru ctx->no_reorder++; VkImageLayout layout = screen->driver_workarounds.general_layout ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; screen->image_barrier(ctx, use_img, layout, VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); - zink_resource_buffer_transfer_dst_barrier(ctx, buf, box->x, box->width); + zink_resource_buffer_transfer_dst_barrier(ctx, buf, box->x, box->width, false); } VkBufferImageCopy region = {0}; diff --git a/src/gallium/drivers/zink/zink_context.h b/src/gallium/drivers/zink/zink_context.h index aeb1a1db728..84b64da8869 100644 --- a/src/gallium/drivers/zink/zink_context.h +++ b/src/gallium/drivers/zink/zink_context.h @@ -215,7 +215,7 @@ zink_check_valid_buffer_src_access(struct zink_context *ctx, struct zink_resourc void zink_resource_image_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned level, const struct pipe_box *box, bool unsync); bool -zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size); +zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size, bool unsync); void zink_synchronization_init(struct zink_screen *screen); void diff --git a/src/gallium/drivers/zink/zink_query.c b/src/gallium/drivers/zink/zink_query.c index 7d6526db7c9..77cb248daa7 100644 --- a/src/gallium/drivers/zink/zink_query.c +++ b/src/gallium/drivers/zink/zink_query.c @@ -791,7 +791,7 @@ copy_pool_results_to_buffer(struct zink_context *ctx, struct zink_query *query, /* if it's a single query that doesn't need special handling, we can copy it and be done */ zink_batch_reference_resource_rw(ctx, res, true); zink_resource_disable_unordered(res, true); - zink_resource_buffer_transfer_dst_barrier(ctx, res, offset, result_size); + zink_resource_buffer_transfer_dst_barrier(ctx, res, offset, result_size, false); util_range_add(&res->base.b, &res->valid_buffer_range, offset, offset + result_size); assert(query_id < NUM_QUERIES); ctx->bs->has_work = true; diff --git a/src/gallium/drivers/zink/zink_synchronization.cpp b/src/gallium/drivers/zink/zink_synchronization.cpp index b389933cf56..78760f2e5a5 100644 --- a/src/gallium/drivers/zink/zink_synchronization.cpp +++ b/src/gallium/drivers/zink/zink_synchronization.cpp @@ -544,7 +544,7 @@ zink_resource_image_transfer_dst_barrier(struct zink_context *ctx, struct zink_r } bool -zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size) +zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size, bool unsync) { if (res->obj->copies_need_reset) zink_resource_copies_reset(res); @@ -555,7 +555,12 @@ zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_ /* must barrier if something read the valid buffer range */ bool valid_read = (res->obj->access || res->obj->unordered_access) && util_ranges_intersect(&res->valid_buffer_range, offset, offset + size) && !can_unordered_write; - if (valid_read || zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics || zink_check_unordered_transfer_access(res, 0, &box)) { + /* the reordered cmdbuf runs after earlier batches only in submission order, which orders nothing: + * a range an unfinished earlier batch may still read needs a barrier */ + bool earlier_read = !unsync && !zink_resource_usage_matches(res, ctx->bs) && + !zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, + ZINK_RESOURCE_ACCESS_READ); + if (valid_read || earlier_read || zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics || zink_check_unordered_transfer_access(res, 0, &box)) { zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); unordered = res->obj->unordered_write; } else { @@ -567,6 +572,10 @@ zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_ res->obj->access = VK_ACCESS_TRANSFER_WRITE_BIT; res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; res->obj->ordered_access_is_copied = true; + /* this batch's reads of the bound buffer are ordered by the reordered cmdbuf's final barrier, but they must + * still reach obj->access for the next batch's barrier (earlier_read) */ + if (!unsync) + resource_check_defer_buffer_barrier(ctx, res, VK_PIPELINE_STAGE_TRANSFER_BIT); } } zink_resource_copy_box_add(ctx, res, 0, &box); @@ -630,6 +639,14 @@ zink_resource_memory_barrier(struct zink_context *ctx, struct zink_resource *res } if (!resource_needs_barrier(res, flags, pipeline, unordered)) return; + + /* An unfinished earlier batch's unordered accesses are still pending: keep them + * as ordered history before this batch's first barrier resets them, ordered or not. */ + if (!usage_matches && !completed) { + res->obj->access |= res->obj->unordered_access; + res->obj->access_stage |= res->obj->unordered_access_stage; + } + if (completed) { /* reset access on complete */ res->obj->access = VK_ACCESS_NONE; From 4960a8aa230174e0ed262e6bca95c0930daa67be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 12:32:07 +0100 Subject: [PATCH 091/109] zink: retry multisampled images without SAMPLED the device cannot sample GL renderbuffers carry PIPE_BIND_SAMPLER_VIEW, but their format and GL_MAX_SAMPLES are chosen without it, and get_image_usage_for_feats() adds SAMPLED to every image. Vulkan allows sampledImageDepthSampleCounts (and the other sampled-image limits) to be a subset of the framebuffer sample counts, so on such a device no usage candidate of a multisampled depth/stencil renderbuffer supports the requested count: glRenderbufferStorageMultisample(4) leaves GL_RENDERBUFFER_SAMPLES at 0 and the framebuffer incomplete, and WebGL contexts that ask for antialiasing are lost. When the sampled-image limits exclude the sample count, add a candidate without SAMPLED. zink_is_format_supported() already reports PIPE_BIND_SAMPLER_VIEW unsupported for that count, so nothing samples the image. Found on the Imagination A9 (iPad 5), which renders, loads and stores 4x depth/stencil but does not sample it. Assisted-by: LLM --- src/gallium/drivers/zink/zink_resource.c | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/gallium/drivers/zink/zink_resource.c b/src/gallium/drivers/zink/zink_resource.c index 4c04c864e7a..1fed7a003f1 100644 --- a/src/gallium/drivers/zink/zink_resource.c +++ b/src/gallium/drivers/zink/zink_resource.c @@ -857,6 +857,23 @@ add_2d_view_flag(struct zink_screen *screen, const struct pipe_resource *templ, } } +/* The sampled-image sample counts of a format, as zink_is_format_supported() checks them. */ +static VkSampleCountFlags +sampled_sample_counts(struct zink_screen *screen, enum pipe_format format) +{ + const VkPhysicalDeviceLimits *l = &screen->info.props.limits; + const struct util_format_description *desc = util_format_description(format); + if (util_format_is_depth_or_stencil(format)) { + VkSampleCountFlags s = ~0u; + if (util_format_has_depth(desc)) + s &= l->sampledImageDepthSampleCounts; + if (util_format_has_stencil(desc)) + s &= l->sampledImageStencilSampleCounts; + return s; + } + return util_format_is_pure_integer(format) ? l->sampledImageIntegerSampleCounts : l->sampledImageColorSampleCounts; +} + static unsigned build_usage_candidates(struct zink_screen *screen, const struct pipe_resource *templ, unsigned bind, VkImageTiling tiling, VkImageCreateFlags base_flags, bool extended, bool have_fmtlist, @@ -902,6 +919,17 @@ build_usage_candidates(struct zink_screen *screen, const struct pipe_resource *t n = emit_usage_candidates(c, n, tiling, usage, flags, false, have_fmtlist, always_mutable, max); } + /* Vulkan allows sampledImage*SampleCounts to be a subset of framebuffer*SampleCounts. GL + * renderbuffers carry PIPE_BIND_SAMPLER_VIEW, but their format and GL_MAX_SAMPLES are chosen + * without it, so a multisampled attachment the device cannot sample at this count would get no + * storage. Retry without SAMPLED: zink_is_format_supported() reports SAMPLER_VIEW unsupported for + * this sample count, so nothing samples the image. + */ + if (templ->nr_samples > 1 && (usage & VK_IMAGE_USAGE_SAMPLED_BIT) && + !(sampled_sample_counts(screen, templ->format) & (VkSampleCountFlags)templ->nr_samples)) + n = emit_usage_candidates(c, n, tiling, usage & ~VK_IMAGE_USAGE_SAMPLED_BIT, flags, false, have_fmtlist, + always_mutable, max); + return n; } From 465c42de643a1154317f234bc699f7a8a0a0d29a Mon Sep 17 00:00:00 2001 From: Mike Blumenkrantz Date: Sat, 31 Oct 2026 01:03:45 -0400 Subject: [PATCH 092/109] util/tc: don't full_copy an rp info that has ended a batch flush copies the recording rp info into the next batch. if that info already ended, the driver advances past it only at the end_call and stays one info behind. a following pass then inherits stale has_resolve and resolve targets affected tests: tc-renderpass-info-continue-clear Fixes: 2c5a2d8b398f ("util/tc: iterate the rp info more accurately during batch execution") Generated-by: LLM Part-of: (cherry picked from commit a7754e6cf1785b2562830e4369bcdb523c445a41) --- src/gallium/auxiliary/util/u_threaded_context.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index 114ad811853..6f160670fd7 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -587,6 +587,10 @@ tc_batch_flush(struct threaded_context *tc, bool full_copy) * renderpass info can only be accessed by its owner batch during execution */ if (tc->renderpass_info_recording) { + /* an rp ended by an end_call is not continued: the driver advances after + * the end_call, so a copy would leave it one info behind + */ + full_copy &= !tc_batch_rp_info(tc->renderpass_info_recording)->end_call; tc_batch_increment_renderpass_info(tc, next_id, full_copy); } From f68475f0461ca0115e3d3e2e1ab3ccddd2611a4f Mon Sep 17 00:00:00 2001 From: Mike Blumenkrantz Date: Sat, 31 Oct 2026 01:03:45 -0400 Subject: [PATCH 093/109] util/tc: only set end_call for a blit that ends the info a blit that overflows the batch on an info that already ended sees the ended flag change when the flush starts a fresh info. it then puts end_call on that info. only set it when the blit itself ended the info Fixes: 2c5a2d8b398f ("util/tc: iterate the rp info more accurately during batch execution") Generated-by: LLM Part-of: (cherry picked from commit 7adb6c6c0139f4df34c48c5134ea35477ccdd163) --- src/gallium/auxiliary/util/u_threaded_context.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index 6f160670fd7..72d52e7dea2 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -5148,7 +5148,7 @@ tc_blit(struct pipe_context *_pipe, const struct pipe_blit_info *info) tc_check_fb_access(tc, info->src.resource, info->dst.resource); } struct tc_blit_call *blit = tc_blit_enqueue(tc, info); - if (ended != tc->renderpass_info_recording->ended) + if (!ended && tc->renderpass_info_recording->ended) tc_batch_rp_info(tc->renderpass_info_recording)->end_call = blit; if (is_resolve) { tc->renderpass_info_recording->resolve_geometry.x = info->src.box.x; From 108617ff55782c6a1a0735cac7ebcb54d2b29d97 Mon Sep 17 00:00:00 2001 From: Mike Blumenkrantz Date: Sat, 31 Oct 2026 01:03:45 -0400 Subject: [PATCH 094/109] util/tc: advance the driver rp info before the first draw or clear batch_execute advanced the rp info after executing the first draw or clear that follows an end_call. that call ran on the info of the ended pass, but recording advances before it. hoist the advance ahead of the call switch affected tests: tc-renderpass-info-continue-draw Fixes: 2c5a2d8b398f ("util/tc: iterate the rp info more accurately during batch execution") Generated-by: LLM Part-of: (cherry picked from commit c901b964a7348b2002dd0c048dd86be4003bb12e) --- .../auxiliary/util/u_threaded_context.c | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index 72d52e7dea2..31d698ba373 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -5789,6 +5789,22 @@ batch_execute(struct tc_batch *batch, struct pipe_context *pipe, bool parsing) #endif TC_TRACE_SCOPE(call->call_id); + /* iterate before executing the first draw or clear after an rp end */ + if (parsing && increment_rp_info_on_draw_clear) { + switch (call->call_id) { + case TC_CALL_clear: + case TC_CALL_draw_single: + case TC_CALL_draw_multi: + case TC_CALL_draw_single_drawid: + case TC_CALL_draw_vstate_multi: + batch->tc->renderpass_info = incr_rp_info(batch->tc->renderpass_info); + increment_rp_info_on_draw_clear = false; + break; + default: + break; + } + } + /* This executes the call using a switch. */ switch (call->call_id) { #define CALL(name) \ @@ -5817,19 +5833,6 @@ batch_execute(struct tc_batch *batch, struct pipe_context *pipe, bool parsing) increment_rp_info_on_draw_clear = false; } else if (call == tc_batch_rp_info(batch->tc->renderpass_info)->end_call) { increment_rp_info_on_draw_clear = true; - } else if (increment_rp_info_on_draw_clear) { - switch (call->call_id) { - case TC_CALL_clear: - case TC_CALL_draw_single: - case TC_CALL_draw_multi: - case TC_CALL_draw_single_drawid: - case TC_CALL_draw_vstate_multi: - batch->tc->renderpass_info = incr_rp_info(batch->tc->renderpass_info); - increment_rp_info_on_draw_clear = false; - break; - default: - break; - } } } } From 06f52426df5d962159c7852150f13f71f34e5518 Mon Sep 17 00:00:00 2001 From: Mike Blumenkrantz Date: Sat, 31 Oct 2026 01:03:45 -0400 Subject: [PATCH 095/109] util/tc: advance the driver rp info for indirect and vstate draws recording advances the rp info for indirect and single vertex-state draws, but the driver side list lacks their call ids. the driver never advances for them affected tests: tc-renderpass-info-continue-draw-indirect tc-renderpass-info-continue-draw-list Fixes: 2c5a2d8b398f ("util/tc: iterate the rp info more accurately during batch execution") Generated-by: LLM Part-of: (cherry picked from commit 01f5bd5d1c762be6acf0f2d8e8e3429d7a547cde) --- src/gallium/auxiliary/util/u_threaded_context.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index 31d698ba373..2aa7944e23b 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -5796,6 +5796,8 @@ batch_execute(struct tc_batch *batch, struct pipe_context *pipe, bool parsing) case TC_CALL_draw_single: case TC_CALL_draw_multi: case TC_CALL_draw_single_drawid: + case TC_CALL_draw_indirect: + case TC_CALL_draw_vstate_single: case TC_CALL_draw_vstate_multi: batch->tc->renderpass_info = incr_rp_info(batch->tc->renderpass_info); increment_rp_info_on_draw_clear = false; From 88b2f6009a83f900be5d49915aba10f6f3a049fc Mon Sep 17 00:00:00 2001 From: Mike Blumenkrantz Date: Sat, 31 Oct 2026 01:03:45 -0400 Subject: [PATCH 096/109] util/tc: move the driver rp info advance call list into a helper the calls the driver advances its rp info for must mirror the recording side callers of tc_parse_draw and the clear handling. keep the list next to tc_parse_draw no functional changes Fixes: 2c5a2d8b398f ("util/tc: iterate the rp info more accurately during batch execution") Generated-by: LLM Part-of: (cherry picked from commit 941404c60357beb694259dd497ac682f5ce21d64) --- .../auxiliary/util/u_threaded_context.c | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index 2aa7944e23b..5c59c850a73 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -353,6 +353,27 @@ tc_check_fb_access(struct threaded_context *tc, struct pipe_resource *src, struc return false; } +/* calls that continue the recording info through tc_parse_draw or a clear: + * the driver must advance its info before executing the first one after an + * end_call, keep this in sync with their callers + */ +static ALWAYS_INLINE bool +tc_call_may_advance_rp_info(enum tc_call_id id) +{ + switch (id) { + case TC_CALL_clear: + case TC_CALL_draw_single: + case TC_CALL_draw_multi: + case TC_CALL_draw_single_drawid: + case TC_CALL_draw_indirect: + case TC_CALL_draw_vstate_single: + case TC_CALL_draw_vstate_multi: + return true; + default: + return false; + } +} + /* update metadata at draw time */ static void tc_parse_draw(struct threaded_context *tc) @@ -5790,21 +5811,10 @@ batch_execute(struct tc_batch *batch, struct pipe_context *pipe, bool parsing) TC_TRACE_SCOPE(call->call_id); /* iterate before executing the first draw or clear after an rp end */ - if (parsing && increment_rp_info_on_draw_clear) { - switch (call->call_id) { - case TC_CALL_clear: - case TC_CALL_draw_single: - case TC_CALL_draw_multi: - case TC_CALL_draw_single_drawid: - case TC_CALL_draw_indirect: - case TC_CALL_draw_vstate_single: - case TC_CALL_draw_vstate_multi: - batch->tc->renderpass_info = incr_rp_info(batch->tc->renderpass_info); - increment_rp_info_on_draw_clear = false; - break; - default: - break; - } + if (parsing && increment_rp_info_on_draw_clear && + tc_call_may_advance_rp_info(call->call_id)) { + batch->tc->renderpass_info = incr_rp_info(batch->tc->renderpass_info); + increment_rp_info_on_draw_clear = false; } /* This executes the call using a switch. */ From 382464090e00d8ddb37aa90d35c8591340f9bff3 Mon Sep 17 00:00:00 2001 From: Mike Blumenkrantz Date: Sat, 31 Oct 2026 01:03:45 -0400 Subject: [PATCH 097/109] util/tc: write the resolve geometry before enqueueing the resolve tc_blit_enqueue can flush the batch. if the info already ended, the flush starts a fresh info and the geometry lands there while has_resolve and resolve[] stay on the old one. only reachable by repeated blits to the window system resolve target. there is no test for it Fixes: 4f79fe382c7a ("util/tc: store resolve geometry to rp info") Generated-by: LLM Part-of: (cherry picked from commit d2a56c6bca316401ae6a8d9281d9addc7550e16c) --- src/gallium/auxiliary/util/u_threaded_context.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index 5c59c850a73..d943fa60943 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -5168,9 +5168,9 @@ tc_blit(struct pipe_context *_pipe, const struct pipe_blit_info *info) } else if (tc->in_renderpass) { tc_check_fb_access(tc, info->src.resource, info->dst.resource); } - struct tc_blit_call *blit = tc_blit_enqueue(tc, info); - if (!ended && tc->renderpass_info_recording->ended) - tc_batch_rp_info(tc->renderpass_info_recording)->end_call = blit; + /* write the geometry before the enqueue, which can flush the batch and + * change the recording info + */ if (is_resolve) { tc->renderpass_info_recording->resolve_geometry.x = info->src.box.x; tc->renderpass_info_recording->resolve_geometry.y = info->src.box.y; @@ -5178,8 +5178,13 @@ tc_blit(struct pipe_context *_pipe, const struct pipe_blit_info *info) tc->renderpass_info_recording->resolve_geometry.width = info->src.box.width; tc->renderpass_info_recording->resolve_geometry.height = info->src.box.height; tc->renderpass_info_recording->resolve_geometry.depth = info->src.box.depth; - blit->base.call_id = TC_CALL_resolve; } + + struct tc_blit_call *blit = tc_blit_enqueue(tc, info); + if (!ended && tc->renderpass_info_recording->ended) + tc_batch_rp_info(tc->renderpass_info_recording)->end_call = blit; + if (is_resolve) + blit->base.call_id = TC_CALL_resolve; } struct tc_generate_mipmap { From c7dec20ec9d128575164a1d5cc25ddb99a2073fe Mon Sep 17 00:00:00 2001 From: Mike Blumenkrantz Date: Sat, 31 Oct 2026 01:03:45 -0400 Subject: [PATCH 098/109] util/tc: don't merge a resolve into a different target tc_blit merges one color and one depth resolve into the rp info. a second resolve of the same source into another target hits an assert, and in a release build overwrites resolve[] and loses the first resolve. only merge into an unset or identical target a color resolve is not merged either when the framebuffer has a window system resolve target. zink resolves into that target first and ignores the merged resolve[0] affected tests: tc-renderpass-info-resolve-twice Fixes: a71b6ac41a19 ("tc: also inline depth resolves") Generated-by: LLM Part-of: (cherry picked from commit 4104830703b8e4b1d965c3741aa04909f004c69d) --- src/gallium/auxiliary/util/u_threaded_context.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index d943fa60943..aafbb748cb7 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -5149,12 +5149,15 @@ tc_blit(struct pipe_context *_pipe, const struct pipe_blit_info *info) tc->renderpass_info_recording->has_resolve = true; tc->renderpass_info_recording->ended = true; is_resolve = true; - } else if (src == info->src.resource) { + } else if (src == info->src.resource && (is_depth || !tc->fb_resolve) && + (!tc->renderpass_info_recording->resolve[is_depth] || + tc->renderpass_info_recording->resolve[is_depth] == info->dst.resource)) { #if TC_DEBUG >= 3 tc_printf("RESOLVE MERGE"); #endif - /* only optimizing out one depth and one color resolve */ - assert(!tc->renderpass_info_recording->resolve[is_depth] || tc->renderpass_info_recording->resolve[is_depth] == info->dst.resource); + /* only optimizing out one depth and one color resolve: a resolve into + * any other target is a normal blit + */ pipe_resource_reference(&tc->renderpass_info_recording->resolve[is_depth], info->dst.resource); tc_set_resource_batch_usage(tc, info->dst.resource); if (tc->renderpass_info_recording->has_resolve) { From 65d103d62b0da87dd514defb68343995d8f6dfea Mon Sep 17 00:00:00 2001 From: Mike Blumenkrantz Date: Sat, 31 Oct 2026 01:03:45 -0400 Subject: [PATCH 099/109] util/tc: parse the rp info for mesh draws tc_draw_mesh_tasks never calls tc_parse_draw. a mesh draw after a resolve does not start a new recording info and the driver does not advance for it. the draw then resolves into the previous pass's target affected tests: ext_mesh_shader-continue-after-resolve Fixes: 2c5a2d8b398f ("util/tc: iterate the rp info more accurately during batch execution") Generated-by: LLM Part-of: (cherry picked from commit 6243b56474dea3c5d9c7132eee99cdddf74f4c08) --- src/gallium/auxiliary/util/u_threaded_context.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index aafbb748cb7..8e1a2ca8ff1 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -368,6 +368,7 @@ tc_call_may_advance_rp_info(enum tc_call_id id) case TC_CALL_draw_indirect: case TC_CALL_draw_vstate_single: case TC_CALL_draw_vstate_multi: + case TC_CALL_draw_mesh_tasks: return true; default: return false; @@ -5740,6 +5741,9 @@ tc_draw_mesh_tasks(struct pipe_context *_pipe, const struct pipe_grid_info *info) { struct threaded_context *tc = threaded_context(_pipe); + if (tc->options.parse_renderpass_info) + tc_parse_draw(tc); + struct tc_draw_mesh_tasks *p = tc_add_call(tc, TC_CALL_draw_mesh_tasks, tc_draw_mesh_tasks); From dc7f2e3e0bfef7b7a06dc23569604097fd24c990 Mon Sep 17 00:00:00 2001 From: Mike Blumenkrantz Date: Sat, 31 Oct 2026 01:03:45 -0400 Subject: [PATCH 100/109] util/tc: don't end the rp info without an end_call the driver advances its info at set_framebuffer_state, a flush call, and an end_call. an internal unsynchronized tc_texture_map, a large tc_texture_subdata that went unsynchronized, and a deferred flush without a fence ended the info with none of these. the driver stays one info behind skip the framebuffer access check for an unsynchronized map. sync a subdata that ended the info so that tc_sync resets it. make the deferred flush the end_call of the info it ends Fixes: 2c5a2d8b398f ("util/tc: iterate the rp info more accurately during batch execution") Generated-by: LLM Part-of: (cherry picked from commit 543a8fbb88b9cc8c5de174dce815caedfdf30edb) --- src/gallium/auxiliary/util/u_threaded_context.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index 8e1a2ca8ff1..4b2cd10100f 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -3055,7 +3055,9 @@ tc_texture_map(struct pipe_context *_pipe, bool is_internal_unsynchronized = usage & PIPE_MAP_UNSYNCHRONIZED && resource->flags & PIPE_RESOURCE_FLAG_MAP_UNSYNCHRONIZED; - if (tc->options.parse_renderpass_info && TC_RENDERPASS_INFO_HAS_WORK(tc->renderpass_info_recording->data32[0])) + /* an unsynchronized map enqueues no call that could be the end_call */ + if (tc->options.parse_renderpass_info && !is_internal_unsynchronized && + TC_RENDERPASS_INFO_HAS_WORK(tc->renderpass_info_recording->data32[0])) tc_check_fb_access(tc, NULL, resource); /* this can only be hit for internal use */ @@ -3760,7 +3762,8 @@ tc_texture_subdata(struct pipe_context *_pipe, struct pipe_context *pipe = tc->pipe; struct threaded_resource *tres = threaded_resource(resource); unsigned unsync_usage = TC_TRANSFER_MAP_THREADED_UNSYNC | PIPE_MAP_UNSYNCHRONIZED | PIPE_MAP_WRITE; - bool can_unsync = !tc_resource_batch_usage_test_busy(tc, resource) && + /* if the info ended here, sync so that tc_sync resets it */ + bool can_unsync = !ended && !tc_resource_batch_usage_test_busy(tc, resource) && tc->options.is_resource_busy && !tc->options.is_resource_busy(tc->pipe->screen, tres->latest, usage | unsync_usage); @@ -4109,10 +4112,13 @@ tc_flush(struct pipe_context *_pipe, struct pipe_fence_handle **fence, if (!tc->options.driver_calls_flush_notify) p_atomic_set(&tc->num_busy_replies, 0); + bool ended_here = false; if (!deferred || !fence) { tc->in_renderpass = false; - if (tc->renderpass_info_recording) + if (tc->renderpass_info_recording) { + ended_here = !tc->renderpass_info_recording->ended; tc->renderpass_info_recording->ended = true; + } } if (async && tc->options.create_fence) { @@ -4138,6 +4144,9 @@ tc_flush(struct pipe_context *_pipe, struct pipe_fence_handle **fence, if (deferred) { /* these have identical fields */ p = (struct tc_flush_call *)tc_add_call(tc, TC_CALL_flush_deferred, tc_flush_deferred_call); + /* the driver has no other call to advance after */ + if (ended_here) + tc_batch_rp_info(tc->renderpass_info_recording)->end_call = p; } else { p = tc_add_call(tc, TC_CALL_flush, tc_flush_call); p->tc = tc; From b00232efe7c40ce5ba674a49b6f7dce72a07c288 Mon Sep 17 00:00:00 2001 From: Mike Blumenkrantz Date: Sat, 31 Oct 2026 01:03:45 -0400 Subject: [PATCH 101/109] util/tc: reset the new rp info fence before linking it tc_batch_increment_renderpass_info signals the old info before it resets the ready fence of the new one. in between, the driver follows old->next to the new info and sees the fence still signaled from its previous use. it reads the info while recording still writes it. a resolve merged after a flush can then be lost reset the fence before the full_copy block affected tests: tc-renderpass-info-continue-depth-resolve was flaky before Fixes: 07017aa137b9 ("util/tc: implement renderpass tracking") Generated-by: LLM Part-of: (cherry picked from commit 2a8ee7ba4a88dc7955518fc62d1f394007e8db73) --- src/gallium/auxiliary/util/u_threaded_context.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index 4b2cd10100f..b25d6aa341a 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -279,6 +279,10 @@ tc_batch_increment_renderpass_info(struct threaded_context *tc, unsigned batch_i batch->renderpass_info_idx++; tc_batch_renderpass_infos_resize(tc, batch); tc_info = batch->renderpass_infos.data; + /* the driver can reach this info through the previous one as soon as that + * is signaled: it has to wait on this one by then + */ + util_queue_fence_reset(&tc_info[batch->renderpass_info_idx].ready); if (full_copy) { /* this should only be called when changing batches */ @@ -314,7 +318,6 @@ tc_batch_increment_renderpass_info(struct threaded_context *tc, unsigned batch_i assert(!full_copy || !tc->renderpass_info_recording || tc_batch_rp_info(tc->renderpass_info_recording)->next); /* signal existing info since it will not be used anymore */ tc_signal_renderpass_info_ready(tc); - util_queue_fence_reset(&tc_info[batch->renderpass_info_idx].ready); /* guard against deadlock scenario */ assert(tc->renderpass_info_recording != &tc_info[batch->renderpass_info_idx].info); tc_info[batch->renderpass_info_idx].end_call = NULL; From d5e6d73ec98990e6075a92e158dd469f16fa6655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 21:52:19 +0100 Subject: [PATCH 102/109] gallium/dri: let the swap throttle keep more than one frame in flight Add a screen capability for the number of frames retained by the swap throttle. Keep a bounded ring of fences in the DRI frontend and share the throttle helper with kopper. Default to one frame, preserving the previous-fence wait for every driver. Drivers can opt into a deeper queue without changing the frontend's fence ownership or disabling throttling. Assisted-by: LLM --- docs/gallium/screen.rst | 3 ++ src/gallium/auxiliary/util/u_screen.c | 1 + src/gallium/frontends/dri/dri_drawable.c | 36 ++++++++++++++++++------ src/gallium/frontends/dri/dri_drawable.h | 11 +++++++- src/gallium/frontends/dri/dri_screen.c | 2 ++ src/gallium/frontends/dri/dri_screen.h | 1 + src/gallium/frontends/dri/kopper.c | 8 +----- src/gallium/include/pipe/p_defines.h | 1 + 8 files changed, 46 insertions(+), 17 deletions(-) diff --git a/docs/gallium/screen.rst b/docs/gallium/screen.rst index 64d8338394e..aff7a4f820f 100644 --- a/docs/gallium/screen.rst +++ b/docs/gallium/screen.rst @@ -550,6 +550,9 @@ Capability about the features and limits of the driver/GPU. * ``pipe_caps.image_store_formatted``: True if a format for image stores does not need to be specified in the shader IR * ``pipe_caps.throttle``: Whether or not gallium frontends should throttle pipe_context execution. 0 = throttling is disabled. +* ``pipe_caps.throttle_frames``: How many frames a throttling frontend lets be + in flight: a swap waits for the frame that many swaps back. 0 and 1 wait for + the previous frame. The default is 1; the DRI frontend supports up to 3. * ``pipe_caps.dmabuf``: Whether Linux DMABUF handles are supported by resource_from_handle and resource_get_handle. Possible bit field values: diff --git a/src/gallium/auxiliary/util/u_screen.c b/src/gallium/auxiliary/util/u_screen.c index d1399bef84d..1756a0bcabf 100644 --- a/src/gallium/auxiliary/util/u_screen.c +++ b/src/gallium/auxiliary/util/u_screen.c @@ -174,6 +174,7 @@ u_init_pipe_screen_caps(struct pipe_screen *pscreen, int accel) VARYING_BIT_BOUNDING_BOX1; caps->throttle = true; + caps->throttle_frames = 1; #if defined(HAVE_LIBDRM) && (DETECT_OS_LINUX || DETECT_OS_BSD || DETECT_OS_MANAGARM) if (pscreen->get_screen_fd) { diff --git a/src/gallium/frontends/dri/dri_drawable.c b/src/gallium/frontends/dri/dri_drawable.c index bdf31f5cdd9..d955111b1dd 100644 --- a/src/gallium/frontends/dri/dri_drawable.c +++ b/src/gallium/frontends/dri/dri_drawable.c @@ -264,8 +264,9 @@ dri_destroy_drawable(struct dri_drawable *drawable) for (i = 0; i < ST_ATTACHMENT_COUNT; i++) pipe_resource_reference(&drawable->msaa_textures[i], NULL); - screen->base.screen->fence_reference(screen->base.screen, - &drawable->throttle_fence, NULL); + for (i = 0; i < DRI_THROTTLE_MAX_FRAMES; i++) + screen->base.screen->fence_reference(screen->base.screen, + &drawable->throttle_fence[i], NULL); /* Notify the st manager that this drawable is no longer valid */ st_api_destroy_drawable(&drawable->base); @@ -686,17 +687,11 @@ dri_flush(struct dri_context *ctx, (reason == __DRI2_THROTTLE_SWAPBUFFER || reason == __DRI2_THROTTLE_FLUSHFRONT)) { - struct pipe_screen *screen = drawable->screen->base.screen; struct pipe_fence_handle *new_fence = NULL; st_context_flush(st, flush_flags, &new_fence, args.ctx ? notify_before_flush_cb : NULL, &args); - /* throttle on the previous fence */ - if (drawable->throttle_fence) { - screen->fence_finish(screen, NULL, drawable->throttle_fence, OS_TIMEOUT_INFINITE); - screen->fence_reference(screen, &drawable->throttle_fence, NULL); - } - drawable->throttle_fence = new_fence; + dri_throttle_frame(drawable, new_fence); } else if (flags & (__DRI2_FLUSH_DRAWABLE | __DRI2_FLUSH_CONTEXT)) { st_context_flush(st, flush_flags, NULL, args.ctx ? notify_before_flush_cb : NULL, &args); @@ -739,6 +734,29 @@ dri_flush_drawable(struct dri_drawable *dPriv) dri_flush(ctx, dPriv, __DRI2_FLUSH_DRAWABLE, -1); } +/** + * Throttle on the frame screen->throttle_frames swaps back (the previous one + * by default), then keep new_fence (a reference this takes over) for a later + * swap. With throttle_frames N, up to N submitted frames stay in flight while + * the application records the next one. + */ +void +dri_throttle_frame(struct dri_drawable *drawable, + struct pipe_fence_handle *new_fence) +{ + struct pipe_screen *screen = drawable->screen->base.screen; + const unsigned frames = drawable->screen->throttle_frames; + struct pipe_fence_handle **slot = + &drawable->throttle_fence[drawable->throttle_slot % frames]; + + if (*slot) { + screen->fence_finish(screen, NULL, *slot, OS_TIMEOUT_INFINITE); + screen->fence_reference(screen, slot, NULL); + } + *slot = new_fence; + drawable->throttle_slot = (drawable->throttle_slot + 1) % frames; +} + /** * dri_throttle - A DRI2ThrottleExtension throttling function. */ diff --git a/src/gallium/frontends/dri/dri_drawable.h b/src/gallium/frontends/dri/dri_drawable.h index 58be595e1a6..be6d67b0289 100644 --- a/src/gallium/frontends/dri/dri_drawable.h +++ b/src/gallium/frontends/dri/dri_drawable.h @@ -33,6 +33,9 @@ #include "frontend/api.h" #include "dri_util.h" +/* most frames a throttled swap lets be in flight (pipe_caps.throttle_frames) */ +#define DRI_THROTTLE_MAX_FRAMES 3 + #ifdef VK_USE_PLATFORM_XCB_KHR #include #endif @@ -59,7 +62,9 @@ struct dri_drawable unsigned int texture_mask, texture_stamp; int swap_interval; - struct pipe_fence_handle *throttle_fence; + /* the last frames' fences, one slot per frame in flight (dri_throttle_frame()) */ + struct pipe_fence_handle *throttle_fence[DRI_THROTTLE_MAX_FRAMES]; + unsigned throttle_slot; bool flushing; /* prevents recursion in dri_flush */ /** @@ -137,6 +142,10 @@ dri_flush(struct dri_context *ctx, unsigned flags, enum __DRI2throttleReason reason); +void +dri_throttle_frame(struct dri_drawable *drawable, + struct pipe_fence_handle *new_fence); + void dri_flush_drawable(struct dri_drawable *dPriv); diff --git a/src/gallium/frontends/dri/dri_screen.c b/src/gallium/frontends/dri/dri_screen.c index cb43894b194..ce3b893b0c4 100644 --- a/src/gallium/frontends/dri/dri_screen.c +++ b/src/gallium/frontends/dri/dri_screen.c @@ -30,6 +30,7 @@ */ #include "dri_screen.h" +#include "dri_drawable.h" #include "dri_context.h" #include "dri_helpers.h" @@ -636,6 +637,7 @@ dri_init_screen(struct dri_screen *screen, screen->target = PIPE_TEXTURE_RECT; screen->throttle = pscreen->caps.throttle; + screen->throttle_frames = CLAMP(pscreen->caps.throttle_frames, 1, DRI_THROTTLE_MAX_FRAMES); if (pscreen->caps.device_protected_context) screen->has_protected_context = true; screen->has_reset_status_query = pscreen->caps.device_reset_status_query; diff --git a/src/gallium/frontends/dri/dri_screen.h b/src/gallium/frontends/dri/dri_screen.h index 0f345b4cc02..ae96a80a940 100644 --- a/src/gallium/frontends/dri/dri_screen.h +++ b/src/gallium/frontends/dri/dri_screen.h @@ -72,6 +72,7 @@ struct dri_screen unsigned int api_mask; bool throttle; + unsigned throttle_frames; /* 1..DRI_THROTTLE_MAX_FRAMES */ bool dmabuf_import; bool has_multibuffer; diff --git a/src/gallium/frontends/dri/kopper.c b/src/gallium/frontends/dri/kopper.c index e4700c82a76..3320e941d6c 100644 --- a/src/gallium/frontends/dri/kopper.c +++ b/src/gallium/frontends/dri/kopper.c @@ -383,7 +383,6 @@ kopper_flush_frontbuffer(struct dri_context *ctx, if (ptex) { ctx->st->pipe->flush_resource(ctx->st->pipe, drawable->textures[ST_ATTACHMENT_FRONT_LEFT]); - struct pipe_screen *screen = drawable->screen->base.screen; struct st_context *st; struct pipe_fence_handle *new_fence = NULL; @@ -392,12 +391,7 @@ kopper_flush_frontbuffer(struct dri_context *ctx, st_context_flush(st, ST_FLUSH_FRONT, &new_fence, NULL, NULL); drawable->flushing = false; - /* throttle on the previous fence */ - if (drawable->throttle_fence) { - screen->fence_finish(screen, NULL, drawable->throttle_fence, OS_TIMEOUT_INFINITE); - screen->fence_reference(screen, &drawable->throttle_fence, NULL); - } - drawable->throttle_fence = new_fence; + dri_throttle_frame(drawable, new_fence); kopper_copy_to_front(st->pipe, ctx->draw, ptex, 0, NULL); } diff --git a/src/gallium/include/pipe/p_defines.h b/src/gallium/include/pipe/p_defines.h index ac8a73ab19e..a1871e3de7d 100644 --- a/src/gallium/include/pipe/p_defines.h +++ b/src/gallium/include/pipe/p_defines.h @@ -1030,6 +1030,7 @@ struct pipe_caps { bool image_load_formatted; bool image_store_formatted; bool throttle; + unsigned throttle_frames; bool cl_gl_sharing; bool prefer_compute_for_multimedia; bool fragment_shader_interlock; From f94c885adf430987ade15a418f9f31e4aa1bd263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Sun, 15 Nov 2026 21:52:57 +0100 Subject: [PATCH 103/109] zink: keep two frames in flight on Imagination GPUs Let the swap throttle retain two frames on the open-source Imagination Vulkan driver. Tile-based GPUs can overlap one frame's geometry phase with the previous frame's fragment phase when both are queued. Keep other drivers at one frame and use a driver-ID table for future tuning. Add zink_frames_in_flight as a driconf option: 0 selects the driver default and 1..3 select an explicit depth. The same-named environment variable provides an override, including depth 1 for latency-sensitive applications. Request enough kopper swapchain images for the extra queued frames, within the surface's maximum image count. This costs one extra image and can add a frame of latency when GPU-bound at the new default. Assisted-by: LLM --- docs/drivers/zink.rst | 12 ++++++++++++ src/gallium/drivers/zink/driinfo_zink.h | 1 + src/gallium/drivers/zink/zink_kopper.c | 10 ++++++++++ src/gallium/drivers/zink/zink_screen.c | 25 +++++++++++++++++++++++++ src/gallium/drivers/zink/zink_types.h | 1 + 5 files changed, 49 insertions(+) diff --git a/docs/drivers/zink.rst b/docs/drivers/zink.rst index 3ecb3665ddf..1ef1641cbd0 100644 --- a/docs/drivers/zink.rst +++ b/docs/drivers/zink.rst @@ -336,6 +336,18 @@ changing the descriptor manager may improve performance: ``db`` Use EXT_descriptor_buffer when possible. +The ``zink_frames_in_flight`` driconf option controls the swap throttle depth. +It accepts integers from 0 to 3. The default, 0, selects two frames for +``VK_DRIVER_ID_IMAGINATION_OPEN_SOURCE_MESA`` and one frame for other drivers. +Values 1 through 3 explicitly select that many frames. Mesa also accepts the +same-named environment variable, for example ``zink_frames_in_flight=1`` to +restore the previous-frame throttle. + +Two frames allow a tile-based GPU to overlap geometry and fragment work from +successive frames. When GPU-bound, this can add one frame of input latency +and requires one more swapchain image per window (12 MiB at 2048x1536 RGBA8), +subject to the surface's maximum image count. + Debugging --------- diff --git a/src/gallium/drivers/zink/driinfo_zink.h b/src/gallium/drivers/zink/driinfo_zink.h index cdf1596cce0..a62aa9ea039 100644 --- a/src/gallium/drivers/zink/driinfo_zink.h +++ b/src/gallium/drivers/zink/driinfo_zink.h @@ -6,6 +6,7 @@ DRI_CONF_SECTION_DEBUG DRI_CONF_SECTION_END DRI_CONF_SECTION_PERFORMANCE +DRI_CONF_OPT_I(zink_frames_in_flight, 0, 0, 3, "Frames in flight (0 = driver default)") DRI_CONF_MESA_GLTHREAD_DRIVER(true) DRI_CONF_OPT_B(zink_shader_object_enable, false, "Enable support for EXT_shader_object") DRI_CONF_SECTION_END diff --git a/src/gallium/drivers/zink/zink_kopper.c b/src/gallium/drivers/zink/zink_kopper.c index c5e5d3ad586..fd67aff701f 100644 --- a/src/gallium/drivers/zink/zink_kopper.c +++ b/src/gallium/drivers/zink/zink_kopper.c @@ -320,6 +320,16 @@ kopper_CreateSwapchain(struct zink_screen *screen, struct kopper_displaytarget * } cswap->scci.presentMode = cdt->present_mode; cswap->scci.minImageCount = cdt->caps.minImageCount; + /* Each frame in flight holds an image, besides the one being displayed and + * the one being drawn. Otherwise acquisition can throttle the queue to one + * frame in flight. + */ + if (screen->frames_in_flight > 1) { + uint32_t want = MAX2(cswap->scci.minImageCount, screen->frames_in_flight + 2); + if (cdt->caps.maxImageCount) + want = MIN2(want, cdt->caps.maxImageCount); + cswap->scci.minImageCount = want; + } cswap->scci.preTransform = cdt->caps.currentTransform; if (cdt->formats[1]) cswap->scci.pNext = &cdt->format_list; diff --git a/src/gallium/drivers/zink/zink_screen.c b/src/gallium/drivers/zink/zink_screen.c index fed6f56a379..ca3fc5286ed 100644 --- a/src/gallium/drivers/zink/zink_screen.c +++ b/src/gallium/drivers/zink/zink_screen.c @@ -733,6 +733,7 @@ zink_init_screen_caps(struct zink_screen *screen) u_init_pipe_screen_caps(&screen->base, screen->is_cpu ? 0 : 1); caps->r11g11b10f_requires_color_buffer_float = screen->is_g5p; + caps->throttle_frames = screen->frames_in_flight; /* Vulkan spec says the builtins count, so make GL count them towards * link-failing. @@ -3749,6 +3750,26 @@ sort_mem_types(struct zink_screen *screen, enum zink_heap heap) } } +static unsigned +zink_default_frames_in_flight(VkDriverId driver_id) +{ + /* Tile-based GPUs can overlap one frame's geometry phase with the previous + * frame's fragment phase. Add drivers here as measurements justify it. + */ + static const struct { + VkDriverId driver_id; + unsigned frames; + } defaults[] = { + { VK_DRIVER_ID_IMAGINATION_OPEN_SOURCE_MESA, 2 }, + }; + + for (unsigned i = 0; i < ARRAY_SIZE(defaults); i++) { + if (defaults[i].driver_id == driver_id) + return defaults[i].frames; + } + return 1; +} + static struct zink_screen * zink_internal_create_screen(const struct pipe_screen_config *config, int64_t dev_major, int64_t dev_minor, uint64_t adapter_luid) { @@ -3869,6 +3890,7 @@ zink_internal_create_screen(const struct pipe_screen_config *config, int64_t dev screen->is_g5p = screen->info.driver_props.driverID == VK_DRIVER_ID_IMAGINATION_OPEN_SOURCE_MESA && !strcmp(screen->info.driver_props.driverName, "Mesa PowerVR G5P"); screen->precise_sync = screen->is_g5p; + screen->frames_in_flight = zink_default_frames_in_flight(screen->info.driver_props.driverID); /* after the device is known: G5P has driconf defaults of its own (device "g5p") */ if (config) { @@ -3879,6 +3901,9 @@ zink_internal_create_screen(const struct pipe_screen_config *config, int64_t dev //screen->driconf.inline_uniforms = driQueryOptionb(config->options, "radeonsi_inline_uniforms"); screen->driconf.emulate_point_smooth = driQueryOptionb(config->options, "zink_emulate_point_smooth"); screen->driconf.zink_shader_object_enable = driQueryOptionb(config->options, "zink_shader_object_enable"); + unsigned frames = driQueryOptioni(config->options, "zink_frames_in_flight"); + if (frames) + screen->frames_in_flight = frames; } if (!screen->info.rb2_feats.nullDescriptor) { diff --git a/src/gallium/drivers/zink/zink_types.h b/src/gallium/drivers/zink/zink_types.h index 436855b8be4..8533ae6ff6c 100644 --- a/src/gallium/drivers/zink/zink_types.h +++ b/src/gallium/drivers/zink/zink_types.h @@ -1503,6 +1503,7 @@ struct zink_video_format_prop { struct zink_screen { /* Must stay first: zink_screen() casts a pipe_screen pointer directly. */ struct pipe_screen base; + unsigned frames_in_flight; /* pipe_caps.throttle_frames */ bool precise_sync; bool is_g5p; From 0531d86f7fc1952799de4681e525c1ac42022a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 16:22:56 +0100 Subject: [PATCH 104/109] vulkan/wsi/wayland: support pre-rotated swapchains Track output transforms and expose the single output orientation in surface capabilities. Apply the selected pre-transform when presenting, including when a replacement swapchain returns to identity. Assisted-by: LLM --- src/vulkan/wsi/wsi_common_wayland.c | 204 +++++++++++++++++++++++++--- 1 file changed, 187 insertions(+), 17 deletions(-) diff --git a/src/vulkan/wsi/wsi_common_wayland.c b/src/vulkan/wsi/wsi_common_wayland.c index 8a1e86f432e..14250de7635 100644 --- a/src/vulkan/wsi/wsi_common_wayland.c +++ b/src/vulkan/wsi/wsi_common_wayland.c @@ -99,12 +99,78 @@ struct dmabuf_feedback { struct dmabuf_feedback_tranche pending_tranche; }; +struct wsi_wl_output { + struct wl_list link; + struct wl_output *output; + uint32_t name; + enum wl_output_transform transform; +}; + +/* Vulkan VkSurfaceTransformFlagBitsKHR defines clockwise rotations, while + * wayland.xml wl_output.transform defines counter-clockwise rotations. Both + * VkSwapchainCreateInfoKHR.preTransform and wl_surface.set_buffer_transform + * describe the transform already applied to the image content. + * https://docs.vulkan.org/refpages/latest/refpages/source/VkSurfaceTransformFlagBitsKHR.html + * https://gitlab.freedesktop.org/wayland/wayland/-/blob/main/protocol/wayland.xml + */ +static VkSurfaceTransformFlagBitsKHR +wl_transform_to_vk(enum wl_output_transform transform) +{ + switch (transform) { + case WL_OUTPUT_TRANSFORM_NORMAL: + return VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; + case WL_OUTPUT_TRANSFORM_90: + return VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR; + case WL_OUTPUT_TRANSFORM_180: + return VK_SURFACE_TRANSFORM_ROTATE_180_BIT_KHR; + case WL_OUTPUT_TRANSFORM_270: + return VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR; + case WL_OUTPUT_TRANSFORM_FLIPPED: + return VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_BIT_KHR; + case WL_OUTPUT_TRANSFORM_FLIPPED_90: + return VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_270_BIT_KHR; + case WL_OUTPUT_TRANSFORM_FLIPPED_180: + return VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_180_BIT_KHR; + case WL_OUTPUT_TRANSFORM_FLIPPED_270: + return VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_90_BIT_KHR; + default: + UNREACHABLE("Invalid Wayland transform"); + } +} + +static enum wl_output_transform +vk_transform_to_wl(VkSurfaceTransformFlagBitsKHR transform) +{ + switch (transform) { + case VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR: + return WL_OUTPUT_TRANSFORM_NORMAL; + case VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR: + return WL_OUTPUT_TRANSFORM_90; + case VK_SURFACE_TRANSFORM_ROTATE_180_BIT_KHR: + return WL_OUTPUT_TRANSFORM_180; + case VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR: + return WL_OUTPUT_TRANSFORM_270; + case VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_BIT_KHR: + return WL_OUTPUT_TRANSFORM_FLIPPED; + case VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_270_BIT_KHR: + return WL_OUTPUT_TRANSFORM_FLIPPED_90; + case VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_180_BIT_KHR: + return WL_OUTPUT_TRANSFORM_FLIPPED_180; + case VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_90_BIT_KHR: + return WL_OUTPUT_TRANSFORM_FLIPPED_270; + default: + UNREACHABLE("Invalid Vulkan surface transform"); + } +} + struct wsi_wl_display { /* The real wl_display */ struct wl_display *wl_display; /* Actually a proxy wrapper around the event queue */ struct wl_display *wl_display_wrapper; struct wl_event_queue *queue; + struct wl_registry *registry; + struct wl_list outputs; struct wl_fixes *wl_fixes; struct wl_shm *wl_shm; @@ -179,6 +245,7 @@ struct wsi_wl_surface { VkIcdSurfaceWayland base; unsigned int chain_count; + enum wl_output_transform buffer_transform; struct wsi_wl_swapchain *chain; struct loader_wayland_surface wayland_surface; @@ -201,6 +268,7 @@ struct wsi_wl_swapchain { struct wl_callback *frame; VkExtent2D extent; + enum wl_output_transform buffer_transform; VkFormat vk_format; enum wsi_wl_buffer_type buffer_type; uint32_t drm_format; @@ -1497,12 +1565,80 @@ static const struct wp_presentation_listener presentation_listener = { #define MESA_WL_FIXES_VERSION 1 #endif +static void +output_handle_geometry(void *data, struct wl_output *output, + int32_t x, int32_t y, int32_t physical_width, + int32_t physical_height, int32_t subpixel, + const char *make, const char *model, int32_t transform) +{ + struct wsi_wl_output *wl_output = data; + wl_output->transform = transform; +} + +static void +output_handle_mode(void *data, struct wl_output *output, uint32_t flags, + int32_t width, int32_t height, int32_t refresh) +{ +} + +static void +output_handle_done(void *data, struct wl_output *output) +{ +} + +static void +output_handle_scale(void *data, struct wl_output *output, int32_t factor) +{ +} + +static void +output_handle_name(void *data, struct wl_output *output, const char *name) +{ +} + +static void +output_handle_description(void *data, struct wl_output *output, + const char *description) +{ +} + +static const struct wl_output_listener output_listener = { + .geometry = output_handle_geometry, + .mode = output_handle_mode, + .done = output_handle_done, + .scale = output_handle_scale, + .name = output_handle_name, + .description = output_handle_description, +}; + +static void +wsi_wl_output_destroy(struct wsi_wl_output *output) +{ + wl_list_remove(&output->link); + if (wl_output_get_version(output->output) >= WL_OUTPUT_RELEASE_SINCE_VERSION) + wl_output_release(output->output); + else + wl_output_destroy(output->output); + free(output); +} + static void registry_handle_global(void *data, struct wl_registry *registry, uint32_t name, const char *interface, uint32_t version) { struct wsi_wl_display *display = data; + if (strcmp(interface, wl_output_interface.name) == 0 && version >= 2) { + struct wsi_wl_output *output = calloc(1, sizeof(*output)); + if (!output) + return; + output->name = name; + output->output = wl_registry_bind(registry, name, &wl_output_interface, + MIN2(version, 4)); + wl_output_add_listener(output->output, &output_listener, output); + wl_list_insert(&display->outputs, &output->link); + } + if (display->sw) { if (strcmp(interface, wl_shm_interface.name) == 0) { display->wl_shm = wl_registry_bind(registry, name, &wl_shm_interface, 1); @@ -1564,9 +1700,14 @@ static void registry_handle_global_remove(void *data, struct wl_registry *registry, uint32_t name) { -#ifdef WL_FIXES_ACK_GLOBAL_REMOVE struct wsi_wl_display *display = data; + struct wsi_wl_output *output, *tmp; + wl_list_for_each_safe(output, tmp, &display->outputs, link) { + if (output->name == name) + wsi_wl_output_destroy(output); + } +#ifdef WL_FIXES_ACK_GLOBAL_REMOVE if (display->wl_fixes && wl_fixes_get_version(display->wl_fixes) >= WL_FIXES_ACK_GLOBAL_REMOVE_SINCE_VERSION) { wl_fixes_ack_global_remove(display->wl_fixes, registry, name); } @@ -1581,6 +1722,18 @@ static const struct wl_registry_listener registry_listener = { static void wsi_wl_display_finish(struct wsi_wl_display *display) { + struct wsi_wl_output *output, *tmp; + wl_list_for_each_safe(output, tmp, &display->outputs, link) + wsi_wl_output_destroy(output); + + if (display->registry) { +#ifdef WL_FIXES_INTERFACE + if (display->wl_fixes) + wl_fixes_destroy_registry(display->wl_fixes, display->registry); +#endif + wl_registry_destroy(display->registry); + } + struct wsi_wl_format *f; u_vector_foreach(f, &display->formats) u_vector_finish(&f->modifiers); @@ -1624,6 +1777,7 @@ wsi_wl_display_init(struct wsi_wayland *wsi_wl, { VkResult result = VK_SUCCESS; memset(display, 0, sizeof(*display)); + wl_list_init(&display->outputs); if (!u_vector_init(&display->formats, 8, sizeof(struct wsi_wl_format))) return VK_ERROR_OUT_OF_HOST_MEMORY; @@ -1657,6 +1811,7 @@ wsi_wl_display_init(struct wsi_wayland *wsi_wl, goto fail; } + display->registry = registry; wl_registry_add_listener(registry, ®istry_listener, display); /* Round-trip to get wl_shm and zwp_linux_dmabuf_v1 globals */ @@ -1754,12 +1909,7 @@ wsi_wl_display_init(struct wsi_wayland *wsi_wl, } out: - /* We don't need this anymore */ -#ifdef WL_FIXES_INTERFACE - if (display->wl_fixes) - wl_fixes_destroy_registry(display->wl_fixes, registry); -#endif - wl_registry_destroy(registry); + /* Keep the registry alive to track output additions and removals. */ /* Destroy default dma-buf feedback object and format table */ if (display->wl_dmabuf_feedback) { @@ -1771,14 +1921,6 @@ wsi_wl_display_init(struct wsi_wayland *wsi_wl, return VK_SUCCESS; fail_registry: - if (registry) { -#ifdef WL_FIXES_INTERFACE - if (display->wl_fixes) - wl_fixes_destroy_registry(display->wl_fixes, registry); -#endif - wl_registry_destroy(registry); - } - fail: wsi_wl_display_finish(display); return result; @@ -1921,6 +2063,30 @@ wsi_wl_surface_get_capabilities(VkIcdSurfaceBase *icd_surface, caps->surfaceCapabilities.minImageCount = wsi_wl_surface_get_min_image_count(display, present_mode); + caps->surfaceCapabilities.supportedTransforms = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; + caps->surfaceCapabilities.currentTransform = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; + if (wl_surface_get_version(surface->surface) >= + WL_SURFACE_SET_BUFFER_TRANSFORM_SINCE_VERSION) { + caps->surfaceCapabilities.supportedTransforms |= + VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR | + VK_SURFACE_TRANSFORM_ROTATE_180_BIT_KHR | + VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR | + VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_BIT_KHR | + VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_90_BIT_KHR | + VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_180_BIT_KHR | + VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_270_BIT_KHR; + + /* Without the application's surface enter/leave events, only a single + * output tells us which transform applies. Otherwise prefer identity. + */ + if (wl_list_length(&display->outputs) == 1) { + struct wsi_wl_output *output = + wl_container_of(display->outputs.next, output, link); + caps->surfaceCapabilities.currentTransform = + wl_transform_to_vk(output->transform); + } + } + if (!wsi_wl_surface->display) wsi_wl_display_finish(&temp_display); @@ -1934,8 +2100,6 @@ wsi_wl_surface_get_capabilities(VkIcdSurfaceBase *icd_surface, wsi_device->maxImageDimension2D, }; - caps->surfaceCapabilities.supportedTransforms = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; - caps->surfaceCapabilities.currentTransform = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; caps->surfaceCapabilities.maxImageArrayLayers = 1; caps->surfaceCapabilities.supportedCompositeAlpha = @@ -3570,6 +3734,11 @@ wsi_wl_swapchain_queue_present(struct wsi_swapchain *wsi_chain, wp_fifo_v1_wait_barrier(chain->fifo); chain->next_present_force_wait_barrier = false; } + if (chain->buffer_transform != wsi_wl_surface->buffer_transform) { + wl_surface_set_buffer_transform(wsi_wl_surface->wayland_surface.wrapper, + chain->buffer_transform); + wsi_wl_surface->buffer_transform = chain->buffer_transform; + } wl_surface_commit(wsi_wl_surface->wayland_surface.wrapper); wl_display_flush(wsi_wl_surface->display->wl_display); @@ -4023,6 +4192,7 @@ wsi_wl_surface_create_swapchain(VkIcdSurfaceBase *icd_surface, chain->base.image_count = num_images; chain->base.set_hdr_metadata = wsi_wl_swapchain_set_hdr_metadata; chain->extent = pCreateInfo->imageExtent; + chain->buffer_transform = vk_transform_to_wl(pCreateInfo->preTransform); chain->vk_format = pCreateInfo->imageFormat; chain->buffer_type = buffer_type; if (buffer_type == WSI_WL_BUFFER_NATIVE) { From 9e86574606277178768058063d710fcc08592378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 16:22:56 +0100 Subject: [PATCH 105/109] zink: present unrotated swapchains Prefer the identity transform because GL rendering is not pre-rotated. Assisted-by: LLM --- src/gallium/drivers/zink/zink_kopper.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/gallium/drivers/zink/zink_kopper.c b/src/gallium/drivers/zink/zink_kopper.c index c5e5d3ad586..12ff17a4377 100644 --- a/src/gallium/drivers/zink/zink_kopper.c +++ b/src/gallium/drivers/zink/zink_kopper.c @@ -320,7 +320,10 @@ kopper_CreateSwapchain(struct zink_screen *screen, struct kopper_displaytarget * } cswap->scci.presentMode = cdt->present_mode; cswap->scci.minImageCount = cdt->caps.minImageCount; - cswap->scci.preTransform = cdt->caps.currentTransform; + /* GL rendering is not pre-rotated. */ + cswap->scci.preTransform = cdt->caps.supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR + ? VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR + : cdt->caps.currentTransform; if (cdt->formats[1]) cswap->scci.pNext = &cdt->format_list; From a2907b7642524b9884181b854ff4a8257cf964ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 17:37:37 +0100 Subject: [PATCH 106/109] vulkan/wsi/wayland: list 8-bit formats first Prefer formats with 8-bit color channels while preserving the compositor's order within each group. Clients choosing the first surface format can otherwise select a float swapchain that costs twice the memory and may not scan out. Share the ordering between both surface format queries, applying the BGRA8_UNORM preference afterward without changing the advertised formats. Assisted-by: LLM --- src/vulkan/wsi/wsi_common_wayland.c | 78 ++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 18 deletions(-) diff --git a/src/vulkan/wsi/wsi_common_wayland.c b/src/vulkan/wsi/wsi_common_wayland.c index 14250de7635..a742c8474ba 100644 --- a/src/vulkan/wsi/wsi_common_wayland.c +++ b/src/vulkan/wsi/wsi_common_wayland.c @@ -38,6 +38,7 @@ #include "vk_instance.h" #include "vk_device.h" +#include "vk_format.h" #include "vk_physical_device.h" #include "vk_util.h" #include "wsi_common_entrypoints.h" @@ -1894,20 +1895,6 @@ wsi_wl_display_init(struct wsi_wayland *wsi_wl, goto fail; } - if (wsi_wl->wsi->force_bgra8_unorm_first) { - /* Find BGRA8_UNORM in the list and swap it to the first position if we - * can find it. Some apps get confused if SRGB is first in the list. - */ - struct wsi_wl_format *first_fmt = u_vector_tail(&display->formats); - struct wsi_wl_format *f, tmp_fmt; - f = find_format(&display->formats, VK_FORMAT_B8G8R8A8_UNORM); - if (f) { - tmp_fmt = *f; - *f = *first_fmt; - *first_fmt = tmp_fmt; - } - } - out: /* Keep the registry alive to track output additions and removals. */ @@ -2387,6 +2374,43 @@ wsi_wl_surface_get_capabilities2(VkIcdSurfaceBase *surface, return result; } +static void +wsi_wl_surface_sort_formats(struct wsi_wl_display *display, + const struct wsi_device *wsi_device, + struct wsi_wl_format **sorted_formats) +{ + unsigned count = 0; + + /* Prefer 8-bit color channels, preserving the compositor's order within + * each group. Higher precision formats can cost more memory and may not + * be supported by display planes. + */ + for (unsigned pass = 0; pass < 2; pass++) { + struct wsi_wl_format *disp_fmt; + u_vector_foreach(disp_fmt, &display->formats) { + VkFormat format = disp_fmt->vk_format; + bool is_8bit = + vk_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, 0) == 8 && + vk_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, 1) == 8 && + vk_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, 2) == 8; + if (is_8bit == (pass == 0)) + sorted_formats[count++] = disp_fmt; + } + } + + if (wsi_device->force_bgra8_unorm_first) { + for (unsigned i = 0; i < count; i++) { + if (sorted_formats[i]->vk_format == VK_FORMAT_B8G8R8A8_UNORM) { + struct wsi_wl_format *format = sorted_formats[i]; + memmove(&sorted_formats[1], &sorted_formats[0], + i * sizeof(*sorted_formats)); + sorted_formats[0] = format; + break; + } + } + } +} + static VkResult wsi_wl_surface_get_formats(VkIcdSurfaceBase *icd_surface, struct wsi_device *wsi_device, @@ -2404,13 +2428,21 @@ wsi_wl_surface_get_formats(VkIcdSurfaceBase *icd_surface, wsi_device->sw, "mesa formats query")) return VK_ERROR_SURFACE_LOST_KHR; + unsigned format_count = u_vector_length(&display.formats); + STACK_ARRAY(struct wsi_wl_format *, sorted_formats, format_count); + if (!sorted_formats) { + wsi_wl_display_finish(&display); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + wsi_wl_surface_sort_formats(&display, wsi_device, sorted_formats); + VK_OUTARRAY_MAKE_TYPED(VkSurfaceFormatKHR, out, pSurfaceFormats, pSurfaceFormatCount); VkColorSpaceKHR *cs; u_vector_foreach(cs, &display.colorspaces) { - struct wsi_wl_format *disp_fmt; - u_vector_foreach(disp_fmt, &display.formats) { + for (unsigned i = 0; i < format_count; i++) { + struct wsi_wl_format *disp_fmt = sorted_formats[i]; /* Skip formats for which we can't support both alpha & opaque * formats. */ @@ -2426,6 +2458,7 @@ wsi_wl_surface_get_formats(VkIcdSurfaceBase *icd_surface, } } + STACK_ARRAY_FINISH(sorted_formats); wsi_wl_display_finish(&display); return vk_outarray_status(&out); @@ -2449,13 +2482,21 @@ wsi_wl_surface_get_formats2(VkIcdSurfaceBase *icd_surface, wsi_device->sw, "mesa formats2 query")) return VK_ERROR_SURFACE_LOST_KHR; + unsigned format_count = u_vector_length(&display.formats); + STACK_ARRAY(struct wsi_wl_format *, sorted_formats, format_count); + if (!sorted_formats) { + wsi_wl_display_finish(&display); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + wsi_wl_surface_sort_formats(&display, wsi_device, sorted_formats); + VK_OUTARRAY_MAKE_TYPED(VkSurfaceFormat2KHR, out, pSurfaceFormats, pSurfaceFormatCount); VkColorSpaceKHR *cs; u_vector_foreach(cs, &display.colorspaces) { - struct wsi_wl_format *disp_fmt; - u_vector_foreach(disp_fmt, &display.formats) { + for (unsigned i = 0; i < format_count; i++) { + struct wsi_wl_format *disp_fmt = sorted_formats[i]; /* Skip formats for which we can't support both alpha & opaque * formats. */ @@ -2471,6 +2512,7 @@ wsi_wl_surface_get_formats2(VkIcdSurfaceBase *icd_surface, } } + STACK_ARRAY_FINISH(sorted_formats); wsi_wl_display_finish(&display); return vk_outarray_status(&out); From 7e355071f40285a3da1886675b845506e79552af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 19:01:22 +0100 Subject: [PATCH 107/109] pco/g5p: vertex constant UBO loads write and address bank 1 A vertex program that allocates its temps in bank 0 (one with control flow, or the bank-0 retry under register pressure) refused every constant-offset load from a UBO range outside the common store: the load's destination came from the bank-0 allocator, and only the all-bank-1 vertex address load is witnessed. Its scratch offset could also land in bank 0, which the vertex addr.add_shift does not read. Allocate both in bank 1, as the vertex storage-buffer load already does. No new encoding: the emitted form is the witnessed one. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 67a9fc4eb5a..c6a160cdd2e 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -2490,6 +2490,18 @@ static unsigned scratch_imm(struct gctx *c, uint32_t value) return reg; } +/* A scratch immediate in vertex bank 1, also when the program allocates its + * temps in bank 0 (alloc_bank1_run()). */ +static unsigned scratch_imm_bank1(struct gctx *c, uint32_t value) +{ + bool bank0 = c->vs_bank0, tex_bank0 = c->tex_bank0; + c->vs_bank0 = c->tex_bank0 = false; + unsigned reg = scratch_imm(c, value); + c->vs_bank0 = bank0; + c->tex_bank0 = tex_bank0; + return reg; +} + static unsigned one_reg(struct gctx *c) { if (c->remat || c->cf_depth > 0) @@ -4362,7 +4374,10 @@ static void emit_memory_ubo(struct gctx *c, nir_intrinsic_instr *in, if (in->def.bit_size != 32 || !n || n > 4) refuse(c, "memory UBO load: only 1..4 aligned 32-bit components"); - const unsigned dst = alloc_run(c, n, "memory UBO load"); + /* A vertex load writes bank 1, the witnessed form, also when the program + * allocates its temps in bank 0 (control flow, or the bank-0 pressure + * retry), as a vertex storage-buffer load does (emit_address_ssbo_load). */ + const unsigned dst = c->vs ? alloc_bank1_run(c, n, "memory UBO load") : alloc_run(c, n, "memory UBO load"); if (c->vs) vs_bank1_only(c, "vertex memory UBO load", dst, -1, -1, -1); const unsigned first = off - r->lo; @@ -4373,7 +4388,10 @@ static void emit_memory_ubo(struct gctx *c, nir_intrinsic_instr *in, for (unsigned i = 0; i < loads; i++) { const unsigned bytes = vector ? 16 : 4; const unsigned at = first + (vector ? 0 : 4 * i); - const unsigned index = scratch_imm(c, at); + /* the vertex addr.add_shift reads its offset from bank 1 only (vs_bank1_index()) */ + const unsigned index = c->vs ? scratch_imm_bank1(c, at) : scratch_imm(c, at); + if (c->vs) + vs_bank1_only(c, "vertex constant UBO address offset", index, -1, -1, -1); unsigned pair; for (pair = 0; pair + 2 <= NUM_DWORDS; pair += 2) From 6823deb94800ad119abf73c7b2a265b70f726fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Mon, 16 Nov 2026 19:29:47 +0100 Subject: [PATCH 108/109] pco/g5p: move vertex attributes out of bank 1 when bank-1-only loads need room A vertex program whose fetched attributes fill bank 1 (r0..r31) refused every addressed UBO load issued before those attributes were read: the load's data, its offset and its address pair must be in bank 1, the witnessed vertex form. A translated D3D9-style skinning vertex shader of a WebGPU game is one: eight vec4 attributes are each masked by a word of a UBO outside the common store, so the first instruction that could free an attribute is itself such a load. Every retry of the pressure ladder refused it ("memory UBO load needs 1 dword(s), 32 of 32 in use"). Add one last vertex retry. A vertex compile that every retry refused for register pressure is compiled again from a clone of its input, in its own instruction order (the scheduling retries hoist products far ahead of their use), with temps in bank 0. Before each instruction it keeps one aligned run of eight bank-1 registers free by moving held values into bank 0, the run needing the fewest moves. Eight registers hold every bank-1 demand of one instruction in the allocator's first-fit order. Like the other between-instruction mover (compact_vs), the retry takes only straight-line programs without stores. Apple's vertex programs keep their temps in bank 0 and read each attribute once into one; Apple's offline compiler does the same for this shape (12 float4 attributes and a 4 KiB constant buffer). The move is the class-98 move with destination bank 0 and source A in bank 1 that cf_operand() and the vertex texture-result park already emit. No new encoding. Programs that compile today keep their bytes: the first compile is unchanged, and only a vertex compile refused for register pressure reaches the retry. Offline replay of the game's 80 captured compiles: 80 admitted (was 79), the 79 admitted before byte-identical. Assisted-by: LLM --- src/imagination/g5p/pco_g5p_general.c | 116 ++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index c6a160cdd2e..8a0d74fca9c 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -246,6 +246,7 @@ struct gctx { bool eager_vout; /* VS temps retry: each output written as soon as its value is final */ bool compact_vs; /* last retry: compact movable bank1 values between instructions */ bool vs_bank0; /* VS pressure retry: temps in bank 0, outputs in bank 1 (vs-two-bank-allocator.md) */ + bool vs_evict; /* last VS retry: held bank-1 values move to bank 0 to keep room for bank-1-only forms */ nir_intrinsic_instr *vend_store; /* eager: the output component written by the final vend */ unsigned vend_comp; /* common store */ @@ -7514,6 +7515,88 @@ static void compact_vertex_bank1(struct gctx *c) } } +/* Last vertex pressure retry (vs_evict). The forms that must use bank 1 (the + * addressed UBO/SSBO loads: data, offset and address pair; the vertex math + * bridge; an output's bank-1 copy) found no room there when the fetched + * attributes fill r0..r31 until their first reads, which come after a UBO + * load (a translated D3D9-style skinning shader: eight vec4 attributes, every + * one masked by a UBO word). Apple's vertex programs hold their temps in + * bank 0 and read each attribute once into one (Apple's offline compiler on + * this shape, 12 float4 attributes and a 4 KiB constant buffer, does the + * same). Before each instruction, keep one + * aligned run of eight bank-1 registers free, which holds every bank-1 demand + * of one instruction in its first-fit order (a memory UBO load's data, offset + * and aligned pair; a register-indexed load's offset, pair and aligned vec4), + * by moving held values into bank 0, the run needing the fewest moves, with the + * class-98 move whose destination is bank 0 and source A bank 1, the move + * cf_operand() and the vertex texture park already emit. */ +static bool evict_movable(struct gctx *c, unsigned r) +{ + if ((c->reserved | c->scratch | c->tex_pending | c->tex_coord_pending) & BITFIELD64_BIT(r)) + return false; + if (!(c->busy & BITFIELD64_BIT(r))) + return true; + bool owned = false; + for (unsigned i = 0; i < c->nvals; i++) + if (c->vals[i].held && c->vals[i].reg == r) { + if (c->vals[i].last <= c->now) + return false; /* an operand of this instruction stays where it is */ + owned = true; + } + return owned; /* a busy register with no value (a pair, a persistent constant) stays */ +} + +static int evict_last(struct gctx *c, unsigned r) +{ + int last = -1; + for (unsigned i = 0; i < c->nvals; i++) + if (c->vals[i].held && c->vals[i].reg == r) + last = MAX2(last, c->vals[i].last); + return last; +} + +/* The aligned run of n bank-1 registers that is cheapest to free (fewest + * moves, then the latest nearest read), or -1. */ +static int evict_run(struct gctx *c, unsigned n) +{ + int best = -1, best_cost = INT_MAX, best_near = -1; + for (unsigned r = 0; r + n <= NUM_DWORDS; r += n) { + int cost = 0, near = INT_MAX; + bool ok = true; + for (unsigned k = r; k < r + n && ok; k++) { + ok = evict_movable(c, k); + if (ok && (c->busy & BITFIELD64_BIT(k))) { + cost++; + near = MIN2(near, evict_last(c, k)); + } + } + if (ok && (cost < best_cost || (cost == best_cost && near > best_near))) { + best = r; + best_cost = cost; + best_near = near; + } + } + return best; +} + +static void evict_vertex_bank1(struct gctx *c) +{ + int run = evict_run(c, 8); + if (run < 0) + return; + uint8_t raw[G5P_USC_MAX]; + u_foreach_bit64(src, BITFIELD64_RANGE(run, 8) & c->busy) { + int dst = alloc_bank0(c); + if (dst < 0) + return; /* bank 0 is full too: the instruction refuses as before */ + put(c, raw, g5p_usc_move(raw, H(c, dst), H(c, src), 0, 1, 0x21), "attribute bank value into bank 0"); + for (unsigned i = 0; i < c->nvals; i++) + if (c->vals[i].held && c->vals[i].reg == src) + c->vals[i].reg = dst; + c->busy &= ~BITFIELD64_BIT(src); + } +} + static void emit_block(struct gctx *c, nir_block *block) { nir_foreach_instr(ins, block) { @@ -7524,6 +7607,8 @@ static void emit_block(struct gctx *c, nir_block *block) decl_alloc(c, c->now); if (c->vs && c->vs_bank0 && c->compact_vs && !c->cf) compact_vertex_bank1(c); + if (c->vs && c->vs_bank0 && c->vs_evict && !c->cf) + evict_vertex_bank1(c); switch (ins->type) { case nir_instr_type_alu: emit_alu(c, nir_instr_as_alu(ins)); @@ -8165,7 +8250,7 @@ bool pco_g5p_compile_general_key(nir_shader *nir, bool half_output, const uint8_ static bool compile_general_link(nir_shader *nir, bool half_output, const uint8_t *attrib_components, const struct pco_g5p_fs_key *key, const struct pco_g5p_vs_link *link, - struct pco_g5p_program *out, bool preserve_cf) + struct pco_g5p_program *out, bool preserve_cf, bool evict) { static const struct pco_g5p_fs_key default_key = {.write_mask = 0xf}; memset(out, 0, sizeof(*out)); @@ -8288,6 +8373,16 @@ static bool compile_general_link(nir_shader *nir, bool half_output, const uint8_ refuse(c, "out of memory"); memset(c->home, -1, c->impl->ssa_alloc * 4); schedule_texture_batches(c); + if (evict) { + /* pco_g5p_compile_general_link()'s last vertex retry: the program's own order, temps in bank 0, attributes + * moved out of bank 1 where a bank-1-only form needs room (evict_vertex_bank1()) */ + /* as the other between-instruction mover (compact_vs): straight-line programs without stores */ + if (!c->vs || c->cf || impl_has_side_effects(c->impl)) + refuse(c, "register pressure: bank-1 eviction is a straight-line vertex retry without stores"); + c->vs_bank0 = c->remat = c->eager_vout = c->vs_evict = true; + ok = two_pass(c); + goto done; + } ok = two_pass(c); if (!ok && !c->vs && strstr(out->refusal, "register pressure")) { /* Retry: constants and uniforms re-read per use (round 19). */ @@ -8420,11 +8515,22 @@ bool pco_g5p_compile_general_link(nir_shader *nir, bool half_output, const uint8 * together. Retry the original fragment control flow only on pressure; * the existing scalar register/mask-stack lowering handles the branches. * Keep all phis scalar in that retry, including texture-result phis. */ - nir_shader *original = nir->info.stage == MESA_SHADER_FRAGMENT && cf_left(nir) + /* A vertex program every pressure retry refused is compiled once more + * from its own order, before the scheduling retries reordered it, with the + * attributes moved out of bank 1 where a bank-1-only form needs room + * (evict_vertex_bank1()). Programs that compile today never get there. */ + nir_shader *original = (nir->info.stage == MESA_SHADER_FRAGMENT && cf_left(nir)) || + nir->info.stage == MESA_SHADER_VERTEX ? nir_shader_clone(NULL, nir) : NULL; - bool ok = compile_general_link(nir, half_output, attrib_components, key, link, out, false); - if (!ok && original && strstr(out->refusal, "register pressure")) - ok = compile_general_link(original, half_output, attrib_components, key, link, out, true); + bool ok = compile_general_link(nir, half_output, attrib_components, key, link, out, false, false); + if (!ok && original && strstr(out->refusal, "register pressure")) { + char first[sizeof(out->refusal)]; + memcpy(first, out->refusal, sizeof(first)); + const bool vs = nir->info.stage == MESA_SHADER_VERTEX; + ok = compile_general_link(original, half_output, attrib_components, key, link, out, !vs, vs); + if (!ok && vs) /* the ladder's own reason stays the one reported */ + memcpy(out->refusal, first, sizeof(first)); + } ralloc_free(original); return ok; } From 9177f66be8e148db450e22c899d3d1813d5fb187 Mon Sep 17 00:00:00 2001 From: Henrik Klee Date: Tue, 17 Nov 2026 17:48:16 +0800 Subject: [PATCH 109/109] ci: build the shipped drivers on pull requests and main The only workflow was upstream's macOS-CI, which asks for the retired macos-11 runner: every push queued it for 24 hours and then cancelled it, and nothing ever compiled a pull request. The new workflow runs the build() and package() of OrchardLinux/packages' mesa PKGBUILD on the checkout, in the Arch Linux ARM image that repository pins and prepares the same way, on an arm64 runner: the Imagination PowerVR Vulkan driver and OpenGL through Zink, llvmpipe and softpipe, with the package's exact meson options, and package()'s check that the drivers the image needs are present. main saves a compiler cache, pull requests only read it. macos.yml is removed. Generated-by: LLM --- .github/workflows/build.yml | 91 +++++++++++++++++++++++++++++++++++++ .github/workflows/macos.yml | 60 ------------------------ 2 files changed, 91 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/macos.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..5c35c379730 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,91 @@ +name: Build +# Compiles the drivers OrchardLinux ships from this fork, the way its package +# builds them: the build() and package() of OrchardLinux/packages' +# packages/mesa/pacman/PKGBUILD, run on this checkout instead of the commit the +# package pins, in the same Arch Linux ARM container the package is built in. +# That is the Imagination PowerVR Vulkan driver (Wayland) and OpenGL through +# Zink, llvmpipe and softpipe (X11 and Wayland), with exactly the package's +# meson options; package() then checks that the drivers the image needs are +# there. Build only: nothing is tested on a GPU or published. +on: + pull_request: + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + aarch64: + runs-on: ubuntu-24.04-arm + timeout-minutes: 120 + # The image OrchardLinux/packages pins for aarch64 (ci/containers.json). + container: docker.io/arch4edu/archlinuxarm@sha256:1db43d1af9917db5fd34af3dab64e2b229a21240e81bb765e3d2d3c34739d587 + env: + ORCHARD_ARCH: aarch64 + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_BASEDIR: ${{ github.workspace }} + CCACHE_MAXSIZE: 2G + defaults: + run: + shell: bash + steps: + # The package's recipe and the packages repository's container setup. The + # image has no git yet, so this comes as an archive: the whole small repository. + - uses: actions/checkout@v4 + with: + repository: OrchardLinux/packages + path: packages + persist-credentials: false + # pacman made usable in a container and the system brought up to date, as + # for every package build there; then the recipe's dependencies. + - name: Prepare the build system + run: | + bash packages/ci/prepare-pacman + pacman -S --noconfirm --needed git ccache + # shellcheck disable=SC1091 + source packages/packages/mesa/pacman/PKGBUILD + pacman -S --noconfirm --needed --asdeps "${depends[@]}" "${makedepends[@]}" + - uses: actions/checkout@v4 + with: + path: mesa + persist-credentials: false + # main keeps the compiler cache of its last build; a pull request only reads it. + - uses: actions/cache@v4 + if: github.event_name != 'pull_request' + with: + path: .ccache + key: mesa-ccache-v1-aarch64-${{ github.sha }} + restore-keys: mesa-ccache-v1-aarch64- + - uses: actions/cache/restore@v4 + if: github.event_name == 'pull_request' + with: + path: .ccache + key: mesa-ccache-v1-aarch64-${{ github.sha }} + restore-keys: mesa-ccache-v1-aarch64- + - name: Build as the package does + run: | + # makepkg's compiler flags; the recipe's options leave out LTO and debug. + # shellcheck disable=SC1091 + source /etc/makepkg.conf + export CFLAGS CXXFLAGS LDFLAGS + # shellcheck disable=SC1091 + source packages/packages/mesa/pacman/PKGBUILD + # The recipe builds the source directory mesa-$_commit: this checkout. + work=$PWD/work + mkdir -p "$work" + ln -s "$GITHUB_WORKSPACE/mesa" "$work/mesa-${_commit}" + cp packages/packages/mesa/gpu-probe.c "$work/" + mkdir -p "$CCACHE_DIR" + ccache --zero-stats >/dev/null + started=$(date +%s) + # meson compiles through ccache when it finds it. + (cd "$work" && build) + export srcdir=$work pkgdir=$work/pkg + (cd "$work" && package) + # Keep only what this build used, so the saved cache stays one build large. + ccache --evict-older-than "$(( $(date +%s) - started + 60 ))s" >/dev/null + { echo 'Compiler cache:'; echo '```'; ccache --show-stats; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + ls -l "$work/pkg/usr/lib/libvulkan_powervr_mesa.so" "$work"/pkg/usr/lib/libgallium-*.so diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index f6605ef5597..00000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: macOS-CI -on: push - -permissions: - contents: read - -jobs: - macOS-CI: - strategy: - matrix: - glx_option: ['dri'] - runs-on: macos-11 - env: - GALLIUM_DUMP_CPU: true - MESON_EXEC: /Users/runner/Library/Python/3.11/bin/meson - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - cat > Brewfile < native_config <