diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..5c35c379730 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,91 @@ +name: Build +# Compiles the drivers OrchardLinux ships from this fork, the way its package +# builds them: the build() and package() of OrchardLinux/packages' +# packages/mesa/pacman/PKGBUILD, run on this checkout instead of the commit the +# package pins, in the same Arch Linux ARM container the package is built in. +# That is the Imagination PowerVR Vulkan driver (Wayland) and OpenGL through +# Zink, llvmpipe and softpipe (X11 and Wayland), with exactly the package's +# meson options; package() then checks that the drivers the image needs are +# there. Build only: nothing is tested on a GPU or published. +on: + pull_request: + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + aarch64: + runs-on: ubuntu-24.04-arm + timeout-minutes: 120 + # The image OrchardLinux/packages pins for aarch64 (ci/containers.json). + container: docker.io/arch4edu/archlinuxarm@sha256:1db43d1af9917db5fd34af3dab64e2b229a21240e81bb765e3d2d3c34739d587 + env: + ORCHARD_ARCH: aarch64 + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_BASEDIR: ${{ github.workspace }} + CCACHE_MAXSIZE: 2G + defaults: + run: + shell: bash + steps: + # The package's recipe and the packages repository's container setup. The + # image has no git yet, so this comes as an archive: the whole small repository. + - uses: actions/checkout@v4 + with: + repository: OrchardLinux/packages + path: packages + persist-credentials: false + # pacman made usable in a container and the system brought up to date, as + # for every package build there; then the recipe's dependencies. + - name: Prepare the build system + run: | + bash packages/ci/prepare-pacman + pacman -S --noconfirm --needed git ccache + # shellcheck disable=SC1091 + source packages/packages/mesa/pacman/PKGBUILD + pacman -S --noconfirm --needed --asdeps "${depends[@]}" "${makedepends[@]}" + - uses: actions/checkout@v4 + with: + path: mesa + persist-credentials: false + # main keeps the compiler cache of its last build; a pull request only reads it. + - uses: actions/cache@v4 + if: github.event_name != 'pull_request' + with: + path: .ccache + key: mesa-ccache-v1-aarch64-${{ github.sha }} + restore-keys: mesa-ccache-v1-aarch64- + - uses: actions/cache/restore@v4 + if: github.event_name == 'pull_request' + with: + path: .ccache + key: mesa-ccache-v1-aarch64-${{ github.sha }} + restore-keys: mesa-ccache-v1-aarch64- + - name: Build as the package does + run: | + # makepkg's compiler flags; the recipe's options leave out LTO and debug. + # shellcheck disable=SC1091 + source /etc/makepkg.conf + export CFLAGS CXXFLAGS LDFLAGS + # shellcheck disable=SC1091 + source packages/packages/mesa/pacman/PKGBUILD + # The recipe builds the source directory mesa-$_commit: this checkout. + work=$PWD/work + mkdir -p "$work" + ln -s "$GITHUB_WORKSPACE/mesa" "$work/mesa-${_commit}" + cp packages/packages/mesa/gpu-probe.c "$work/" + mkdir -p "$CCACHE_DIR" + ccache --zero-stats >/dev/null + started=$(date +%s) + # meson compiles through ccache when it finds it. + (cd "$work" && build) + export srcdir=$work pkgdir=$work/pkg + (cd "$work" && package) + # Keep only what this build used, so the saved cache stays one build large. + ccache --evict-older-than "$(( $(date +%s) - started + 60 ))s" >/dev/null + { echo 'Compiler cache:'; echo '```'; ccache --show-stats; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + ls -l "$work/pkg/usr/lib/libvulkan_powervr_mesa.so" "$work"/pkg/usr/lib/libgallium-*.so diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index f6605ef5597..00000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: macOS-CI -on: push - -permissions: - contents: read - -jobs: - macOS-CI: - strategy: - matrix: - glx_option: ['dri'] - runs-on: macos-11 - env: - GALLIUM_DUMP_CPU: true - MESON_EXEC: /Users/runner/Library/Python/3.11/bin/meson - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - cat > Brewfile < native_config < #include "util/anon_file.h" +#include "util/u_debug.h" #include "util/perf/cpu_trace.h" #include "util/u_vector.h" #include "util/format/u_formats.h" @@ -758,6 +759,18 @@ dri2_wl_create_window_surface(_EGLDisplay *disp, _EGLConfig *conf, dri2_surf->base.Width = window->width; dri2_surf->base.Height = window->height; + /* A surface takes a third (and fourth) color buffer while the compositor + * holds two, and frees a spare only after it has gone unused for + * BUFFER_TRIM_AGE_HYSTERESIS frames. A surface that stops drawing, such as + * a desktop shell's wallpaper after its first frames, keeps the spare for + * good: a whole surface-sized buffer. A client that redraws rarely can + * stay double buffered instead and wait for a release when it draws. */ + int max_buffers = ARRAY_SIZE(dri2_surf->color_buffers); + dri2GalliumConfigQueryi(dri2_dpy->dri_screen_render_gpu, + "egl_wayland_max_buffers", &max_buffers); + dri2_surf->max_color_buffers = + CLAMP(max_buffers, 2, ARRAY_SIZE(dri2_surf->color_buffers)); + visual_idx = dri2_wl_visual_idx_from_config(config); assert(visual_idx != -1); assert(dri2_wl_visuals[visual_idx].pipe_format != PIPE_FORMAT_NONE); @@ -1292,8 +1305,10 @@ wait_for_free_buffer(struct dri2_egl_display *dri2_dpy, /* There might be a buffer release already queued that wasn't processed */ wl_display_dispatch_queue_pending(dri2_dpy->wl_dpy, dri2_surf->wl_queue); + const unsigned max_buffers = dri2_surf->max_color_buffers; + while (dri2_surf->back == NULL) { - for (int i = 0; i < ARRAY_SIZE(dri2_surf->color_buffers); i++) { + for (int i = 0; i < max_buffers; i++) { /* Get an unlocked buffer, preferably one with a dri_buffer * already allocated and with minimum age. */ diff --git a/src/gallium/auxiliary/util/u_screen.c b/src/gallium/auxiliary/util/u_screen.c index d1399bef84d..1756a0bcabf 100644 --- a/src/gallium/auxiliary/util/u_screen.c +++ b/src/gallium/auxiliary/util/u_screen.c @@ -174,6 +174,7 @@ u_init_pipe_screen_caps(struct pipe_screen *pscreen, int accel) VARYING_BIT_BOUNDING_BOX1; caps->throttle = true; + caps->throttle_frames = 1; #if defined(HAVE_LIBDRM) && (DETECT_OS_LINUX || DETECT_OS_BSD || DETECT_OS_MANAGARM) if (pscreen->get_screen_fd) { diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c index 114ad811853..b25d6aa341a 100644 --- a/src/gallium/auxiliary/util/u_threaded_context.c +++ b/src/gallium/auxiliary/util/u_threaded_context.c @@ -279,6 +279,10 @@ tc_batch_increment_renderpass_info(struct threaded_context *tc, unsigned batch_i batch->renderpass_info_idx++; tc_batch_renderpass_infos_resize(tc, batch); tc_info = batch->renderpass_infos.data; + /* the driver can reach this info through the previous one as soon as that + * is signaled: it has to wait on this one by then + */ + util_queue_fence_reset(&tc_info[batch->renderpass_info_idx].ready); if (full_copy) { /* this should only be called when changing batches */ @@ -314,7 +318,6 @@ tc_batch_increment_renderpass_info(struct threaded_context *tc, unsigned batch_i assert(!full_copy || !tc->renderpass_info_recording || tc_batch_rp_info(tc->renderpass_info_recording)->next); /* signal existing info since it will not be used anymore */ tc_signal_renderpass_info_ready(tc); - util_queue_fence_reset(&tc_info[batch->renderpass_info_idx].ready); /* guard against deadlock scenario */ assert(tc->renderpass_info_recording != &tc_info[batch->renderpass_info_idx].info); tc_info[batch->renderpass_info_idx].end_call = NULL; @@ -353,6 +356,28 @@ tc_check_fb_access(struct threaded_context *tc, struct pipe_resource *src, struc return false; } +/* calls that continue the recording info through tc_parse_draw or a clear: + * the driver must advance its info before executing the first one after an + * end_call, keep this in sync with their callers + */ +static ALWAYS_INLINE bool +tc_call_may_advance_rp_info(enum tc_call_id id) +{ + switch (id) { + case TC_CALL_clear: + case TC_CALL_draw_single: + case TC_CALL_draw_multi: + case TC_CALL_draw_single_drawid: + case TC_CALL_draw_indirect: + case TC_CALL_draw_vstate_single: + case TC_CALL_draw_vstate_multi: + case TC_CALL_draw_mesh_tasks: + return true; + default: + return false; + } +} + /* update metadata at draw time */ static void tc_parse_draw(struct threaded_context *tc) @@ -587,6 +612,10 @@ tc_batch_flush(struct threaded_context *tc, bool full_copy) * renderpass info can only be accessed by its owner batch during execution */ if (tc->renderpass_info_recording) { + /* an rp ended by an end_call is not continued: the driver advances after + * the end_call, so a copy would leave it one info behind + */ + full_copy &= !tc_batch_rp_info(tc->renderpass_info_recording)->end_call; tc_batch_increment_renderpass_info(tc, next_id, full_copy); } @@ -3029,7 +3058,9 @@ tc_texture_map(struct pipe_context *_pipe, bool is_internal_unsynchronized = usage & PIPE_MAP_UNSYNCHRONIZED && resource->flags & PIPE_RESOURCE_FLAG_MAP_UNSYNCHRONIZED; - if (tc->options.parse_renderpass_info && TC_RENDERPASS_INFO_HAS_WORK(tc->renderpass_info_recording->data32[0])) + /* an unsynchronized map enqueues no call that could be the end_call */ + if (tc->options.parse_renderpass_info && !is_internal_unsynchronized && + TC_RENDERPASS_INFO_HAS_WORK(tc->renderpass_info_recording->data32[0])) tc_check_fb_access(tc, NULL, resource); /* this can only be hit for internal use */ @@ -3734,7 +3765,8 @@ tc_texture_subdata(struct pipe_context *_pipe, struct pipe_context *pipe = tc->pipe; struct threaded_resource *tres = threaded_resource(resource); unsigned unsync_usage = TC_TRANSFER_MAP_THREADED_UNSYNC | PIPE_MAP_UNSYNCHRONIZED | PIPE_MAP_WRITE; - bool can_unsync = !tc_resource_batch_usage_test_busy(tc, resource) && + /* if the info ended here, sync so that tc_sync resets it */ + bool can_unsync = !ended && !tc_resource_batch_usage_test_busy(tc, resource) && tc->options.is_resource_busy && !tc->options.is_resource_busy(tc->pipe->screen, tres->latest, usage | unsync_usage); @@ -4083,10 +4115,13 @@ tc_flush(struct pipe_context *_pipe, struct pipe_fence_handle **fence, if (!tc->options.driver_calls_flush_notify) p_atomic_set(&tc->num_busy_replies, 0); + bool ended_here = false; if (!deferred || !fence) { tc->in_renderpass = false; - if (tc->renderpass_info_recording) + if (tc->renderpass_info_recording) { + ended_here = !tc->renderpass_info_recording->ended; tc->renderpass_info_recording->ended = true; + } } if (async && tc->options.create_fence) { @@ -4112,6 +4147,9 @@ tc_flush(struct pipe_context *_pipe, struct pipe_fence_handle **fence, if (deferred) { /* these have identical fields */ p = (struct tc_flush_call *)tc_add_call(tc, TC_CALL_flush_deferred, tc_flush_deferred_call); + /* the driver has no other call to advance after */ + if (ended_here) + tc_batch_rp_info(tc->renderpass_info_recording)->end_call = p; } else { p = tc_add_call(tc, TC_CALL_flush, tc_flush_call); p->tc = tc; @@ -5124,12 +5162,15 @@ tc_blit(struct pipe_context *_pipe, const struct pipe_blit_info *info) tc->renderpass_info_recording->has_resolve = true; tc->renderpass_info_recording->ended = true; is_resolve = true; - } else if (src == info->src.resource) { + } else if (src == info->src.resource && (is_depth || !tc->fb_resolve) && + (!tc->renderpass_info_recording->resolve[is_depth] || + tc->renderpass_info_recording->resolve[is_depth] == info->dst.resource)) { #if TC_DEBUG >= 3 tc_printf("RESOLVE MERGE"); #endif - /* only optimizing out one depth and one color resolve */ - assert(!tc->renderpass_info_recording->resolve[is_depth] || tc->renderpass_info_recording->resolve[is_depth] == info->dst.resource); + /* only optimizing out one depth and one color resolve: a resolve into + * any other target is a normal blit + */ pipe_resource_reference(&tc->renderpass_info_recording->resolve[is_depth], info->dst.resource); tc_set_resource_batch_usage(tc, info->dst.resource); if (tc->renderpass_info_recording->has_resolve) { @@ -5143,9 +5184,9 @@ tc_blit(struct pipe_context *_pipe, const struct pipe_blit_info *info) } else if (tc->in_renderpass) { tc_check_fb_access(tc, info->src.resource, info->dst.resource); } - struct tc_blit_call *blit = tc_blit_enqueue(tc, info); - if (ended != tc->renderpass_info_recording->ended) - tc_batch_rp_info(tc->renderpass_info_recording)->end_call = blit; + /* write the geometry before the enqueue, which can flush the batch and + * change the recording info + */ if (is_resolve) { tc->renderpass_info_recording->resolve_geometry.x = info->src.box.x; tc->renderpass_info_recording->resolve_geometry.y = info->src.box.y; @@ -5153,8 +5194,13 @@ tc_blit(struct pipe_context *_pipe, const struct pipe_blit_info *info) tc->renderpass_info_recording->resolve_geometry.width = info->src.box.width; tc->renderpass_info_recording->resolve_geometry.height = info->src.box.height; tc->renderpass_info_recording->resolve_geometry.depth = info->src.box.depth; - blit->base.call_id = TC_CALL_resolve; } + + struct tc_blit_call *blit = tc_blit_enqueue(tc, info); + if (!ended && tc->renderpass_info_recording->ended) + tc_batch_rp_info(tc->renderpass_info_recording)->end_call = blit; + if (is_resolve) + blit->base.call_id = TC_CALL_resolve; } struct tc_generate_mipmap { @@ -5707,6 +5753,9 @@ tc_draw_mesh_tasks(struct pipe_context *_pipe, const struct pipe_grid_info *info) { struct threaded_context *tc = threaded_context(_pipe); + if (tc->options.parse_renderpass_info) + tc_parse_draw(tc); + struct tc_draw_mesh_tasks *p = tc_add_call(tc, TC_CALL_draw_mesh_tasks, tc_draw_mesh_tasks); @@ -5785,6 +5834,13 @@ batch_execute(struct tc_batch *batch, struct pipe_context *pipe, bool parsing) #endif TC_TRACE_SCOPE(call->call_id); + /* iterate before executing the first draw or clear after an rp end */ + if (parsing && increment_rp_info_on_draw_clear && + tc_call_may_advance_rp_info(call->call_id)) { + batch->tc->renderpass_info = incr_rp_info(batch->tc->renderpass_info); + increment_rp_info_on_draw_clear = false; + } + /* This executes the call using a switch. */ switch (call->call_id) { #define CALL(name) \ @@ -5813,19 +5869,6 @@ batch_execute(struct tc_batch *batch, struct pipe_context *pipe, bool parsing) increment_rp_info_on_draw_clear = false; } else if (call == tc_batch_rp_info(batch->tc->renderpass_info)->end_call) { increment_rp_info_on_draw_clear = true; - } else if (increment_rp_info_on_draw_clear) { - switch (call->call_id) { - case TC_CALL_clear: - case TC_CALL_draw_single: - case TC_CALL_draw_multi: - case TC_CALL_draw_single_drawid: - case TC_CALL_draw_vstate_multi: - batch->tc->renderpass_info = incr_rp_info(batch->tc->renderpass_info); - increment_rp_info_on_draw_clear = false; - break; - default: - break; - } } } } diff --git a/src/gallium/drivers/zink/driinfo_zink.h b/src/gallium/drivers/zink/driinfo_zink.h index cdf1596cce0..a62aa9ea039 100644 --- a/src/gallium/drivers/zink/driinfo_zink.h +++ b/src/gallium/drivers/zink/driinfo_zink.h @@ -6,6 +6,7 @@ DRI_CONF_SECTION_DEBUG DRI_CONF_SECTION_END DRI_CONF_SECTION_PERFORMANCE +DRI_CONF_OPT_I(zink_frames_in_flight, 0, 0, 3, "Frames in flight (0 = driver default)") DRI_CONF_MESA_GLTHREAD_DRIVER(true) DRI_CONF_OPT_B(zink_shader_object_enable, false, "Enable support for EXT_shader_object") DRI_CONF_SECTION_END diff --git a/src/gallium/drivers/zink/zink_clear.c b/src/gallium/drivers/zink/zink_clear.c index 156175f7695..fefd29e503f 100644 --- a/src/gallium/drivers/zink/zink_clear.c +++ b/src/gallium/drivers/zink/zink_clear.c @@ -574,7 +574,7 @@ zink_clear_buffer(struct pipe_context *pctx, - size is the number of bytes to fill, and must be either a multiple of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of the buffer */ - zink_resource_buffer_transfer_dst_barrier(ctx, res, offset, size); + zink_resource_buffer_transfer_dst_barrier(ctx, res, offset, size, false); VkCommandBuffer cmdbuf = zink_get_cmdbuf(ctx, NULL, res); zink_batch_reference_resource_rw(ctx, res, true); VKCTX(CmdFillBuffer)(cmdbuf, res->obj->buffer, offset, size, *(uint32_t*)clear_value); diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c index 57efe18739d..129a742d98b 100644 --- a/src/gallium/drivers/zink/zink_context.c +++ b/src/gallium/drivers/zink/zink_context.c @@ -5446,7 +5446,7 @@ zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zin /* unsync should only occur during subdata with staging resource */ if (!unsync) zink_screen(ctx->base.screen)->buffer_barrier(ctx, src, VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); - bool unordered_dst = zink_resource_buffer_transfer_dst_barrier(ctx, dst, dst_offset, size); + bool unordered_dst = zink_resource_buffer_transfer_dst_barrier(ctx, dst, dst_offset, size, unsync); bool can_unorder = unordered_dst && unordered_src && !ctx->no_reorder; VkCommandBuffer cmdbuf = unsync ? ctx->bs->unsynchronized_cmdbuf : can_unorder ? ctx->bs->reordered_cmdbuf : zink_get_cmdbuf(ctx, src, dst); @@ -5524,7 +5524,7 @@ zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, stru ctx->no_reorder++; VkImageLayout layout = screen->driver_workarounds.general_layout ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; screen->image_barrier(ctx, use_img, layout, VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); - zink_resource_buffer_transfer_dst_barrier(ctx, buf, box->x, box->width); + zink_resource_buffer_transfer_dst_barrier(ctx, buf, box->x, box->width, false); } VkBufferImageCopy region = {0}; diff --git a/src/gallium/drivers/zink/zink_context.h b/src/gallium/drivers/zink/zink_context.h index aeb1a1db728..84b64da8869 100644 --- a/src/gallium/drivers/zink/zink_context.h +++ b/src/gallium/drivers/zink/zink_context.h @@ -215,7 +215,7 @@ zink_check_valid_buffer_src_access(struct zink_context *ctx, struct zink_resourc void zink_resource_image_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned level, const struct pipe_box *box, bool unsync); bool -zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size); +zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size, bool unsync); void zink_synchronization_init(struct zink_screen *screen); void diff --git a/src/gallium/drivers/zink/zink_kopper.c b/src/gallium/drivers/zink/zink_kopper.c index c5e5d3ad586..9f6cee614c7 100644 --- a/src/gallium/drivers/zink/zink_kopper.c +++ b/src/gallium/drivers/zink/zink_kopper.c @@ -320,7 +320,20 @@ kopper_CreateSwapchain(struct zink_screen *screen, struct kopper_displaytarget * } cswap->scci.presentMode = cdt->present_mode; cswap->scci.minImageCount = cdt->caps.minImageCount; - cswap->scci.preTransform = cdt->caps.currentTransform; + /* Each frame in flight holds an image, besides the one being displayed and + * the one being drawn. Otherwise acquisition can throttle the queue to one + * frame in flight. + */ + if (screen->frames_in_flight > 1) { + uint32_t want = MAX2(cswap->scci.minImageCount, screen->frames_in_flight + 2); + if (cdt->caps.maxImageCount) + want = MIN2(want, cdt->caps.maxImageCount); + cswap->scci.minImageCount = want; + } + /* GL rendering is not pre-rotated. */ + cswap->scci.preTransform = cdt->caps.supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR + ? VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR + : cdt->caps.currentTransform; if (cdt->formats[1]) cswap->scci.pNext = &cdt->format_list; diff --git a/src/gallium/drivers/zink/zink_query.c b/src/gallium/drivers/zink/zink_query.c index 7d6526db7c9..77cb248daa7 100644 --- a/src/gallium/drivers/zink/zink_query.c +++ b/src/gallium/drivers/zink/zink_query.c @@ -791,7 +791,7 @@ copy_pool_results_to_buffer(struct zink_context *ctx, struct zink_query *query, /* if it's a single query that doesn't need special handling, we can copy it and be done */ zink_batch_reference_resource_rw(ctx, res, true); zink_resource_disable_unordered(res, true); - zink_resource_buffer_transfer_dst_barrier(ctx, res, offset, result_size); + zink_resource_buffer_transfer_dst_barrier(ctx, res, offset, result_size, false); util_range_add(&res->base.b, &res->valid_buffer_range, offset, offset + result_size); assert(query_id < NUM_QUERIES); ctx->bs->has_work = true; diff --git a/src/gallium/drivers/zink/zink_resource.c b/src/gallium/drivers/zink/zink_resource.c index 4c04c864e7a..1fed7a003f1 100644 --- a/src/gallium/drivers/zink/zink_resource.c +++ b/src/gallium/drivers/zink/zink_resource.c @@ -857,6 +857,23 @@ add_2d_view_flag(struct zink_screen *screen, const struct pipe_resource *templ, } } +/* The sampled-image sample counts of a format, as zink_is_format_supported() checks them. */ +static VkSampleCountFlags +sampled_sample_counts(struct zink_screen *screen, enum pipe_format format) +{ + const VkPhysicalDeviceLimits *l = &screen->info.props.limits; + const struct util_format_description *desc = util_format_description(format); + if (util_format_is_depth_or_stencil(format)) { + VkSampleCountFlags s = ~0u; + if (util_format_has_depth(desc)) + s &= l->sampledImageDepthSampleCounts; + if (util_format_has_stencil(desc)) + s &= l->sampledImageStencilSampleCounts; + return s; + } + return util_format_is_pure_integer(format) ? l->sampledImageIntegerSampleCounts : l->sampledImageColorSampleCounts; +} + static unsigned build_usage_candidates(struct zink_screen *screen, const struct pipe_resource *templ, unsigned bind, VkImageTiling tiling, VkImageCreateFlags base_flags, bool extended, bool have_fmtlist, @@ -902,6 +919,17 @@ build_usage_candidates(struct zink_screen *screen, const struct pipe_resource *t n = emit_usage_candidates(c, n, tiling, usage, flags, false, have_fmtlist, always_mutable, max); } + /* Vulkan allows sampledImage*SampleCounts to be a subset of framebuffer*SampleCounts. GL + * renderbuffers carry PIPE_BIND_SAMPLER_VIEW, but their format and GL_MAX_SAMPLES are chosen + * without it, so a multisampled attachment the device cannot sample at this count would get no + * storage. Retry without SAMPLED: zink_is_format_supported() reports SAMPLER_VIEW unsupported for + * this sample count, so nothing samples the image. + */ + if (templ->nr_samples > 1 && (usage & VK_IMAGE_USAGE_SAMPLED_BIT) && + !(sampled_sample_counts(screen, templ->format) & (VkSampleCountFlags)templ->nr_samples)) + n = emit_usage_candidates(c, n, tiling, usage & ~VK_IMAGE_USAGE_SAMPLED_BIT, flags, false, have_fmtlist, + always_mutable, max); + return n; } diff --git a/src/gallium/drivers/zink/zink_screen.c b/src/gallium/drivers/zink/zink_screen.c index 50f6a86b5c8..ca3fc5286ed 100644 --- a/src/gallium/drivers/zink/zink_screen.c +++ b/src/gallium/drivers/zink/zink_screen.c @@ -733,6 +733,7 @@ zink_init_screen_caps(struct zink_screen *screen) u_init_pipe_screen_caps(&screen->base, screen->is_cpu ? 0 : 1); caps->r11g11b10f_requires_color_buffer_float = screen->is_g5p; + caps->throttle_frames = screen->frames_in_flight; /* Vulkan spec says the builtins count, so make GL count them towards * link-failing. @@ -1197,6 +1198,19 @@ zink_init_screen_caps(struct zink_screen *screen) assert(screen->info.feats.features.tessellationShader == VK_FALSE); caps->max_varyings = 16; } + /* Between 64 and 120 output components, halving for streamout falls below + * the ES 3.0 minimum of 15 varyings. Zink inlines every transform-feedback + * output into an existing output (update_so_info()), and the GLSL linker + * limits the vertex stage's outputs to maxVertexOutputComponents + * (check_against_output_limit()), so 16 varyings cannot push the vertex + * shader past the device limit: such a program fails to link instead. + */ + else if (zink_driverid(screen) == VK_DRIVER_ID_IMAGINATION_OPEN_SOURCE_MESA && + !screen->info.feats.features.geometryShader && + !screen->info.feats.features.tessellationShader && + screen->info.props.limits.maxVertexOutputComponents > 64 && + screen->info.props.limits.maxVertexOutputComponents < 120) + caps->max_varyings = 16; caps->dmabuf = #if defined(HAVE_LIBDRM) && (DETECT_OS_LINUX || DETECT_OS_BSD) @@ -3736,6 +3750,26 @@ sort_mem_types(struct zink_screen *screen, enum zink_heap heap) } } +static unsigned +zink_default_frames_in_flight(VkDriverId driver_id) +{ + /* Tile-based GPUs can overlap one frame's geometry phase with the previous + * frame's fragment phase. Add drivers here as measurements justify it. + */ + static const struct { + VkDriverId driver_id; + unsigned frames; + } defaults[] = { + { VK_DRIVER_ID_IMAGINATION_OPEN_SOURCE_MESA, 2 }, + }; + + for (unsigned i = 0; i < ARRAY_SIZE(defaults); i++) { + if (defaults[i].driver_id == driver_id) + return defaults[i].frames; + } + return 1; +} + static struct zink_screen * zink_internal_create_screen(const struct pipe_screen_config *config, int64_t dev_major, int64_t dev_minor, uint64_t adapter_luid) { @@ -3856,6 +3890,7 @@ zink_internal_create_screen(const struct pipe_screen_config *config, int64_t dev screen->is_g5p = screen->info.driver_props.driverID == VK_DRIVER_ID_IMAGINATION_OPEN_SOURCE_MESA && !strcmp(screen->info.driver_props.driverName, "Mesa PowerVR G5P"); screen->precise_sync = screen->is_g5p; + screen->frames_in_flight = zink_default_frames_in_flight(screen->info.driver_props.driverID); /* after the device is known: G5P has driconf defaults of its own (device "g5p") */ if (config) { @@ -3866,6 +3901,9 @@ zink_internal_create_screen(const struct pipe_screen_config *config, int64_t dev //screen->driconf.inline_uniforms = driQueryOptionb(config->options, "radeonsi_inline_uniforms"); screen->driconf.emulate_point_smooth = driQueryOptionb(config->options, "zink_emulate_point_smooth"); screen->driconf.zink_shader_object_enable = driQueryOptionb(config->options, "zink_shader_object_enable"); + unsigned frames = driQueryOptioni(config->options, "zink_frames_in_flight"); + if (frames) + screen->frames_in_flight = frames; } if (!screen->info.rb2_feats.nullDescriptor) { diff --git a/src/gallium/drivers/zink/zink_synchronization.cpp b/src/gallium/drivers/zink/zink_synchronization.cpp index b389933cf56..78760f2e5a5 100644 --- a/src/gallium/drivers/zink/zink_synchronization.cpp +++ b/src/gallium/drivers/zink/zink_synchronization.cpp @@ -544,7 +544,7 @@ zink_resource_image_transfer_dst_barrier(struct zink_context *ctx, struct zink_r } bool -zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size) +zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size, bool unsync) { if (res->obj->copies_need_reset) zink_resource_copies_reset(res); @@ -555,7 +555,12 @@ zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_ /* must barrier if something read the valid buffer range */ bool valid_read = (res->obj->access || res->obj->unordered_access) && util_ranges_intersect(&res->valid_buffer_range, offset, offset + size) && !can_unordered_write; - if (valid_read || zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics || zink_check_unordered_transfer_access(res, 0, &box)) { + /* the reordered cmdbuf runs after earlier batches only in submission order, which orders nothing: + * a range an unfinished earlier batch may still read needs a barrier */ + bool earlier_read = !unsync && !zink_resource_usage_matches(res, ctx->bs) && + !zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, + ZINK_RESOURCE_ACCESS_READ); + if (valid_read || earlier_read || zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics || zink_check_unordered_transfer_access(res, 0, &box)) { zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); unordered = res->obj->unordered_write; } else { @@ -567,6 +572,10 @@ zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_ res->obj->access = VK_ACCESS_TRANSFER_WRITE_BIT; res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; res->obj->ordered_access_is_copied = true; + /* this batch's reads of the bound buffer are ordered by the reordered cmdbuf's final barrier, but they must + * still reach obj->access for the next batch's barrier (earlier_read) */ + if (!unsync) + resource_check_defer_buffer_barrier(ctx, res, VK_PIPELINE_STAGE_TRANSFER_BIT); } } zink_resource_copy_box_add(ctx, res, 0, &box); @@ -630,6 +639,14 @@ zink_resource_memory_barrier(struct zink_context *ctx, struct zink_resource *res } if (!resource_needs_barrier(res, flags, pipeline, unordered)) return; + + /* An unfinished earlier batch's unordered accesses are still pending: keep them + * as ordered history before this batch's first barrier resets them, ordered or not. */ + if (!usage_matches && !completed) { + res->obj->access |= res->obj->unordered_access; + res->obj->access_stage |= res->obj->unordered_access_stage; + } + if (completed) { /* reset access on complete */ res->obj->access = VK_ACCESS_NONE; diff --git a/src/gallium/drivers/zink/zink_types.h b/src/gallium/drivers/zink/zink_types.h index 436855b8be4..8533ae6ff6c 100644 --- a/src/gallium/drivers/zink/zink_types.h +++ b/src/gallium/drivers/zink/zink_types.h @@ -1503,6 +1503,7 @@ struct zink_video_format_prop { struct zink_screen { /* Must stay first: zink_screen() casts a pipe_screen pointer directly. */ struct pipe_screen base; + unsigned frames_in_flight; /* pipe_caps.throttle_frames */ bool precise_sync; bool is_g5p; diff --git a/src/gallium/frontends/dri/dri_drawable.c b/src/gallium/frontends/dri/dri_drawable.c index bdf31f5cdd9..d955111b1dd 100644 --- a/src/gallium/frontends/dri/dri_drawable.c +++ b/src/gallium/frontends/dri/dri_drawable.c @@ -264,8 +264,9 @@ dri_destroy_drawable(struct dri_drawable *drawable) for (i = 0; i < ST_ATTACHMENT_COUNT; i++) pipe_resource_reference(&drawable->msaa_textures[i], NULL); - screen->base.screen->fence_reference(screen->base.screen, - &drawable->throttle_fence, NULL); + for (i = 0; i < DRI_THROTTLE_MAX_FRAMES; i++) + screen->base.screen->fence_reference(screen->base.screen, + &drawable->throttle_fence[i], NULL); /* Notify the st manager that this drawable is no longer valid */ st_api_destroy_drawable(&drawable->base); @@ -686,17 +687,11 @@ dri_flush(struct dri_context *ctx, (reason == __DRI2_THROTTLE_SWAPBUFFER || reason == __DRI2_THROTTLE_FLUSHFRONT)) { - struct pipe_screen *screen = drawable->screen->base.screen; struct pipe_fence_handle *new_fence = NULL; st_context_flush(st, flush_flags, &new_fence, args.ctx ? notify_before_flush_cb : NULL, &args); - /* throttle on the previous fence */ - if (drawable->throttle_fence) { - screen->fence_finish(screen, NULL, drawable->throttle_fence, OS_TIMEOUT_INFINITE); - screen->fence_reference(screen, &drawable->throttle_fence, NULL); - } - drawable->throttle_fence = new_fence; + dri_throttle_frame(drawable, new_fence); } else if (flags & (__DRI2_FLUSH_DRAWABLE | __DRI2_FLUSH_CONTEXT)) { st_context_flush(st, flush_flags, NULL, args.ctx ? notify_before_flush_cb : NULL, &args); @@ -739,6 +734,29 @@ dri_flush_drawable(struct dri_drawable *dPriv) dri_flush(ctx, dPriv, __DRI2_FLUSH_DRAWABLE, -1); } +/** + * Throttle on the frame screen->throttle_frames swaps back (the previous one + * by default), then keep new_fence (a reference this takes over) for a later + * swap. With throttle_frames N, up to N submitted frames stay in flight while + * the application records the next one. + */ +void +dri_throttle_frame(struct dri_drawable *drawable, + struct pipe_fence_handle *new_fence) +{ + struct pipe_screen *screen = drawable->screen->base.screen; + const unsigned frames = drawable->screen->throttle_frames; + struct pipe_fence_handle **slot = + &drawable->throttle_fence[drawable->throttle_slot % frames]; + + if (*slot) { + screen->fence_finish(screen, NULL, *slot, OS_TIMEOUT_INFINITE); + screen->fence_reference(screen, slot, NULL); + } + *slot = new_fence; + drawable->throttle_slot = (drawable->throttle_slot + 1) % frames; +} + /** * dri_throttle - A DRI2ThrottleExtension throttling function. */ diff --git a/src/gallium/frontends/dri/dri_drawable.h b/src/gallium/frontends/dri/dri_drawable.h index 58be595e1a6..be6d67b0289 100644 --- a/src/gallium/frontends/dri/dri_drawable.h +++ b/src/gallium/frontends/dri/dri_drawable.h @@ -33,6 +33,9 @@ #include "frontend/api.h" #include "dri_util.h" +/* most frames a throttled swap lets be in flight (pipe_caps.throttle_frames) */ +#define DRI_THROTTLE_MAX_FRAMES 3 + #ifdef VK_USE_PLATFORM_XCB_KHR #include #endif @@ -59,7 +62,9 @@ struct dri_drawable unsigned int texture_mask, texture_stamp; int swap_interval; - struct pipe_fence_handle *throttle_fence; + /* the last frames' fences, one slot per frame in flight (dri_throttle_frame()) */ + struct pipe_fence_handle *throttle_fence[DRI_THROTTLE_MAX_FRAMES]; + unsigned throttle_slot; bool flushing; /* prevents recursion in dri_flush */ /** @@ -137,6 +142,10 @@ dri_flush(struct dri_context *ctx, unsigned flags, enum __DRI2throttleReason reason); +void +dri_throttle_frame(struct dri_drawable *drawable, + struct pipe_fence_handle *new_fence); + void dri_flush_drawable(struct dri_drawable *dPriv); diff --git a/src/gallium/frontends/dri/dri_screen.c b/src/gallium/frontends/dri/dri_screen.c index cb43894b194..ce3b893b0c4 100644 --- a/src/gallium/frontends/dri/dri_screen.c +++ b/src/gallium/frontends/dri/dri_screen.c @@ -30,6 +30,7 @@ */ #include "dri_screen.h" +#include "dri_drawable.h" #include "dri_context.h" #include "dri_helpers.h" @@ -636,6 +637,7 @@ dri_init_screen(struct dri_screen *screen, screen->target = PIPE_TEXTURE_RECT; screen->throttle = pscreen->caps.throttle; + screen->throttle_frames = CLAMP(pscreen->caps.throttle_frames, 1, DRI_THROTTLE_MAX_FRAMES); if (pscreen->caps.device_protected_context) screen->has_protected_context = true; screen->has_reset_status_query = pscreen->caps.device_reset_status_query; diff --git a/src/gallium/frontends/dri/dri_screen.h b/src/gallium/frontends/dri/dri_screen.h index 0f345b4cc02..ae96a80a940 100644 --- a/src/gallium/frontends/dri/dri_screen.h +++ b/src/gallium/frontends/dri/dri_screen.h @@ -72,6 +72,7 @@ struct dri_screen unsigned int api_mask; bool throttle; + unsigned throttle_frames; /* 1..DRI_THROTTLE_MAX_FRAMES */ bool dmabuf_import; bool has_multibuffer; diff --git a/src/gallium/frontends/dri/dri_util.c b/src/gallium/frontends/dri/dri_util.c index d64aca22854..9acedef54c1 100644 --- a/src/gallium/frontends/dri/dri_util.c +++ b/src/gallium/frontends/dri/dri_util.c @@ -66,6 +66,8 @@ driOptionDescription __dri2ConfigOptions[] = { DRI_CONF_SECTION_PERFORMANCE DRI_CONF_VBLANK_MODE(DRI_CONF_VBLANK_DEF_INTERVAL_1) + DRI_CONF_OPT_I(egl_wayland_max_buffers, 4, 2, 4, + "Cap on EGL Wayland window-surface colour buffers") DRI_CONF_BLOCK_ON_DEPLETED_BUFFERS(false) DRI_CONF_SECTION_END }; diff --git a/src/gallium/frontends/dri/kopper.c b/src/gallium/frontends/dri/kopper.c index e4700c82a76..3320e941d6c 100644 --- a/src/gallium/frontends/dri/kopper.c +++ b/src/gallium/frontends/dri/kopper.c @@ -383,7 +383,6 @@ kopper_flush_frontbuffer(struct dri_context *ctx, if (ptex) { ctx->st->pipe->flush_resource(ctx->st->pipe, drawable->textures[ST_ATTACHMENT_FRONT_LEFT]); - struct pipe_screen *screen = drawable->screen->base.screen; struct st_context *st; struct pipe_fence_handle *new_fence = NULL; @@ -392,12 +391,7 @@ kopper_flush_frontbuffer(struct dri_context *ctx, st_context_flush(st, ST_FLUSH_FRONT, &new_fence, NULL, NULL); drawable->flushing = false; - /* throttle on the previous fence */ - if (drawable->throttle_fence) { - screen->fence_finish(screen, NULL, drawable->throttle_fence, OS_TIMEOUT_INFINITE); - screen->fence_reference(screen, &drawable->throttle_fence, NULL); - } - drawable->throttle_fence = new_fence; + dri_throttle_frame(drawable, new_fence); kopper_copy_to_front(st->pipe, ctx->draw, ptex, 0, NULL); } diff --git a/src/gallium/include/pipe/p_defines.h b/src/gallium/include/pipe/p_defines.h index ac8a73ab19e..a1871e3de7d 100644 --- a/src/gallium/include/pipe/p_defines.h +++ b/src/gallium/include/pipe/p_defines.h @@ -1030,6 +1030,7 @@ struct pipe_caps { bool image_load_formatted; bool image_store_formatted; bool throttle; + unsigned throttle_frames; bool cl_gl_sharing; bool prefer_compute_for_multimedia; bool fragment_shader_interlock; diff --git a/src/imagination/common/device_info/apple-a9.h b/src/imagination/common/device_info/apple-a9.h index 1bd1bb2d47d..6627b624ed8 100644 --- a/src/imagination/common/device_info/apple-a9.h +++ b/src/imagination/common/device_info/apple-a9.h @@ -30,6 +30,23 @@ static const struct pvr_device_features pvr_device_features_9_V_6_2 = { .has_apple_g5p = true, + .has_apple_compute_dispatch = true, + /* AGXMetalA9 ComputeProgramVariant -> +0xb08 (maximum threadgroup size). + * Bank 0 uses DOUTU temps; bank 1 uses the kernel-record allocation. + */ + .apple_compute_dispatch = { + .bank0_budget = 3329U, + .bank1_budget = 393216U, + .register_granularity = 8U, + .bank0_scale = 4U, + .bank0_rounding_bias = 28U, + .bank0_granularity = 32U, + .bank1_scale = 64U, + .bank1_rounding_bias = 448U, + .bank1_granularity = 512U, + .task_size = 32U, + .max_workgroup_size = 512U, + }, .has_num_clusters = true, /* A9 */ .has_tile_size_x = true, /* A9 */ .has_tile_size_y = true, /* A9 */ diff --git a/src/imagination/common/pvr_compute_limits.h b/src/imagination/common/pvr_compute_limits.h new file mode 100644 index 00000000000..a9ad3b991e8 --- /dev/null +++ b/src/imagination/common/pvr_compute_limits.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef PVR_COMPUTE_LIMITS_H +#define PVR_COMPUTE_LIMITS_H + +#include "hwdef/rogue_hw_utils.h" + +/* Check residency, independently of the API's workgroup size limits. + * max_lanes reports the largest resident workgroup for this allocation. + */ +static inline bool +pvr_compute_wg_residency_ok(const struct pvr_device_info *dev_info, + unsigned regs, + unsigned bank1_regs, + unsigned wg_size, + bool has_barriers, + unsigned *max_lanes) +{ + if (PVR_HAS_FEATURE(dev_info, apple_compute_dispatch)) { + const struct pvr_apple_compute_dispatch_limits *limits = + &dev_info->features.apple_compute_dispatch; + const unsigned t = ALIGN_POT(MAX2(regs, 1), limits->register_granularity); + const unsigned f = ALIGN_POT(bank1_regs ? bank1_regs : regs, + limits->register_granularity); + const unsigned a = (limits->bank0_scale * t + limits->bank0_rounding_bias) & + ~(limits->bank0_granularity - 1); + const unsigned b = (limits->bank1_scale * f + limits->bank1_rounding_bias) & + ~(limits->bank1_granularity - 1); + unsigned max = MIN2(a ? limits->task_size * ((limits->bank0_budget - a) / a) : + limits->max_workgroup_size, + b ? limits->bank1_budget / b - limits->task_size : + limits->max_workgroup_size); + + /* Apple reserves one task in each bank. This matches Metal's + * maxTotalThreadsPerThreadgroup, including for barriered workgroups. + */ + max = MIN2(max, limits->max_workgroup_size); + if (max > limits->task_size) + max &= ~(limits->task_size - 1); + *max_lanes = max; + return wg_size <= max; + } + + UNREACHABLE("compute residency requires Apple dispatch limits"); +} + +#endif /* PVR_COMPUTE_LIMITS_H */ diff --git a/src/imagination/common/pvr_debug.c b/src/imagination/common/pvr_debug.c index e910f4b217f..5e97f7cda14 100644 --- a/src/imagination/common/pvr_debug.c +++ b/src/imagination/common/pvr_debug.c @@ -38,6 +38,12 @@ static const struct debug_named_value debug_control[] = { "Dump descriptor set and pipeline layouts." }, { "info", PVR_DEBUG_INFO, "Display information about the driver and device." }, + { "shader_dump", PVR_DEBUG_SHADER_DUMP, "Capture shader compilation inputs and results." }, + { "capture_stub", PVR_DEBUG_CAPTURE_STUB, "Replace refused shaders with diagnostic stubs; renders incorrect pixels." }, + { "errors", PVR_DEBUG_ERRORS, "Log every Vulkan error the driver returns, with its origin." }, + { "compute_log", PVR_DEBUG_COMPUTE_LOG, "Log compute pipelines and dispatches." }, + { "bo_log", PVR_DEBUG_BO_LOG, + "Log device-memory and driver BO allocations with size, heap and purpose; live-set summary on any failed allocation and at device destroy." }, DEBUG_NAMED_VALUE_END }; /* clang-format on */ diff --git a/src/imagination/common/pvr_debug.h b/src/imagination/common/pvr_debug.h index 33d22424ab1..86198a951f5 100644 --- a/src/imagination/common/pvr_debug.h +++ b/src/imagination/common/pvr_debug.h @@ -38,6 +38,15 @@ extern uint32_t PVR_DEBUG; #define PVR_DEBUG_TRACK_BOS BITFIELD_BIT(1) #define PVR_DEBUG_VK_DUMP_DESCRIPTOR_SET_LAYOUT BITFIELD_BIT(2) #define PVR_DEBUG_INFO BITFIELD_BIT(3) +#define PVR_DEBUG_BO_LOG BITFIELD_BIT(4) + +#define PVR_DEBUG_SHADER_DUMP BITFIELD_BIT(6) + +#define PVR_DEBUG_CAPTURE_STUB BITFIELD_BIT(7) + +#define PVR_DEBUG_ERRORS BITFIELD_BIT(8) + +#define PVR_DEBUG_COMPUTE_LOG BITFIELD_BIT(9) void pvr_process_debug_variable(void); diff --git a/src/imagination/common/pvr_device_info.h b/src/imagination/common/pvr_device_info.h index 9731be888e4..ea6b3471f27 100644 --- a/src/imagination/common/pvr_device_info.h +++ b/src/imagination/common/pvr_device_info.h @@ -253,6 +253,21 @@ struct pvr_device_ident { const char *public_name; }; +/* Per-pipeline residency parameters for Apple's compute dispatch. */ +struct pvr_apple_compute_dispatch_limits { + uint32_t bank0_budget; + uint32_t bank1_budget; + uint32_t register_granularity; + uint32_t bank0_scale; + uint32_t bank0_rounding_bias; + uint32_t bank0_granularity; + uint32_t bank1_scale; + uint32_t bank1_rounding_bias; + uint32_t bank1_granularity; + uint32_t task_size; + uint32_t max_workgroup_size; +}; + struct pvr_device_features { /* * Apple A-chip GPUs (G9P on the A10): Rogue-like data masters, PDS and @@ -360,6 +375,10 @@ struct pvr_device_features { * other field's position. */ bool has_apple_g5p : 1; + + /* Only set where Apple's compute residency parameters are established. */ + bool has_apple_compute_dispatch : 1; + struct pvr_apple_compute_dispatch_limits apple_compute_dispatch; }; struct pvr_device_enhancements { diff --git a/src/imagination/g5p/g5p_barrier.h b/src/imagination/g5p/g5p_barrier.h index 9403f0fe0e0..a9b01688410 100644 --- a/src/imagination/g5p/g5p_barrier.h +++ b/src/imagination/g5p/g5p_barrier.h @@ -10,7 +10,7 @@ static inline void g5p_shared_barrier_fenced(struct g9p_code *c, unsigned r, unsigned T, unsigned k, unsigned st, unsigned N, int fence_pair) { - assert(!c->fragment && k < 128 && N && N <= 31); + assert(!c->fragment && k < 128 && N && N <= 15); g9p_wait(c); G9P_EMIT(c, 0x1181, 0x0101 | T << 9, 0x0120, 0xa5bd, 0x0070); unsigned skip = g9p_jump(c, 0x0120, 0); diff --git a/src/imagination/g5p/g5p_fs_common.h b/src/imagination/g5p/g5p_fs_common.h index d731dbd7f88..eab1f4661df 100644 --- a/src/imagination/g5p/g5p_fs_common.h +++ b/src/imagination/g5p/g5p_fs_common.h @@ -24,7 +24,7 @@ g5p_stage_common_plan(unsigned start, unsigned textures, unsigned samplers, uint64_t push_dwords, struct g5p_fs_common_plan *p) { if (start > 4 || textures > 16 || samplers > 17 || - dynamic_ptrs > 16 || size_queries > 4 || ubos > 8) + dynamic_ptrs > 16 || size_queries > 4 || ubos > 13) return false; *p = (struct g5p_fs_common_plan){0}; /* Each retry only enables one of two monotone choices. A layout which diff --git a/src/imagination/g5p/g5p_image.h b/src/imagination/g5p/g5p_image.h index 709f6f41487..f128ae00828 100644 --- a/src/imagination/g5p/g5p_image.h +++ b/src/imagination/g5p/g5p_image.h @@ -2,26 +2,36 @@ #ifndef G5P_IMAGE_H #define G5P_IMAGE_H #include "../g9p/g9p_isa.h" -/* RE ยง4 raw32-bit-channel image access. Format conversion is not implicit: - * only the pipeline's witnessed/probed raw format may use this form. - * - * The coordinate halfword's bit 15 (the sample encoder's coord_control) is - * set on reads only. Apple's A9 compiler (build/gpu/compute-isa-re out-img, - * out-image32: every texture write kernel) writes through integer - * coordinates as `f51a 01c0 0900 ...` with that bit clear; the bit appears on - * a write only with opcode f518 after the coordinates were converted to - * float. The old `f51a ... 8900` store mixed the two: its pixels reached - * memory, but a later job's imageLoad read pre-store data for the last - * workgroups written (release9d imgdiag1: 384 of 4096 words, always whole - * 8x8 groups of the tail, with or without glFinish between the jobs). */ +/* Apple's captured image and array shaders pack 16/8-bit channels into two dwords (the sample class with bit 1 + * clear, as g5p_usc_sample()'s narrow result) or supply four raw dwords; the captured array shaders use the 2D + * array dimension with integer x,y,layer coordinates. W0: that class byte and the dimension byte (all four + * components in its high nibble and the dimension in its low one, as g5p_usc_sample_ext()); W1: the data + * register's half, the integer-coordinate bit 6 (texture_base()'s read bit), the store bit 7 and the descriptor's + * qword at bit 9. Keep raw32's qualified coordinate control and completion unchanged. */ +#define G5P_IMAGE_CLASS 0x1a /* texture_base()'s sample class */ +#define G5P_IMAGE_WIDE 0x02 /* clear: 16/8-bit channels packed into two dwords */ +#define G5P_IMAGE_RGBA 0xf0 +#define G5P_IMAGE_DIM_2D 0x5 +#define G5P_IMAGE_INTEGER_COORDS 0x40 +#define G5P_IMAGE_STORE 0x80 static inline void -g5p_image_raw32(struct g9p_code *c, bool store, unsigned data, - unsigned coords, unsigned descriptor_dword, unsigned lod_dword) +g5p_image_access(struct g9p_code *c, bool store, bool packed, bool array, + unsigned data, unsigned coords, unsigned descriptor_dword, unsigned lod_dword) { - assert(!c->fragment && data+3<32 && coords+1<32 && !(descriptor_dword&1) && lod_dword<32); - G9P_EMIT(c,0xf51a,0x0100 | (descriptor_dword/2)<<9 | (store?0xc0:0x40) | 2*data, + assert(!c->fragment && data + (packed ? 1 : 3) < 32 && coords + (array ? 2 : 1) < 32 && + !(descriptor_dword & 1) && lod_dword < 32); + const unsigned dim = G5P_IMAGE_RGBA | (array ? G9P_TEX_2D_ARRAY : G5P_IMAGE_DIM_2D); + G9P_EMIT(c, dim << 8 | (G5P_IMAGE_CLASS & ~(packed ? G5P_IMAGE_WIDE : 0)), + G9P_CONT | (descriptor_dword / 2) << 9 | G5P_IMAGE_INTEGER_COORDS | (store ? G5P_IMAGE_STORE : 0) | + 2 * data, (store?0x0100:0x8100) | coords<<9 | (store?0:8),0x0140 | lod_dword, store?0x8100:0x8180,0x0008); if(!store)g9p_wait(c); } +static inline void +g5p_image_raw32(struct g9p_code *c, bool store, unsigned data, + unsigned coords, unsigned descriptor_dword, unsigned lod_dword) +{ + g5p_image_access(c, store, false, false, data, coords, descriptor_dword, lod_dword); +} #endif diff --git a/src/imagination/g5p/g5p_mrt_out.h b/src/imagination/g5p/g5p_mrt_out.h index 144cf91f826..c02e2161fec 100644 --- a/src/imagination/g5p/g5p_mrt_out.h +++ b/src/imagination/g5p/g5p_mrt_out.h @@ -23,7 +23,9 @@ #include #include -#define G5P_MRT_MAX_TARGETS 4 +/* 1..4: packed-f16 targets (two dwords each); 5..8: compact UNORM8 targets + * (one packed dword each, verified on the A9 over 104 test images) */ +#define G5P_MRT_MAX_TARGETS 8 #define G5P_MRT_FS_SLOTS 8 /* FRAG_RESULT_DATA0..DATA7 */ #define G5P_MRT_HALVES_PER_TARGET 4 #define G5P_MRT_MAX_HALVES (G5P_MRT_MAX_TARGETS * G5P_MRT_HALVES_PER_TARGET) @@ -49,7 +51,7 @@ static inline const char *g5p_mrt_refusal_str(enum g5p_mrt_refusal r) case G5P_MRT_NO_TARGETS: return "no colour targets"; case G5P_MRT_TOO_MANY_TARGETS: - return "more than four colour targets"; + return "more than eight colour targets"; case G5P_MRT_THREE_TARGETS: return "three colour targets: Apple's compiler was only observed for 1, 2 and 4 " "(the pixel output width 6 is not a kernel width)"; @@ -106,7 +108,8 @@ static inline unsigned g5p_mrt_emit_words(unsigned field) /* Word counts for which an Apple A9 witness exists for a homogeneous set. */ static inline bool g5p_mrt_emit_words_witnessed(unsigned words) { - return words == 1 || words == 2 || words == 4 || words == 8; + /* 1, 2, 4, 8 (homogeneous 8-bit RGBA and raw sets), 5..7 (compact UNORM8) */ + return words >= 1 && words <= 8 && words != 3; } static inline enum g5p_mrt_refusal g5p_mrt_out_plan_make(unsigned targets, struct g5p_mrt_out_plan *p) @@ -119,11 +122,13 @@ static inline enum g5p_mrt_refusal g5p_mrt_out_plan_make(unsigned targets, struc if (targets == 3) return G5P_MRT_THREE_TARGETS; p->targets = targets; - p->words = targets * G5P_MRT_WORDS_PER_TARGET; + /* 5..8: one packed UNORM8 dword per target (the compact layout), the + * pass at the kernel's widest pixel output width 8 */ + p->words = targets > 4 ? targets : targets * G5P_MRT_WORDS_PER_TARGET; p->emit_field = g5p_mrt_emit_field(p->words); - p->pixel_output_width = p->words; /* [D] 2/4/8, matches the kernel table; mrt2/mrt4 capture confirms */ + p->pixel_output_width = targets > 4 ? 8 : p->words; /* [D] 2/4/8, matches the kernel table; mrt2/mrt4 capture confirms */ for (unsigned t = 0; t < targets; t++) - p->first_half[t] = g5p_mrt_half(t, 0); + p->first_half[t] = targets > 4 ? 2 * t : g5p_mrt_half(t, 0); return G5P_MRT_OK; } diff --git a/src/imagination/g5p/g5p_mrt_programs.h b/src/imagination/g5p/g5p_mrt_programs.h index cbecff1198c..73810cb749f 100644 --- a/src/imagination/g5p/g5p_mrt_programs.h +++ b/src/imagination/g5p/g5p_mrt_programs.h @@ -27,24 +27,28 @@ #include #include -#define G5P_MRT_MAX 4 +#include "g5p_usc.h" + +#define G5P_MRT_MAX 8 /* 5..8: compact UNORM8 targets */ #define G5P_MRT_PBE_DWORDS 7 /* per target */ #define G5P_MRT_PBE_STRIDE 8 /* dwords per target in the common store */ +#define G5P_MRT_PBE_HALF_OFFSET 6 /* a target's output-half offset in its PBE block */ #define G5P_MRT_CLEAR_DWORDS 4 /* f32 clear colour per target */ #define G5P_MRT_CLEAR_USC_MAX (16 * 4 * G5P_MRT_MAX + 8) -#define G5P_MRT_EOT_USC_MAX 130 +#define G5P_MRT_EOT_USC_MAX 194 -/* Target counts with an Apple witness for the whole pass: 1, 2 and 4. */ +/* Target counts with an Apple witness for the whole pass: 1, 2 and 4; 5..8 + * as compact UNORM8 (verified on the A9 over 104 test images). */ static inline bool g5p_mrt_count_ok(unsigned n) { - return n == 1 || n == 2 || n == 4; + return n == 1 || n == 2 || (n >= 4 && n <= 8); } /* Pixel output width of the pass (UAPI pixel_output_width): two output * registers per 8-bit RGBA target. iOS record+0x6dc: 2, 4, 8. */ static inline uint32_t g5p_mrt_pixel_width(unsigned n) { - return 2 * n; + return n > 4 ? 8 : 2 * n; /* compact: one dword per target, the kernel's width 8 */ } /* PBE dwords the end-of-tile DOUTD moves: 7, 15, 31. */ @@ -58,7 +62,8 @@ static inline unsigned g5p_mrt_pbe_dwords(unsigned n) * 0x402, 0x402, 0x802; Apple's sizes are twice DIV_ROUND_UP(dwords, 16)). */ static inline uint32_t g5p_mrt_eot_info(unsigned n) { - return n == 4 ? 0x802 : 0x402; + /* the same values for 1, 2, 4; 0xc02 and 0x1002 for 5..8 (39..63 dwords) */ + return ((g5p_mrt_pbe_dwords(n) + 15) / 16) * 0x400 | 2; } /* Partial-render restore SIZEINFO (restore[1], record+0x51c): usc_sharedsize @@ -92,9 +97,39 @@ static inline unsigned g5p_mrt_pbe_block(uint32_t *out, unsigned n, const uint64 return g5p_mrt_pbe_block_halves(out, n, pbe, 4); } +/* 5..8 UNORM targets: one packed dword per target; BGRA8 + * byte order is B,G,R,A. Each f32 clear channel in common dword 4t + c is + * converted to UNORM8 into one byte of an output half (g5p_usc_unorm8_byte(), + * the packing form recovered from controlled probes), then one emitpix + * writes the N packed dwords and the program ends. Returns its size, + * 16 * 4N + 6, or 0 when an encoder refuses (never for 5..8 targets). */ +static inline unsigned g5p_mrt_clear_compact(uint8_t *out, unsigned n, bool bgra) +{ + unsigned o = 0, len; + for (unsigned t = 0; t < n; t++) + for (unsigned h = 0; h < 2; h++) + for (unsigned byte = 0; byte < 2; byte++) { + const unsigned c = h * 2 + byte; + const unsigned channel = bgra && !(c & 1) ? 2 - c : c; + const unsigned src = 4 * t + channel, dst = 2 * t + h; + len = g5p_usc_unorm8_byte(out + o, dst, byte, src); + if (!len) + return 0; + o += len; + } + /* emitpix of the N packed dwords from output half 0, then END */ + len = g5p_usc_emit_mrt(out + o, n, 0); + if (!len) + return 0; + o += len; + return o + g5p_usc_special(out + o, true); +} + /* The clear program. Returns its size, 16 * 4N + 6. */ static inline unsigned g5p_mrt_clear_usc(uint8_t *out, unsigned n) { + if (n > 4) + return g5p_mrt_clear_compact(out, n, true); unsigned o = 0; for (unsigned t = 0; t < n; t++) for (int c = 3; c >= 0; c--) { @@ -111,16 +146,17 @@ static inline unsigned g5p_mrt_clear_usc(uint8_t *out, unsigned n) } /* The end-of-tile program. Returns its size: 66 for one or two targets, 130 - * for four. */ + * for four; 0 when an encoder refuses (never for the counts g5p_mrt_count_ok() accepts). */ static inline unsigned g5p_mrt_eot_usc_mode(uint8_t *out, unsigned n, bool integer) { unsigned o = 0; for (unsigned k = 0; k < n; k++) { - uint8_t t = n - 1 - k; - const uint8_t ins[12] = {0x88, 0x01, (uint8_t)(2 * t), 0x01, (uint8_t)(0x99 + 0x20 * k), 0x01, - 0x00, 0x41, 0x28, 0x01, 0x39, 0x00}; - memcpy(out + o, ins, sizeof(ins)); - o += sizeof(ins); + /* output half 2 * (n - 1 - k) = target k's output-half offset (in its PBE block) */ + const unsigned len = g5p_usc_mov_shared_half(out + o, 2 * (n - 1 - k), + G5P_MRT_PBE_STRIDE * k + G5P_MRT_PBE_HALF_OFFSET); + if (!len) + return 0; + o += len; } for (unsigned k = 0; k < n; k++) { bool last = k + 1 == n; @@ -149,6 +185,7 @@ static inline unsigned g5p_mrt_eot_usc_mode(uint8_t *out, unsigned n, bool integ */ static inline unsigned g5p_mrt_eot_usc(uint8_t *out, unsigned n) { - return g5p_mrt_eot_usc_mode(out, n, false); + /* 5..8: packed bytes through a physical R32UI store, the integer source mode */ + return g5p_mrt_eot_usc_mode(out, n, n > 4); } #endif diff --git a/src/imagination/g5p/g5p_shared_atomic.h b/src/imagination/g5p/g5p_shared_atomic.h new file mode 100644 index 00000000000..56a98f6581c --- /dev/null +++ b/src/imagination/g5p/g5p_shared_atomic.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef G5P_SHARED_ATOMIC_H +#define G5P_SHARED_ATOMIC_H +#include "compiler/nir/nir.h" +#include "../g9p/g9p_isa.h" + +/* The exclusive (load-linked, store-conditional) form of g9p_shared_access(): Apple's A9 LL/SC loop sets the size + * field to 16, W0 bit 15 alone (observed role). The retry jump is g5p_barrier.h's LL/SC retry: G9P_JMP_ANY with + * the condition bits 5..6 set. */ +#define G5P_SHARED_EXCLUSIVE 16u +#define G5P_JMP_SC_RETRY (G9P_JMP_ANY | 3u << 5) + +static inline void +g5p_shared_exclusive(struct g9p_code *c, bool store, unsigned r, + unsigned k, int offset_half) +{ + g9p_shared_access(c, G5P_SHARED_EXCLUSIVE, store, r, k, offset_half); +} + +/* The device-proven g5p_shared_barrier_fenced() LL/SC retry structure. + * SC always runs, including compare failure, so cmpxchg is strong. */ +static inline void +g5p_shared_atomic(struct g9p_code *c, nir_atomic_op op, unsigned old, + unsigned val, unsigned cmp, unsigned next, unsigned k, + int offset_half, unsigned st) +{ + assert(!c->fragment && k < 4096 && old < 32 && next < 32); + c->uses_stack = true; + g9p_wait(c); + /* push two levels */ + G9P_EMIT(c, G9P_W0_STACK, g9p_st_dst(c, st), g9p_st_src(c, st), + G9P_STACK_CTL | G9P_STACK_PUSH | 2 << G9P_STACK_LEVELS_SHIFT); + unsigned ll = c->n; + g5p_shared_exclusive(c, false, old, k, offset_half); + g9p_wait(c); + switch (op) { + case nir_atomic_op_iadd: + g9p_alu2(c, G9P_IADD, next, old, val, false); break; + case nir_atomic_op_iand: + g9p_alu2(c, G9P_AND, next, old, val, false); break; + case nir_atomic_op_ior: + g9p_alu2(c, G9P_OR, next, old, val, false); break; + case nir_atomic_op_ixor: + g9p_alu2(c, G9P_XOR, next, old, val, false); break; + case nir_atomic_op_xchg: + g9p_alu2(c, G9P_OR, next, val, val, false); break; + case nir_atomic_op_imin: + case nir_atomic_op_umin: + case nir_atomic_op_imax: + case nir_atomic_op_umax: + case nir_atomic_op_cmpxchg: { + bool cas = op == nir_atomic_op_cmpxchg; + bool is_min = op == nir_atomic_op_imin || op == nir_atomic_op_umin; + unsigned cond = cas ? G9P_EQ : + op == nir_atomic_op_imin || op == nir_atomic_op_imax ? G9P_SLT : G9P_ULT; + unsigned x = cas ? val : is_min ? old : val; + unsigned y = cas ? old : is_min ? val : old; + /* Same compare/select sequence as g9p_compile.c emit_select(). */ + g9p_add_imm(c, next, y, 0); + g9p_cmp_reg(c, cond, old, cas ? cmp : val, false); + g9p_push_if(c, st, false); + g9p_add_imm(c, next, x, 0); + g9p_pop_levels(c, st, 1); + break; + } + default: UNREACHABLE("unsupported shared atomic"); + } + g5p_shared_exclusive(c, true, next, k, offset_half); + unsigned retry = g9p_jump(c, G5P_JMP_SC_RETRY, 0); + g9p_jump_patch(c, retry, 2 * ((int)ll - (int)retry)); + /* pop both levels, the stack register still live (no last-read bit) */ + G9P_EMIT(c, G9P_W0_STACK, g9p_st_dst(c, st), g9p_st_src(c, st), + G9P_STACK_CTL | G9P_STACK_POP | 2 << G9P_STACK_LEVELS_SHIFT); + g9p_wait(c); +} +#endif diff --git a/src/imagination/g5p/g5p_usc.c b/src/imagination/g5p/g5p_usc.c index 8ada20f53a8..ef9c7c6c372 100644 --- a/src/imagination/g5p/g5p_usc.c +++ b/src/imagination/g5p/g5p_usc.c @@ -15,7 +15,7 @@ const char *const g5p_binop_name[G5P_BIN_COUNT] = {"fadd", "fmul", "iadd", "imul const char *const g5p_minmax_name[G5P_MINMAX_COUNT] = {"umin", "umax", "imin", "imax"}; const char *const g5p_convert_name[G5P_CONVERT_COUNT] = {"f2i32", "f2u32", "i2f32", "u2f32", "f2i32_rte"}; const char *const g5p_unary_name[G5P_UNARY_COUNT] = {"floor", "log2", "exp2", "not", "popcount", "bitreverse"}; -const char *const g5p_tex_dim_name[G5P_TEX_DIM_COUNT] = {"2d", "2d_array", "3d", "cube"}; +const char *const g5p_tex_dim_name[G5P_TEX_DIM_COUNT] = {"2d", "2d_array", "3d", "cube", "2d_ms"}; const char *const g5p_cmp_name[G5P_CMP_COUNT] = {"flt", "fgt", "feq", "ilt", "igt", "ult", "ugt", "ieq"}; const char *const g5p_sfu_name[G5P_SFU_COUNT] = {"rcp_approx", "rsqrt_approx", "sin_core", "cos_core"}; @@ -26,7 +26,7 @@ static const uint8_t convert_code[G5P_CONVERT_COUNT] = {0x60, 0x60, 0x68, 0x68, static const uint8_t cmp_code[G5P_CMP_COUNT] = {0x59, 0x5d, 0x55, 0xbe, 0xbf, 0xb6, 0xb7, 0xbd}; static const uint8_t sfu_code[G5P_SFU_COUNT] = {0x80, 0x81, 0x82, 0x86}; static const uint8_t unary_code[G5P_UNARY_COUNT] = {0x40, 0x84, 0x85, 0xc5, 0xe8, 0xe4}; -static const uint8_t dim_code[G5P_TEX_DIM_COUNT] = {0xf5, 0xf7, 0xfb, 0xfd}; +static const uint8_t dim_code[G5P_TEX_DIM_COUNT] = {0xf5, 0xf7, 0xfb, 0xfd, 0xf9}; static const uint8_t mov_trailer[4] = {0x54, 0x01, 0x38, 0x00}; /* Split immediate map (g5p_isa.IMM_MAP): bit -> (byte, bit). */ @@ -742,6 +742,16 @@ unsigned g5p_usc_mov_shared(uint8_t *out, unsigned dst_half, unsigned src_dword, return finish(out, raw, 12); } +/* Byte fields of the address add (g5p_usc_address_add_shift()) and later encoders. */ +#define G5P_HEADER_FRAGMENT 0x01 /* the header's continuation into the destination parcel, nothing else */ +#define G5P_HEADER_SRC_B_BANK1 0x10 /* source B in register bank 1 (g9p_w0()'s W0 bit 12; vertex code) */ +#define G5P_PARCEL_CONTINUES 0x01 +#define G5P_SRC_LAST_READ 0x80 /* a register source read for the last time (in the group bit's place) */ +#define G5P_CLASS_ADDR_ADD 0x89 /* the address add's class byte (the binary class with the 16-bit result) */ +#define G5P_ADDR_ADD_ROUTING_NONE 0x00 +#define G5P_ADDR_ADD_CTL 0x06 /* set in every witnessed address add (observed role) */ +#define G5P_ADDR_ADD_SHIFT_BIT 5 /* the shift, bits 5..7 of the control byte */ + /* ---- ISA round 2: inline shared sources (g5p_usc_resources.py) ---- */ static unsigned alu_header(uint8_t *raw, unsigned cls, unsigned dst_half, unsigned bank, unsigned header, unsigned source_extension, unsigned dst_control) @@ -783,6 +793,74 @@ static unsigned add_route(uint8_t *raw, unsigned n, int route) return n; } +/* Names shared by the fragment encoders. */ +#define G5P_CLASS_TERNARY_RESULT_HALF 0x8a /* g5p_usc_literal_ternary() with result_half */ +#define G5P_TERNARY_CTL_RESULT_HALF 0x02 + +/* One byte of a packed UNORM8 colour, for the clear of 5..8 colour targets (g5p_mrt_programs.h): the f32 in + * common-store dword src_dword, converted to an unsigned normalized byte, into byte 0 or byte 1 of the 16-bit + * dst_half, bank 0. Two of them fill one half: the byte-1 form reads dst_half as B and keeps the byte that the + * byte-0 form wrote. It is the float-to-integer conversion in the three-source class with the 16-bit result, + * laid out as literal_prefix() followed by a common-store C: + * A a short literal, the conversion's target format: G5P_SMALL_UN8, the format code of + * g5p_usc_convert_small() (g5p_usc_f2u16_half() puts G5P_SMALL_U16 in the same place); + * B byte 0: unused (selector 0); byte 1: dst_half, group 1; + * C the common-store dword (shared_src(), group 1); + * opcode the float-to-integer conversion (convert_code[], as g5p_usc_f2u16_half()); + * control bit 0 the continuation into the last parcel; bit 1 the 16-bit result; bit 2 set by every + * conversion form (g5p_usc_convert2(), g5p_usc_f2u16_half(), g5p_usc_convert_small()); the byte-1 + * form adds bits 4 and 6, which were only observed together (bit 6 also selects the source's high + * byte in g5p_usc_convert_small()); + * routing the low byte of the last parcel, observed per byte; its high byte, the source control, is 0. + * The form was recovered from controlled probes; the clear was tested on the A9 at 5, 6, 7 and 8 targets. */ +#define G5P_CONVERT_CTL 0x04 +#define G5P_UNORM8_CTL_BYTE1 0x50 +#define G5P_UNORM8_ROUTING_BYTE0 0x10 +#define G5P_UNORM8_ROUTING_BYTE1 0x1c +#define G5P_SOURCE_CONTROL_NONE 0x00 + +unsigned g5p_usc_unorm8_byte(uint8_t *out, unsigned dst_half, unsigned byte, unsigned src_dword) +{ + CHECK(byte <= 1, "a half has bytes 0 and 1"); + const unsigned dst_bank = 0, c_group = 1; + const unsigned b_half = byte ? dst_half : 0, b_group = byte ? 1 : 0; + uint8_t raw[G5P_USC_MAX]; + unsigned n = literal_prefix(raw, G5P_CLASS_TERNARY_RESULT_HALF, dst_half, dst_bank, G5P_SMALL_UN8, b_half, + b_group, G5P_HEADER_FRAGMENT, 0); + if (!n) + return 0; + if (!shared_src(raw + n, src_dword, c_group)) + return 0; + n += 4; + raw[n++] = convert_code[G5P_F2U32]; + raw[n++] = G5P_TERNARY_CTL_RESULT_HALF | G5P_CONVERT_CTL | (byte ? G5P_UNORM8_CTL_BYTE1 : 0) | + G5P_PARCEL_CONTINUES; + raw[n++] = byte ? G5P_UNORM8_ROUTING_BYTE1 : G5P_UNORM8_ROUTING_BYTE0; + raw[n++] = G5P_SOURCE_CONTROL_NONE; + return finish(out, raw, n); +} + +/* A move of a common-store dword into a 16-bit half of bank 0 (the end-of-tile program's state moves, + * g5p_mrt_programs.h): alu_header() with the 16-bit move class (G5P_CLASS_MOVE_HALF, as g5p_usc_half_immediate()) + * and the fragment header, A = the common-store dword (shared_src(), group 1), then the half move's opcode and + * control (G5P_HALF_MOVE_OP, G5P_HALF_MOVE_CTL: the bytes of g5p_usc_half_immediate()'s last parcel, named by + * that observed role) and no source control. */ +#define G5P_CLASS_MOVE_HALF 0x88 +#define G5P_HALF_MOVE_OP 0x28 +#define G5P_HALF_MOVE_CTL 0x39 +unsigned g5p_usc_mov_shared_half(uint8_t *out, unsigned dst_half, unsigned src_dword) +{ + uint8_t raw[12]; + if (!alu_header(raw, G5P_CLASS_MOVE_HALF, dst_half, 0, G5P_HEADER_FRAGMENT, 0, 0) || + !shared_src(raw + 4, src_dword, 1)) + return 0; + raw[8] = G5P_HALF_MOVE_OP; + raw[9] = G5P_PARCEL_CONTINUES; + raw[10] = G5P_HALF_MOVE_CTL; + raw[11] = G5P_SOURCE_CONTROL_NONE; + return finish(out, raw, 12); +} + unsigned g5p_usc_alu_shared(uint8_t *out, enum g5p_binop op, unsigned dst_half, unsigned a_dword, unsigned b_half, unsigned bank, unsigned b_group, unsigned header, unsigned source_extension, unsigned dst_control, bool negate_b, @@ -910,6 +988,8 @@ static unsigned texture_base(uint8_t *raw, const struct g5p_tex *t) CHECK(t->class_variant <= 1, "class_variant must be an integer in [0,1]"); CHECK(t->dim < G5P_TEX_DIM_COUNT, "unmeasured texture dimension"); CHECK(!t->read || t->dim == G5P_TEX_2D, "only 2d read has been isolated"); + CHECK(t->dim != G5P_TEX_2D_MS || (!t->read && !t->class_variant), + "the 2D-multisample fetch is witnessed only as the narrow class0 sample"); raw[0] = 0x1a | t->class_variant; raw[1] = dim_code[t->dim]; raw[2] = t->dst_half | (t->read ? 0x40 : 0); @@ -925,13 +1005,53 @@ unsigned g5p_usc_sample(uint8_t *out, const struct g5p_tex *t) uint8_t raw[G5P_USC_MAX]; if (!texture_base(raw, t)) return 0; + CHECK(t->dim != G5P_TEX_2D_MS || (t->narrow && t->coord_control), + "the 2D-multisample sample is witnessed only narrow with coordinate control"); if (t->narrow) { - CHECK(!t->class_variant && t->dim == G5P_TEX_2D, "narrow sample witnessed only as 2d class0"); + CHECK(!t->class_variant && (t->dim == G5P_TEX_2D || t->dim == G5P_TEX_2D_MS), + "narrow sample witnessed only as 2d and 2d-multisample class0"); raw[0] &= ~2; } return finish(out, raw, 6); } +/* A move of a special value: the layout of g5p_usc_immediate()'s short move, with the source kind "special + * value" where the immediate sets "literal". Fields, by byte: + * 0 the single-source class of g5p_usc_move() and g5p_usc_immediate(); + * 1 the header: only the continuation into the destination parcel; + * 2 the destination half, 3 its bank (destination()); + * 4 bit 0: slot A is not a register; bits 1..6: the special value's index (where the immediate keeps its + * bits 0..5); bit 7: zero (the register form's group bit); + * 5 slot B unused (selector 0) with the continuation; + * 6 the source kind (G5P_MOV_SOURCE_*); the immediate's long-form bit 0 and literal bits stay zero; + * 7 the continuation, the immediate's literal bits zero; + * 8 the move's trailer (mov_trailer, as g5p_usc_move()). + * The A9 compiler emits this form for two indices: G5P_SPECIAL_OWN_SAMPLE, and 24 for the primitive's facing, + * which no encoder here takes. */ +#define G5P_CLASS_MOVE 0x98 +#define G5P_SRC_A_NOT_REGISTER 0x01 +#define G5P_SRC_B_UNUSED 0x00 +#define G5P_MOV_SOURCE_SPECIAL 0x10 /* observed with the two indices above; the immediate sets 0x20 */ +#define G5P_SPECIAL_OWN_SAMPLE 12 /* the sample the task shades */ + +unsigned g5p_usc_ms_own_sample(uint8_t *out, unsigned dst_half) +{ + CHECK(half_register(dst_half, "the sample operand's destination must be a 32-bit aligned half index"), + g5p_usc_error); + const unsigned dst_bank = 0; + uint8_t raw[12]; + raw[0] = G5P_CLASS_MOVE; + raw[1] = G5P_HEADER_FRAGMENT; + if (!destination(raw + 2, dst_half, dst_bank)) + return 0; + raw[4] = G5P_SRC_A_NOT_REGISTER | G5P_SPECIAL_OWN_SAMPLE << 1; + raw[5] = G5P_SRC_B_UNUSED | G5P_PARCEL_CONTINUES; + raw[6] = G5P_MOV_SOURCE_SPECIAL; + raw[7] = G5P_PARCEL_CONTINUES; + memcpy(raw + 8, mov_trailer, sizeof(mov_trailer)); + return finish(out, raw, 12); +} + unsigned g5p_usc_texture_lod(uint8_t *out, const struct g5p_tex *t) { CHECK(t->lod_bank == 1 || t->lod_bank == 3, "only LOD source banks 1 (shared) and 3 (temp) are measured"); @@ -984,7 +1104,7 @@ unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t) CHECK((unsigned)t->tuple <= G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL_COMPARE_OFFSET, "unqualified texture recipe"); CHECK(!coupled || t->lod_shared == (t->tuple == G5P_TEX_TUPLE_COUPLED_SHARED_LEVEL), "unqualified independent texture continuation choices"); - static const uint8_t dim_low[G5P_TEX_DIM_COUNT] = {0x5, 0x7, 0xb, 0xd}; + static const uint8_t dim_low[G5P_TEX_DIM_COUNT] = {0x5, 0x7, 0xb, 0xd, 0x9}; static const uint8_t lod_mode[] = {0, 0x40, 0x20, 0x60}; CHECK(half_register(t->dst_half, "dst_half requires a measured 32-bit aligned half index"), g5p_usc_error); /* A9 Metal v_images31 and v_units16_sample witness the complete 7-bit @@ -993,6 +1113,7 @@ unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t) CHECK(t->sampler_qword <= 124, "sampler_qword must be an integer in [0,124]"); CHECK(t->coord_dword <= 31 && t->lod_dword <= 31 && t->extras_half <= 63, "operand beyond r31"); CHECK(t->dim < G5P_TEX_DIM_COUNT, "unmeasured texture dimension"); + CHECK(t->dim != G5P_TEX_2D_MS, "no general-form 2D-multisample fetch is witnessed"); CHECK(t->mask == 0xf || (t->mask == 1 && t->compare), "only rgba and compare results are measured"); CHECK(!t->read || (!t->compare && !t->offset && t->lod == G5P_TEX_LOD_LEVEL && t->dim != G5P_TEX_CUBE), "read takes a level and no compare/offset (cube reads are refused by Apple)"); @@ -1037,7 +1158,7 @@ unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t) raw[9] = 0; if (coupled) { /* Complete native operand/result recipe, reconstructed against the - * paired Metal programs and exact in r1c's non-cube device rungs. + * paired Metal programs and exact in the non-cube device cases. * Individual destination/coordinate continuation bits are unknown; * keep this suffix indivisible rather than guessing mixed forms. */ raw[9] = (1u << 7) | 1u; @@ -1059,6 +1180,22 @@ unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t) return finish(out, raw, 10); } +/* A9 compute's array or cube sample with the level in a register (Apple's A9 compiler's form): the coupled + * temporary-level tuple of g5p_usc_sample_ext(). Its fetch form (read) keeps the coordinate operand live: it + * clears the coordinate's last-read bit, which every other sample form sets (texture_base()'s coordinate byte). */ +#define G5P_TEX_COORD_BYTE 5 +#define G5P_TEX_COORD_LAST_READ 0x80 +unsigned g5p_usc_sample_register_lod(uint8_t *out, const struct g5p_tex_ext *t) +{ + CHECK(t->tuple == G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL && t->lod == G5P_TEX_LOD_LEVEL && !t->lod_shared && + (t->dim == G5P_TEX_2D_ARRAY || t->dim == G5P_TEX_CUBE), + "the register-LOD form is an array or cube sample at a level"); + const unsigned n = g5p_usc_sample_ext(out, t); + if (n && t->read) + out[G5P_TEX_COORD_BYTE] &= ~G5P_TEX_COORD_LAST_READ; + return n; +} + unsigned g5p_usc_half_immediate(uint8_t *out, unsigned dst_half, uint16_t value) { CHECK(dst_half <= 63, "dst_half beyond r31"); @@ -1190,10 +1327,25 @@ unsigned g5p_usc_address_add_shift(uint8_t *out, unsigned pair_half, unsigned ba CHECK(half_register(pair_half, "pair_half requires a 64-bit aligned register pair") && !(pair_half & 2), "pair_half requires a 64-bit aligned register pair"); CHECK(half_register(index_half, "index_half requires a measured 32-bit register"), g5p_usc_error); - CHECK(base_common_dword < 32 && !(base_common_dword & 1), "pointer base must be an even common dword below 32"); + CHECK(base_common_dword < 256 && !(base_common_dword & 1), "pointer base must be an even common dword below 256"); CHECK(shift <= 4, "only shifts 0-4 are corpus-verified"); - uint8_t raw[12] = {0x89, vs ? 0x11 : 0x01, pair_half, vs ? 0x03 : 0x01, 0x81 | 4 * base_common_dword, - (kill ? 0x80 : 0) | 1 | index_half << 1, 0x00, 0x41, 0xa1, 0x01, 0x00, 0x06 | shift << 5}; + /* The layout of g5p_usc_alu_shared(): alu_header() (vertex code: destination bank 1, and B's bank bit in + * the header), A = the pointer base (shared_src(), group 1; above dword 31 its high bits are the field that + * the vertex XFB record's address add and every shared ALU operand use), B = the index with its last read, + * the integer add, then the last parcel: no routing, and the control byte with the shift. */ + uint8_t raw[12], a[4]; + if (!alu_header(raw, G5P_CLASS_ADDR_ADD, pair_half, vs, G5P_HEADER_FRAGMENT | (vs ? G5P_HEADER_SRC_B_BANK1 : 0), + 0, 0) || + !shared_src(a, base_common_dword, 1)) + return 0; + raw[4] = a[0]; + raw[5] = (index_half << 1) | (kill ? G5P_SRC_LAST_READ : 0) | G5P_PARCEL_CONTINUES; + raw[6] = a[2]; + raw[7] = a[3]; + raw[8] = binop_code[G5P_BIN_IADD]; + raw[9] = G5P_PARCEL_CONTINUES; + raw[10] = G5P_ADDR_ADD_ROUTING_NONE; + raw[11] = G5P_ADDR_ADD_CTL | shift << G5P_ADDR_ADD_SHIFT_BIT; return finish(out, raw, 12); } @@ -1353,6 +1505,50 @@ unsigned g5p_usc_discard_test(uint8_t *out, enum g5p_cmp op, unsigned dst_half, return finish(out, raw, n); } +/* The 4x discard test is a packed ternary of g5p_usc_literal_ternary()'s layout: the 16-bit-result class, + * A a long literal (literal_prefix()), B the tested value, C a short literal, then the opcode, the operation + * control and a last parcel of routing and relation. Its values: + * A -0.0f, the literal Apple's compiler compares against zero with (as the rsqrt and sign recipes); + * C the kill's sample mask, all four samples; + * opcode the float-equality compare of g5p_usc_compare_write(); + * control bit 1 the 16-bit result (as g5p_usc_binary()'s result_half flag); bit 4 observed with a short C + * literal (set in this form and in the cos-scale and bool-to-half recipes, clear in the rsqrt and + * sign recipes' long C literals; the rcp-refine recipe also sets it, with a register C); bit 0 the + * continuation into the last parcel; + * routing the low byte, the routing of the literal forms (g5p_usc_binary_literal2()); + * relation the high byte, as relation_control in g5p_usc_discard_test() and g5p_usc_compare_write(): 0, so + * with this opcode the test is A != B, as Apple's compiler emits it for (v != 0.0f) discard. The == + * relation (0x10) killed the samples a discard should keep on the device. Its continuation bit is + * clear: the parcel ends the instruction. */ +#define G5P_F32_NEG_ZERO 0x80000000u +#define G5P_MS4_ALL_SAMPLES 0xfu +#define G5P_TERNARY_CTL_C_SHORT 0x10 +#define G5P_ROUTING_LITERAL 0x70 +#define G5P_RELATION_NE 0x00 + +unsigned g5p_usc_discard_test_ms4(uint8_t *out, unsigned dst_half, struct g5p_src a) +{ + CHECK(!a.neg && !a.abs, "the 4x discard test is witnessed without source modifiers"); + const unsigned dst_bank = 0; + const int a_long = 1, c_long = 0; + uint8_t raw[G5P_USC_MAX], c[G5P_USC_MAX]; + unsigned n = literal_prefix(raw, G5P_CLASS_TERNARY_RESULT_HALF, dst_half, dst_bank, G5P_F32_NEG_ZERO, a.half, + a.group, G5P_HEADER_FRAGMENT, a_long); + if (!n) + return 0; + /* C's literal: its parcels from byte 4 on of a literal prefix (as g5p_usc_literal_ternary()). */ + unsigned m = literal_prefix(c, 0, 0, 0, G5P_MS4_ALL_SAMPLES, 0, 0, G5P_HEADER_FRAGMENT, c_long); + if (!m) + return 0; + memcpy(raw + n, c + 4, m - 4); + n += m - 4; + raw[n++] = cmp_code[G5P_FEQ]; + raw[n++] = G5P_TERNARY_CTL_RESULT_HALF | G5P_TERNARY_CTL_C_SHORT | G5P_PARCEL_CONTINUES; + raw[n++] = G5P_ROUTING_LITERAL; + raw[n++] = G5P_RELATION_NE; + return finish(out, raw, n); +} + unsigned g5p_usc_discard(uint8_t *out, int src_half) { if (src_half < 0) { @@ -2110,6 +2306,35 @@ unsigned g5p_usc_alu2_wide(uint8_t *out, unsigned op, unsigned dst, unsigned a, return put_words(out, w, 5); } +#define G5P_CLASS_BINARY 0x99 /* g5p_usc_binary()'s class byte with the 32-bit result */ +#define G5P_HEADER_SRC_A_BANK1 0x20 /* source A in register bank 1 (g9p_w0()'s W0 bit 13) */ +unsigned g5p_usc_vs_or_wide(uint8_t *out, unsigned dst_half, unsigned dst_bank, unsigned a, unsigned a_group, + unsigned b, unsigned b_group) +{ + CHECK(dst_half < 256 && !(dst_half & 1) && dst_bank <= 1 && a < 128 && b < 128, "registers r0..r127"); + /* g5p_usc_binary()'s layout for A | B with both sources' bank bits in the header; a source above r31 keeps + * r & 31 in its selector, sets A's selector bit 0 and puts r >> 5 in an extension parcel (A at bit 1, B at + * bit 9 of the parcel, as g5p_usc_alu2_wide()). */ + const bool wide = a >= 32 || b >= 32; + uint8_t raw[10]; + raw[0] = G5P_CLASS_BINARY; + raw[1] = G5P_HEADER_FRAGMENT | G5P_HEADER_SRC_A_BANK1 | G5P_HEADER_SRC_B_BANK1; + if (!destination(raw + 2, dst_half, dst_bank)) + return 0; + SEL(sa, 2 * (a & 31), a_group & 1, wide); + SEL(sb, 2 * (b & 31), b_group & 1, G5P_PARCEL_CONTINUES); + raw[4] = sa; + raw[5] = sb; + unsigned n = 6; + if (wide) { + raw[n++] = (a >> 5) << 1; + raw[n++] = (b >> 5) << 1 | G5P_PARCEL_CONTINUES; + } + raw[n++] = binop_code[G5P_BIN_OR]; + raw[n++] = 0; + return finish(out, raw, n); +} + /* Graphics storage differential host compiles, compute-gfx-storage-isa.md. * Unlike compute/vertex class10, fragment stores use the pixel side-effect * port70/71 and the observed active-fragment predicate. */ diff --git a/src/imagination/g5p/g5p_usc.h b/src/imagination/g5p/g5p_usc.h index dfa571ea8b3..8c05bb9e149 100644 --- a/src/imagination/g5p/g5p_usc.h +++ b/src/imagination/g5p/g5p_usc.h @@ -38,7 +38,10 @@ enum g5p_small_format { G5P_SMALL_FORMAT_COUNT }; enum g5p_unary { G5P_FLOOR, G5P_LOG2, G5P_EXP2, G5P_NOT, G5P_POPCOUNT, G5P_BITREVERSE, G5P_UNARY_COUNT }; -enum g5p_tex_dim { G5P_TEX_2D, G5P_TEX_2D_ARRAY, G5P_TEX_3D, G5P_TEX_CUBE, G5P_TEX_DIM_COUNT }; +/* G5P_TEX_2D_MS (code 0xf9): the 2D-multisample fetch, its sample operand in the register after the + * coordinates (Apple's 4x LOAD background and Metal texture2d_ms probes); only the narrow + * sample (g5p_usc_sample()) takes it. */ +enum g5p_tex_dim { G5P_TEX_2D, G5P_TEX_2D_ARRAY, G5P_TEX_3D, G5P_TEX_CUBE, G5P_TEX_2D_MS, G5P_TEX_DIM_COUNT }; /* ISA round 2 (isa-g5p.md "ISA round 2"): compare/write relations, SFU * cores and the literal kinds of encode_binary_literal. */ enum g5p_cmp { G5P_FLT, G5P_FGT, G5P_FEQ, G5P_ILT, G5P_IGT, G5P_ULT, G5P_UGT, G5P_IEQ, G5P_CMP_COUNT }; @@ -157,6 +160,12 @@ struct g5p_tex { unsigned length; /* 0 = minimum */ }; unsigned g5p_usc_sample(uint8_t *out, const struct g5p_tex *t); +/* The 2D-multisample fetch's sample operand "the task's own sample" (a sample-rate program reads the + * sample it shades): a 12-byte move of the special value with that index into dst_half, bank 0 (the fields + * are listed at the encoder). It is not g5p_usc_immediate()'s move: its source kind is "special value" where + * the immediate's is "literal". Witnesses: the 4x LOAD background (an iOS capture of a 4x colour load, half + * 4) and texture2d_ms::read(c, sample_id) (a Metal probe, half 12, which the fetch reads unchanged). */ +unsigned g5p_usc_ms_own_sample(uint8_t *out, unsigned dst_half); unsigned g5p_usc_texture_lod(uint8_t *out, const struct g5p_tex *t); unsigned g5p_usc_gather(uint8_t *out, const struct g5p_tex *t); @@ -193,6 +202,9 @@ struct g5p_tex_ext { enum g5p_tex_tuple tuple; bool lod_shared; /* prefix level source: common store instead of register */ }; +/* A9 compute's register-level array or cube sample and fetch: the coupled temporary-level tuple, the fetch's + * coordinate kept live (fields at the encoder). */ +unsigned g5p_usc_sample_register_lod(uint8_t *out, const struct g5p_tex_ext *t); unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t); /* 16-bit immediate into one half (class 0x88, the 32-bit move's split map; * Apple's texel-offset probes witness bits 0-9). */ @@ -217,6 +229,13 @@ unsigned g5p_usc_cube_gradient_tuple(uint8_t *out); /* f32 -> u16 (truncating) into one half: Apple's array-layer conversion * `89 01 DH 01 09 SS 20 01 60 06`. */ unsigned g5p_usc_f2u16_half(uint8_t *out, unsigned dst_half, unsigned src_half, unsigned group); +/* One byte of a packed UNORM8 colour: the f32 in common-store dword src_dword as an unsigned normalized byte in + * byte 0 or 1 of dst_half, bank 0; the byte-1 form keeps byte 0 of dst_half (the fields are listed at the + * encoder). The clear of 5..8 colour targets (g5p_mrt_programs.h). */ +/* Common-store dword src_dword (group 1) into the 16-bit dst_half of bank 0 (fields at the encoder): the + * end-of-tile program's state moves (g5p_mrt_programs.h). */ +unsigned g5p_usc_mov_shared_half(uint8_t *out, unsigned dst_half, unsigned src_dword); +unsigned g5p_usc_unorm8_byte(uint8_t *out, unsigned dst_half, unsigned byte, unsigned src_dword); /* length 0 = minimum */ unsigned g5p_usc_buffer_load(uint8_t *out, unsigned dst_half, unsigned byte_count, unsigned base_common_dword, unsigned index, unsigned length); @@ -293,6 +312,15 @@ unsigned g5p_usc_sfu_core(uint8_t *out, enum g5p_sfu op, unsigned dst_half, stru unsigned g5p_usc_square(uint8_t *out, unsigned dst_half, struct g5p_src s, struct g5p_ctl k); unsigned g5p_usc_discard_test(uint8_t *out, enum g5p_cmp op, unsigned dst_half, struct g5p_src a, struct g5p_src b, unsigned relation_control); +/* The discard test at 4 raster samples: A != 0.0 writes the kill's sample mask 15 (all four samples) into the + * destination, where the one-sample test writes 1 (our one-sample g5p_usc_discard_test() kills where A != 0.0 + * too). Apple's compiler, the same fragment program under raster_sample_count 4 (if (v != 0.0f) + * discard_fragment(), v in h0 and, read again, h4): a packed ternary with the 16-bit result, -0.0f as a long + * literal A, v as B and the mask 15 as a short literal C, the float-equality opcode and the != relation (the + * fields are listed at the encoder). At one sample the same compiler writes the mask 1 with a two-source form. + * With the == relation the form killed the samples it should keep on the device. The kill that follows is + * g5p_usc_discard() of the destination; an unconditional discard is g5p_usc_discard(-1) at 4 samples as at one. */ +unsigned g5p_usc_discard_test_ms4(uint8_t *out, unsigned dst_half, struct g5p_src a); unsigned g5p_usc_discard(uint8_t *out, int src_half); /* Fragment depth replacement (A9 depth(any) compiler witnesses). */ unsigned g5p_usc_depth_feedback(uint8_t *out, unsigned src_half); @@ -450,7 +478,7 @@ unsigned g5p_usc_address_add(uint8_t *out, unsigned dst_half, unsigned ptr_dword * `10 (0x23+8(n-1)) (0x80|data_half) (addr_half<<1|1) OO 88`. * Vertex tasks commit only the 4-byte form (xfb.md, device run 2). */ /* Host fragment store predicate and class70/71 memory side effect; - * helper-lane interpretation is inferred, pending graphics-storage rungs. */ + * helper-lane interpretation is inferred, pending graphics-storage cases. */ unsigned g5p_usc_fs_store_predicate(uint8_t *out); unsigned g5p_usc_fs_store(uint8_t *out,unsigned bytes,unsigned data_half,unsigned addr_half,unsigned slot); unsigned g5p_usc_global_store(uint8_t *out, unsigned bytes, unsigned data_half, unsigned addr_half, @@ -492,6 +520,12 @@ unsigned g5p_usc_cf_compare_imm(uint8_t *out, enum g5p_cmp op, unsigned reg, uns * r31 takes the extension word. op: the binop code. */ unsigned g5p_usc_alu2_wide(uint8_t *out, unsigned op, unsigned dst, unsigned a, unsigned a_group, unsigned b, unsigned b_group); +/* The vertex form: A | B with both sources in bank 1 (both bank bits in the header), any of r0..r127 (above r31 the extension + * word), into dst_half of bank dst_bank. Apple's A9 vertex programs read attributes above r31 in place this way + * (Apple's compiler, 44 attribute dwords: r44 <- r40 | r36 with the extension word). With a == b it copies + * a register (the G9P home move). */ +unsigned g5p_usc_vs_or_wide(uint8_t *out, unsigned dst_half, unsigned dst_bank, unsigned a, unsigned a_group, + unsigned b, unsigned b_group); /* Derivatives (Apple's A9 dfdx/dfdy, tools/mesa-a9-hypr-re f_ddx_*, f_ddy_*): * the class-98 unary form with op 44 and a modifier word, 0200 across x and diff --git a/src/imagination/g5p/g5p_vtex.h b/src/imagination/g5p/g5p_vtex.h index 31cdda806d3..25e686015f8 100644 --- a/src/imagination/g5p/g5p_vtex.h +++ b/src/imagination/g5p/g5p_vtex.h @@ -15,6 +15,6 @@ static inline bool g5p_vtex_fetch_offset_probe(void) { return true; } static inline bool g5p_vtex_gradient_probe(void) { return false; } static inline bool g5p_vtex_integer_probe(void) { return false; } static inline bool g5p_vtex_xyz_offset_probe(void) { return false; } -static inline bool g5p_vtex_size_lod_probe(void) { return false; } +static inline bool g5p_vtex_size_lod_probe(void) { return true; } static inline bool g5p_vtex_shadow_probe(void) { return false; } #endif diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index c0a425b3728..8a0d74fca9c 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -62,6 +62,9 @@ * buffer starts at CODE_INITIAL and doubles. */ #define CODE_INITIAL 16384 #define NUM_DWORDS 32 +/* Vertex input dwords: the PDS fetches up to this many into bank 1. Those above r31 are read through a move + * (g5p_usc_vs_or_wide(), Apple's vertex form for attributes above r31), as the fragment homes are. */ +#define VS_INPUT_DWORDS 64 #define F32_ONE 0x3f800000u #define LAST_END (INT_MAX - 1) /* read by the output block */ #define DISCARD_AT (INT_MAX - 2) /* read by the deferred discards, before the output */ @@ -92,6 +95,7 @@ struct tex_bind { unsigned element, selement; /* array elements of the texture / sampler binding */ unsigned image, sampler; /* common dwords */ bool needs_sampler, needs_fetch_sampler; + unsigned image_flags; /* PCO_G5P_UPLOAD_IMAGE_AS_2D / _ARRAY: how its samples read the state */ }; struct range { @@ -102,12 +106,14 @@ struct range { }; enum { BLEND_NONE, BLEND_PREMULT, BLEND_STRAIGHT_OVER, BLEND_REPLACE, BLEND_GENERAL }; +/* nir_tex_instr.backend_flags: an array sample of layer 0 drawn as 2D (layer0_as_2d()) */ +#define G5P_TEX_LAYER0_AS_2D (1u << 30) /* The general blend epilogue (g5p_blend.c, hyprland-gpu.md ยง11.2) for * settings without a measured epilogue. Premultiplied, straight-over and * replace keep their paths. By default only the settings of the device - * blend ladder (linux-a9 blend-ladder.c, release7 2026-11-02: 45 rungs, - * every pixel within 1 LSB) and of the bench ladder's three blend rungs + * blend ladder (45 cases on the A9, + * every pixel within 1 LSB) and of the bench ladder's three blend cases * take their original byte-identical shape. All other core equations use * the same tile protocol with canonical native-f16 ALU lowering. */ bool pco_g5p_blend_general(void) @@ -180,6 +186,9 @@ struct gctx { nir_def *discards[16]; /* demote_if/terminate_if conditions, emitted before the output */ unsigned ndiscards; bool discard_always; + /* the kill held for the destination-reading epilogue (fs_output(): Apple's order, the tile read + * issued before the kill): -2 none, -1 unconditional, else the condition register */ + int deferred_kill; int fragcoord; /* dword of the raw X (Y at +1), -1 when unused */ bool remat; /* register-pressure retry: constants/uniforms re-read per use */ bool count_pass; /* round 19: first codegen pass only records register reads */ @@ -233,9 +242,11 @@ struct gctx { unsigned nvout; const struct pco_g5p_vs_link *link; unsigned psize_dword; /* a point pipeline's point size: the last output dword */ + unsigned clip_dword; /* the first user clip distance (Apple's A9 vertex: they follow the point size) */ bool eager_vout; /* VS temps retry: each output written as soon as its value is final */ bool compact_vs; /* last retry: compact movable bank1 values between instructions */ bool vs_bank0; /* VS pressure retry: temps in bank 0, outputs in bank 1 (vs-two-bank-allocator.md) */ + bool vs_evict; /* last VS retry: held bank-1 values move to bank 0 to keep room for bank-1-only forms */ nir_intrinsic_instr *vend_store; /* eager: the output component written by the final vend */ unsigned vend_comp; /* common store */ @@ -244,10 +255,15 @@ struct gctx { bool cube_gradient; /* fixed native bank0 r0..r8 operand tuple */ struct { unsigned set, binding, element, lod, common; } tsize[4]; /* textureSize/QueryLevels */ unsigned nsize; - struct range ranges[8]; + struct range ranges[16]; /* 13 UBOs per stage (maxPerStageDescriptorUniformBuffers) + push constants */ unsigned nranges; - unsigned ssbo_set[2], ssbo_binding[2]; + /* Storage buffers: one pointer pair each. Up to two take Apple's witnessed + * fixed-load bases common0/common2 (the hardware-tested layout, byte-identical); + * more, or any with blend constants, sit after the stage's other common + * dwords (ssbo_high) and every access goes through addr.add_shift. */ + unsigned ssbo_set[PCO_G5P_MAX_SSBOS], ssbo_binding[PCO_G5P_MAX_SSBOS], ssbo_common[PCO_G5P_MAX_SSBOS]; unsigned nssbo; + bool ssbo_high; /* register-indexed UBO loads: one pointer (UBO VA + constant base) each. * The pairs sit at even common dwords below 32 (addr.add_shift's base * field), so at most 16; plan_common() refuses a pair past dword 30 and @@ -392,6 +408,8 @@ static void vs_bank1_only(struct gctx *c, const char *what, int r0, int r1, int refuse(c, "%s with a register in vertex bank 0 (no witnessed mixed-bank form)", what); } +static unsigned vs_bank1_index(struct gctx *c, unsigned reg); + /* ---- registers (dwords) ---- */ /* A vertex bank-0 register: the first free one, else -1. */ static int alloc_bank0(struct gctx *c) @@ -759,6 +777,7 @@ static bool alu_supported(struct gctx *c, nir_alu_instr *a) /* Register-indexed UBO loads (defined with the common-store plan). */ static bool dyn_offset_alu(nir_alu_instr *a); static bool dyn_ubo_load(nir_intrinsic_instr *in); +static bool eval_offset(nir_def *d, unsigned comp, uint32_t *v); static bool dyn_ubo_offset(nir_intrinsic_instr *in, struct ref *idx, unsigned *shift, uint32_t *base); /* Vertex iadd that only feeds UBO/push offsets folds into the upload plan. */ @@ -1175,11 +1194,15 @@ static void scan_cf(struct gctx *c, struct exec_list *list, struct unsupported * case nir_intrinsic_load_ubo: ok = true; break; + case nir_intrinsic_get_ssbo_size: + ok = in->def.bit_size == 32 && in->def.num_components == 1; + break; case nir_intrinsic_store_ssbo: ok = in->src[0].ssa->bit_size==32 && in->num_components==1 && nir_intrinsic_write_mask(in)==1; break; case nir_intrinsic_load_ssbo: - ok = !c->vs; + /* 4- or 16-byte loads (g5p_nir_prepare splits the rest) */ + ok = in->def.bit_size == 32 && (in->def.num_components == 1 || in->def.num_components == 4); break; case nir_intrinsic_load_vertex_id: case nir_intrinsic_load_instance_id: @@ -1399,6 +1422,19 @@ static bool smooth_centre(nir_intrinsic_instr *in) interp == PCO_G5P_INTERP_SMOOTH && !centroid; } +/* A vertex output store's location, its constant slot offset included (a + * compact gl_ClipDistance[4..7] store is CLIP_DIST0 at offset 1); other + * offsets are refused by collect_vs_outputs(). */ +static unsigned +out_location(nir_intrinsic_instr *in) +{ + unsigned loc = nir_intrinsic_io_semantics(in).location; + nir_src *off = nir_get_io_offset_src(in); + if ((loc == VARYING_SLOT_CLIP_DIST0 || loc == VARYING_SLOT_CLIP_DIST1) && off && nir_src_is_const(*off)) + loc += nir_src_as_uint(*off); + return loc; +} + /* First output dword of a vertex output location: 0 for the position, -1 * when the location is not written (its fragment program does not read it), * -2 for a location without an output slot. */ @@ -1408,6 +1444,8 @@ static int vs_base(struct gctx *c, unsigned loc) return 0; if (loc == VARYING_SLOT_PSIZ && c->link && c->link->point_size) return c->psize_dword; + if ((loc == VARYING_SLOT_CLIP_DIST0 || loc == VARYING_SLOT_CLIP_DIST1) && c->p->clip_count) + return c->clip_dword + 4 * (loc - VARYING_SLOT_CLIP_DIST0); if (loc < VARYING_SLOT_VAR0 || loc > VARYING_SLOT_VAR31) return -2; unsigned d = c->p->vout_dword[loc - VARYING_SLOT_VAR0]; @@ -1468,6 +1506,21 @@ static void plan_vs_layout(struct gctx *c) * alpha for the size) */ if (c->link && c->link->point_size) c->psize_dword = c->p->outputs++; + /* User clip distances: the vertex's last dwords, after the point size, + * one per declared distance (Apple's applegpu_g5p layout, LD_MD +164; the + * TA's user planes 0..n-1, TA_OUTPUT_SEL plane0..7). Disabled GL planes + * are the program's business (Zink's nir_lower_clip_disable, Apple's + * clipDistanceEnableMask in the program key). */ + const unsigned clips = c->nir->info.clip_distance_array_size; + if (c->nir->info.cull_distance_array_size) + refuse(c, "cull distances (A9's TA user planes are clip planes; Apple's A9 driver sets no cull plane)"); + if (clips && !(c->link && c->link->clip_distance)) + refuse(c, "user clip distances"); + if (clips > 8) + refuse(c, "%u clip distances (the TA has 8 user planes)", clips); + c->clip_dword = c->p->outputs; + c->p->clip_count = clips; + c->p->outputs += clips; /* The A9 TA vertex holds 128 dwords, the position included * (varyings-capacity-1: 124 varying components exact, 128 wrong as a * whole with no GPU event); refuse past it by name, never draw it. */ @@ -1480,7 +1533,7 @@ static void plan_vs_layout(struct gctx *c) if (ins->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *in = nir_instr_as_intrinsic(ins); - if (in->intrinsic != nir_intrinsic_store_output || vs_base(c, nir_intrinsic_io_semantics(in).location) < 0) + if (in->intrinsic != nir_intrinsic_store_output || vs_base(c, out_location(in)) < 0) continue; last = in; last_comp = util_last_bit(nir_intrinsic_write_mask(in)) - 1; @@ -1528,8 +1581,8 @@ static void plan_varyings(struct gctx *c) mask[n] |= BITFIELD64_MASK(in->def.num_components) << nir_intrinsic_component(in); } if (util_bitcount(read) > PCO_G5P_MAX_FS_VARYINGS) - refuse(c, "%u varying locations read (%u vec4: the reported 64 components)", util_bitcount(read), - PCO_G5P_MAX_FS_VARYINGS); + refuse(c, "%u varying locations read (%u vec4: the reported %u components)", util_bitcount(read), + PCO_G5P_MAX_FS_VARYINGS, 4 * PCO_G5P_MAX_FS_VARYINGS); bool smooth = false; u_foreach_bit(n, read) smooth |= interp[n] == PCO_G5P_INTERP_SMOOTH; @@ -1723,6 +1776,12 @@ static void liveness(struct gctx *c) mark(c, ix, idx); continue; } + if (in->intrinsic == nir_intrinsic_load_ssbo) { + uint32_t off; + if (!eval_offset(in->src[1].ssa, 0, &off)) + mark(c, resolve_value(c, in->src[1].ssa, 0), idx); + continue; + } if (in->intrinsic == nir_intrinsic_store_global) { unsigned b, k; struct ref off; @@ -1757,7 +1816,7 @@ static void liveness(struct gctx *c) continue; if (in->instr.block->cf_node.parent->type != nir_cf_node_function) refuse(c, "store_output inside control flow"); - unsigned loc = nir_intrinsic_io_semantics(in).location; + unsigned loc = c->vs ? out_location(in) : nir_intrinsic_io_semantics(in).location; unsigned comp = nir_intrinsic_component(in); u_foreach_bit(k, nir_intrinsic_write_mask(in)) { if (c->vs) { @@ -2000,6 +2059,8 @@ static void tex_bindings(struct gctx *c, nir_tex_instr *t, struct tex_bind *b) sv->data.binding, element, selement}; b->needs_sampler = t->op != nir_texop_txf; b->needs_fetch_sampler = t->op == nir_texop_txf; + b->image_flags = (t->backend_flags & G5P_TEX_LAYER0_AS_2D) ? PCO_G5P_UPLOAD_IMAGE_AS_2D : + t->is_array ? PCO_G5P_UPLOAD_IMAGE_ARRAY : 0; } static struct tex_bind *tex_for(struct gctx *c, nir_tex_instr *t) @@ -2008,7 +2069,8 @@ static struct tex_bind *tex_for(struct gctx *c, nir_tex_instr *t) tex_bindings(c, t, &b); for (unsigned i = 0; i < c->ntex; i++) if (c->tex[i].set == b.set && c->tex[i].binding == b.binding && c->tex[i].sset == b.sset && - c->tex[i].sbinding == b.sbinding && c->tex[i].element == b.element && c->tex[i].selement == b.selement) + c->tex[i].sbinding == b.sbinding && c->tex[i].element == b.element && c->tex[i].selement == b.selement && + c->tex[i].image_flags == b.image_flags) { c->tex[i].needs_sampler |= b.needs_sampler; c->tex[i].needs_fetch_sampler |= b.needs_fetch_sampler; return &c->tex[i]; } @@ -2106,6 +2168,13 @@ static void plan_common(struct gctx *c) c->uptr[c->nuptr].element = element; c->uptr[c->nuptr++].base = base; } + } else if (in->intrinsic == nir_intrinsic_get_ssbo_size) { + /* the bound size, a CPU-written dword (PCO_G5P_SSBO_SIZE_OFFSET) */ + if (!block_binding_element(in->src[0].ssa, &set, &binding, &element)) + refuse(c, "get_ssbo_size: block index is not a constant descriptor binding"); + struct range *r = range_for(c, PCO_G5P_UPLOAD_SSBO_ADDR, set, binding, element); + r->lo = PCO_G5P_SSBO_SIZE_OFFSET; + r->hi = PCO_G5P_SSBO_SIZE_OFFSET + 4; } else if (in->intrinsic == nir_intrinsic_load_ubo) { if (!block_binding_element(in->src[0].ssa, &set, &binding, &element)) refuse(c, "load_ubo: block index is not a constant descriptor binding"); @@ -2121,25 +2190,33 @@ static void plan_common(struct gctx *c) for (unsigned i = 0; i < c->nssbo; i++) found |= c->ssbo_set[i] == set && c->ssbo_binding[i] == binding; if (!found) { - if (c->nssbo == 2) - refuse(c, "load_ssbo: only pointer bases common0/common2 are witnessed (two buffers)"); + if (c->nssbo == PCO_G5P_MAX_SSBOS) + refuse(c, "more than %u storage buffers", PCO_G5P_MAX_SSBOS); c->ssbo_set[c->nssbo] = set; c->ssbo_binding[c->nssbo++] = binding; } } } unsigned next = 0; - if (!c->vs && c->blend == BLEND_GENERAL && c->binfo.uses_constants) { - /* Apple's layout: the blend colour in common 0..3, everything else after it */ - if (c->nssbo) - refuse(c, "blend constants with storage buffers (both want common dwords 0..3)"); + bool blend_constants = !c->vs && c->blend == BLEND_GENERAL && c->binfo.uses_constants; + /* Apple's layout: the blend colour in common 0..3, everything else after + * it. Storage buffers keep common0/common2 only when they are at most two + * and the blend colour does not want those dwords; otherwise their + * pointers follow the stage's other common dwords (below). */ + c->ssbo_high = c->nssbo > 2 || (c->nssbo && blend_constants); + if (blend_constants) { add_upload(c, PCO_G5P_UPLOAD_BLEND_CONSTANTS, 0, 0, 0, 0, 4, G5P_BLEND_CONST_BASE); next = 4; } - for (unsigned i = 0; i < c->nssbo; i++) + for (unsigned i = 0; !c->ssbo_high && i < c->nssbo; i++) { + c->ssbo_common[i] = 2 * i; add_upload(c, PCO_G5P_UPLOAD_SSBO_ADDR, c->ssbo_set[i], c->ssbo_binding[i], 0, 0, 2, 2 * i); - if (c->nssbo) + } + if (c->nssbo && !c->ssbo_high) next = MAX2(next, 4); + /* High storage pointers: an even dword each after the data, so the + * plan reserves their dwords (plus one for alignment) at its end. */ + const unsigned ssbo_reserve = c->ssbo_high ? 2 * c->nssbo + 1 : 0; bool qualified = c->vs && g5p_vtex_qualified(); /* Preserve the flown <=4-texture layout. Larger fragment programs use * Apple's f16 capacity layout: all eight-dword images, then samplers. @@ -2167,7 +2244,7 @@ static void plan_common(struct gctx *c) } if (!g5p_stage_common_plan(next, c->ntex, sampler_count + fetch_sampler, compact, c->nuptr, c->nsize, ubos, - ubo_dwords, push_dwords, &common_plan)) + ubo_dwords, push_dwords + ssbo_reserve, &common_plan)) refuse(c, "common layout cannot fit pointers below 32 and data below 256 dwords"); compact = common_plan.compact; if (common_plan.low_pointers) { @@ -2202,7 +2279,7 @@ static void plan_common(struct gctx *c) next = compact ? c->fetch_sampler_common + 2 * (qualified && fetch_sampler) : t->image + 10; if (t->image / 2 > (compact ? 120 : 28) || t->sampler / 2 > (compact ? 124 : 32)) refuse(c, "texture descriptors beyond the bounded qualified selectors"); - add_upload(c, PCO_G5P_UPLOAD_IMAGE, t->set, t->binding, t->element, 0, 8, t->image); + add_upload(c, PCO_G5P_UPLOAD_IMAGE, t->set, t->binding, t->element, t->image_flags, 8, t->image); if (!qualified || t->needs_sampler) add_upload(c, PCO_G5P_UPLOAD_SAMPLER, t->sset, t->sbinding, t->selement, 32, 2, t->sampler); } @@ -2266,7 +2343,16 @@ static void plan_common(struct gctx *c) (r->hi - r->lo) / 4, r->common); } if (planned) - assert(next == common_plan.end); + assert(next + ssbo_reserve == common_plan.end); + /* addr.add_shift reads a pointer pair at any even common dword below 256 + * (the shared-operand field's high bits, g5p_usc_address_add_shift). */ + for (unsigned i = 0; c->ssbo_high && i < c->nssbo; i++) { + c->ssbo_common[i] = next = align(next, 2); + next += 2; + if (next > limit) + refuse(c, "storage-buffer pointers exceed %u common-store dwords", limit); + add_upload(c, PCO_G5P_UPLOAD_SSBO_ADDR, c->ssbo_set[i], c->ssbo_binding[i], 0, 0, 2, c->ssbo_common[i]); + } /* Transform feedback block: pointer pairs at even dwords (address add). */ if (xfb) { c->xfb_common = align(next, 2); @@ -2362,7 +2448,11 @@ static unsigned reg_of(struct gctx *c, struct ref r, const char *what) case VAL_HOME: { /* parked above r31: back into a register for this instruction */ unsigned reg = alloc_reg(c, -1); - put(c, raw, g5p_usc_alu2_wide(raw, G5P_OP_OR, reg, v->reg, 0, v->reg, 0), "move from a home above r31"); + if (c->vs) /* a vertex input fetched above r31 (setup_inputs()): v->reg is its bank-1 register */ + put(c, raw, g5p_usc_vs_or_wide(raw, H(c, reg), vbank(c, reg), v->reg, 0, v->reg, 0), + "move a vertex input from above r31"); + else + put(c, raw, g5p_usc_alu2_wide(raw, G5P_OP_OR, reg, v->reg, 0, v->reg, 0), "move from a home above r31"); c->scratch |= BITFIELD64_BIT(reg); return reg; } @@ -2401,6 +2491,18 @@ static unsigned scratch_imm(struct gctx *c, uint32_t value) return reg; } +/* A scratch immediate in vertex bank 1, also when the program allocates its + * temps in bank 0 (alloc_bank1_run()). */ +static unsigned scratch_imm_bank1(struct gctx *c, uint32_t value) +{ + bool bank0 = c->vs_bank0, tex_bank0 = c->tex_bank0; + c->vs_bank0 = c->tex_bank0 = false; + unsigned reg = scratch_imm(c, value); + c->vs_bank0 = bank0; + c->tex_bank0 = tex_bank0; + return reg; +} + static unsigned one_reg(struct gctx *c) { if (c->remat || c->cf_depth > 0) @@ -3431,7 +3533,7 @@ static void emit_isub(struct gctx *c, nir_alu_instr *a) * tools/mesa-a9-vtx-re/probes.metal, byte-equal to these encoders). The * fragment form (bank 0, header 1) in a vertex program reads and writes * other registers: x >> 2 then read again gave x on hardware (vertex-input's - * gl_VertexIndex rungs). Dynamic amounts have no vertex probe. */ + * gl_VertexIndex cases). Dynamic amounts have no vertex probe. */ static void emit_shift(struct gctx *c, nir_alu_instr *a) { struct ref x = alu_src(c, a, 0), amt = alu_src(c, a, 1); @@ -4160,13 +4262,24 @@ bool pco_g5p_fuse_ffma(void) return true; } -/* The index is clamp_dyn_ubo's umin(i, last), a constant bound inside the block. */ +/* The index is clamp_dyn_ubo's umin(i, last), a constant bound inside the + * block; or nir_opt_algebraic removed that umin because the index is + * already proven to be at most `last` (x & 3 into uvec4[4]). The second form + * is accepted only with a proven bound inside the largest UBO + * (maxUniformBufferRange 64 KiB = 4096 elements of 16 bytes), never a + * wild 32-bit index. */ static bool clamped_index(struct ref ix) { nir_alu_instr *a = alu_of(ix.def); - if (!a || a->op != nir_op_umin) - return false; - return nir_src_is_const(a->src[0].src) || nir_src_is_const(a->src[1].src); + if (a && a->op == nir_op_umin && + (nir_src_is_const(a->src[0].src) || nir_src_is_const(a->src[1].src))) + return true; + nir_instr *ins = nir_def_instr(ix.def); + nir_shader *shader = nir_cf_node_get_function(&ins->block->cf_node)->function->shader; + struct hash_table *ht = _mesa_pointer_hash_table_create(NULL); + uint32_t bound = nir_unsigned_upper_bound(shader, ht, nir_get_scalar(ix.def, ix.comp)); + _mesa_hash_table_destroy(ht, NULL); + return bound < 4096; } static void emit_dyn_ubo(struct gctx *c, nir_intrinsic_instr *in) @@ -4177,11 +4290,14 @@ static void emit_dyn_ubo(struct gctx *c, nir_intrinsic_instr *in) uint32_t base; block_binding_element(in->src[0].ssa, &set, &binding, &element); dyn_ubo_offset(in, &ix, &shift, &base); - unsigned bytes = 4 * n; /* Witnessed: 16-byte loads (Apple's v_cexpr, vec4 elements at shift 4) and * 4-byte loads (G9P's, at any byte offset); the shift field is - * corpus-verified 0-4. vec2/vec3 (8-byte, 12-byte) loads are not. */ - if (in->def.bit_size != 32 || !((n == 4 && shift == 4) || (n == 1 && shift != 1)) || base % bytes) + * corpus-verified 0-4. vec2/vec3 (8-byte, 12-byte) loads are not: a + * vec2/vec3 at the start of a 16-byte element (nir_opt_shrink_vectors of + * a vec4 read) loads the whole element and drops the unused registers. */ + const unsigned loaded = (n == 2 || n == 3) && shift == 4 && !(base % 16) ? 4 : n; + unsigned bytes = 4 * loaded; + if (in->def.bit_size != 32 || !((loaded == 4 && shift == 4) || (n == 1 && shift != 1)) || base % bytes) refuse(c, "load_ubo: register-indexed %u x %u-bit at shift %u, base %u (witnessed: vec4 at shift 4, " "scalar at shift 0/2-4, element-aligned base)", n, in->def.bit_size, shift, base); /* Apple's form for an unsigned index is this one: umin, then addr.add_shift @@ -4204,8 +4320,10 @@ static void emit_dyn_ubo(struct gctx *c, nir_intrinsic_instr *in) c->uptr[i].element == element && c->uptr[i].base == base) common = c->uptr[i].common; unsigned index = reg_of(c, ix, "register-indexed UBO load index"); + /* a bank-0 index is copied into bank 1 first (the witnessed mixed-bank move), as a storage + * buffer's offset is (vs_bank1_index()): nfsu2's vertex pulling indexes its uniforms so */ if (c->vs) - vs_bank1_only(c, "vertex register-indexed UBO load", index, -1, -1, -1); + index = vs_bank1_index(c, index); bool kill = dies(c, ix); /* Registers are read at issue: like Apple's v_cexpr (index r0 -> pair * r0:r1 -> data r0..r3), the pair may take a dying index's register and @@ -4221,18 +4339,22 @@ static void emit_dyn_ubo(struct gctx *c, nir_intrinsic_instr *in) c->max_dword = MAX2(c->max_dword, pair + 2); put(c, raw, g5p_usc_address_add_shift(raw, 2 * pair, common, 2 * index, shift, kill, c->vs), "addr.add_shift"); unsigned dst = 0; - for (dst = 0; dst + n <= NUM_DWORDS; dst += n) - if (!((c->busy | c->reserved | c->tex_coord_pending | c->tex_pending) & BITFIELD64_RANGE(dst, n))) + for (dst = 0; dst + loaded <= NUM_DWORDS; dst += loaded) + if (!((c->busy | c->reserved | c->tex_coord_pending | c->tex_pending) & BITFIELD64_RANGE(dst, loaded))) break; - if (dst + n > NUM_DWORDS) - refuse(c, "register pressure: load_ubo needs %u aligned dword(s)", n); - c->busy |= BITFIELD64_RANGE(dst, n); - c->max_dword = MAX2(c->max_dword, dst + n); + if (dst + loaded > NUM_DWORDS) + refuse(c, "register pressure: load_ubo needs %u aligned dword(s)", loaded); + c->busy |= BITFIELD64_RANGE(dst, loaded); + c->max_dword = MAX2(c->max_dword, dst + loaded); + if (loaded != n) /* the new whole-element form; hardware-tested forms keep their bytes */ + c->max_written = MAX2(c->max_written, dst + loaded); if (c->vs) put(c, raw, g5p_usc_vertex_address_load(raw, 2 * dst, bytes, 2 * pair), "vertex address load"); else put(c, raw, g5p_usc_address_load(raw, 2 * dst, bytes, 2 * pair, 0), "fragment address load"); put(c, raw, g5p_usc_wait(raw, 0), "wait"); + /* the load has completed (wait): the element's unused tail is free */ + c->busy &= ~BITFIELD64_RANGE(dst + n, loaded - n); for (unsigned k = 0; k < n; k++) { struct val *v = V(c, &in->def, k); v->state = VAL_REG; @@ -4253,7 +4375,10 @@ static void emit_memory_ubo(struct gctx *c, nir_intrinsic_instr *in, if (in->def.bit_size != 32 || !n || n > 4) refuse(c, "memory UBO load: only 1..4 aligned 32-bit components"); - const unsigned dst = alloc_run(c, n, "memory UBO load"); + /* A vertex load writes bank 1, the witnessed form, also when the program + * allocates its temps in bank 0 (control flow, or the bank-0 pressure + * retry), as a vertex storage-buffer load does (emit_address_ssbo_load). */ + const unsigned dst = c->vs ? alloc_bank1_run(c, n, "memory UBO load") : alloc_run(c, n, "memory UBO load"); if (c->vs) vs_bank1_only(c, "vertex memory UBO load", dst, -1, -1, -1); const unsigned first = off - r->lo; @@ -4264,7 +4389,10 @@ static void emit_memory_ubo(struct gctx *c, nir_intrinsic_instr *in, for (unsigned i = 0; i < loads; i++) { const unsigned bytes = vector ? 16 : 4; const unsigned at = first + (vector ? 0 : 4 * i); - const unsigned index = scratch_imm(c, at); + /* the vertex addr.add_shift reads its offset from bank 1 only (vs_bank1_index()) */ + const unsigned index = c->vs ? scratch_imm_bank1(c, at) : scratch_imm(c, at); + if (c->vs) + vs_bank1_only(c, "vertex constant UBO address offset", index, -1, -1, -1); unsigned pair; for (pair = 0; pair + 2 <= NUM_DWORDS; pair += 2) @@ -4343,7 +4471,7 @@ static void stage_ssbo_store(struct gctx *c,nir_intrinsic_instr *in) nir_intrinsic_offset_shift(in)!=0 || nir_intrinsic_align_mul(in)<4 || nir_intrinsic_align_offset(in)%4) refuse(c,"store_ssbo: aligned scalar32-bit byte offsets only"); unsigned set,binding,base=0;block_binding(in->src[1].ssa,&set,&binding); - for(unsigned i=0;inssbo;i++)if(c->ssbo_set[i]==set && c->ssbo_binding[i]==binding)base=2*i; + for(unsigned i=0;inssbo;i++)if(c->ssbo_set[i]==set && c->ssbo_binding[i]==binding)base=c->ssbo_common[i]; struct ref offset=resolve_value(c,in->src[2].ssa,0),value=resolve_value(c,in->src[0].ssa,0); unsigned index=reg_of(c,offset,"store_ssbo byte offset"),data=reg_of(c,value,"store_ssbo value"); if(c->vs)vs_bank1_only(c,"vertex SSBO store",index,data,-1,-1); @@ -4365,6 +4493,89 @@ static void stage_ssbo_store(struct gctx *c,nir_intrinsic_instr *in) c->p->evidence|=c->vs?PCO_G5P_EV_VS_STORE:PCO_G5P_EV_FS_STORE; } +/* A vertex address operand from bank 0 is copied into bank 1 (the only + * witnessed vertex addr.add_shift form reads bank 1). */ +static unsigned vs_bank1_index(struct gctx *c, unsigned reg) +{ + if (!c->vs || vbank(c, reg)) + return reg; + uint8_t raw[G5P_USC_MAX]; + bool bank0 = c->vs_bank0; + c->vs_bank0 = false; + unsigned b1 = alloc_reg(c, -1); + c->vs_bank0 = bank0; + put(c, raw, g5p_usc_move(raw, H(c, b1), H(c, reg), 1, 0, vhdr(c, 0x21, reg, -1, -1)), + "bank0 address offset into bank1"); + c->scratch |= BITFIELD64_BIT(b1); + vs_bank1_only(c, "vertex storage-buffer address offset", b1, -1, -1, -1); + return b1; +} + +/* A storage-buffer load through its pointer pair: pair = pointer + byte + * offset (addr.add_shift, shift 0), then the addressed 4/16-byte load that + * Apple's register-indexed constant-buffer programs use (emit_dyn_ubo). + * The offset is the load's own register, or a scratch immediate for a + * constant offset that the fixed form cannot encode. g5p_nir_prepare() + * leaves only scalar loads and 16-byte-aligned vec4 loads. */ +static void emit_address_ssbo_load(struct gctx *c, nir_intrinsic_instr *in, unsigned base) +{ + uint8_t raw[G5P_USC_MAX]; + const unsigned n = in->def.num_components, bytes = 4 * n; + uint32_t off; + bool constant = eval_offset(in->src[1].ssa, 0, &off); + if (constant && off % 4) + refuse(c, "load_ssbo: unaligned constant offset %u", off); + bool kill; + unsigned index; + struct ref ix = {0}; + if (constant) { + index = scratch_imm(c, off); + kill = true; + } else { + if (n == 4 && nir_intrinsic_align(in) < 16) + refuse(c, "load_ssbo: 16-byte load without 16-byte alignment"); + ix = resolve_value(c, in->src[1].ssa, 0); + index = reg_of(c, ix, "load_ssbo byte offset"); + kill = dies(c, ix); + } + unsigned raw_index = index; + index = vs_bank1_index(c, index); + if (index != raw_index) + kill = true; /* the bank-1 copy is scratch */ + if (!constant && kill && index == raw_index) + drop(c, V(c, ix.def, ix.comp)); + unsigned pair; + for (pair = 0; pair + 2 <= NUM_DWORDS; pair += 2) + if (!((c->busy | c->reserved | c->tex_coord_pending | c->tex_pending) & BITFIELD64_RANGE(pair, 2))) + break; + if (pair + 2 > NUM_DWORDS) + refuse(c, "load_ssbo: register pressure for an address pair"); + c->busy |= BITFIELD64_RANGE(pair, 2); + c->max_dword = MAX2(c->max_dword, pair + 2); + c->max_written = MAX2(c->max_written, pair + 2); + put(c, raw, g5p_usc_address_add_shift(raw, 2 * pair, base, H(c, index), 0, kill, c->vs), + "storage-buffer address"); + /* a vertex load writes bank 1 (the witnessed form), also when the program allocates in + * bank 0: its results are ordinary bank-1 registers, as a vertex sample's are */ + const unsigned dst = c->vs ? alloc_bank1_run(c, n, "load_ssbo") : alloc_run(c, n, "load_ssbo"); + if (c->vs) { + vs_bank1_only(c, "vertex storage-buffer load", dst, -1, -1, -1); + put(c, raw, g5p_usc_vertex_address_load(raw, 2 * dst, bytes, 2 * pair), "vertex storage-buffer load"); + } else { + put(c, raw, g5p_usc_address_load(raw, 2 * dst, bytes, 2 * pair, 0), "fragment storage-buffer load"); + } + put(c, raw, g5p_usc_wait(raw, 0), "wait"); + c->busy &= ~BITFIELD64_RANGE(pair, 2); + free_scratch(c); + for (unsigned k = 0; k < n; k++) { + struct val *v = V(c, &in->def, k); + v->state = VAL_REG; + v->reg = dst + k; + v->held = true; + } + c->p->evidence |= PCO_G5P_EV_BUFFER_LOAD; +} + static void emit_intrinsic(struct gctx *c, nir_intrinsic_instr *in) { uint8_t raw[G5P_USC_MAX]; @@ -4450,17 +4661,33 @@ static void emit_intrinsic(struct gctx *c, nir_intrinsic_instr *in) } break; } + case nir_intrinsic_get_ssbo_size: { + unsigned set, binding, element; + if (!block_binding_element(in->src[0].ssa, &set, &binding, &element)) + refuse(c, "get_ssbo_size: block index is not a constant descriptor binding"); + struct range *r = range_for(c, PCO_G5P_UPLOAD_SSBO_ADDR, set, binding, element); + struct val *v = V(c, &in->def, 0); + v->state = VAL_SHARED; + v->imm = r->common; + break; + } case nir_intrinsic_load_ssbo: { unsigned set, binding; block_binding(in->src[0].ssa, &set, &binding); unsigned base = 0; for (unsigned i = 0; i < c->nssbo; i++) if (c->ssbo_set[i] == set && c->ssbo_binding[i] == binding) - base = 2 * i; - unsigned off = load_offset(c, in, 1, 0); + base = c->ssbo_common[i]; + uint32_t off = 0; unsigned n = in->def.num_components, bytes = n == 1 ? 4 : n == 4 ? 16 : 0; - if (!bytes || off % bytes || off / bytes > 255) - refuse(c, "load_ssbo: only 4/16-byte element loads at a constant element index <= 255 are witnessed"); + if (!bytes || in->def.bit_size != 32) + refuse(c, "load_ssbo: only 4/16-byte 32-bit loads"); + bool fixed = eval_offset(in->src[1].ssa, 0, &off) && !c->vs && base <= 2 && + !(off % bytes) && off / bytes <= 255; + if (!fixed) { + emit_address_ssbo_load(c, in, base); + break; + } unsigned dst = alloc_run(c, n, "load_ssbo"); put(c, raw, g5p_usc_buffer_load(raw, 2 * dst, bytes, base, off / bytes, 0), "fixed buffer load"); put(c, raw, g5p_usc_wait(raw, 0), "wait"); @@ -4910,6 +5137,35 @@ static void emit_vertex_qualified_texture(struct gctx *c, nir_tex_instr *t) c->p->evidence |= PCO_G5P_EV_TEX_TYPES; } +/* The 4x LOAD background's sample (key ms_own_sample): Apple's loader form. The coordinates in a + * three-register run, the own-sample operand in its third register (the fetch reads it implicitly, so it is + * part of the run), then the narrow sample of dimension G5P_TEX_2D_MS with coordinate control. Only the narrow form + * is witnessed: anything else is refused. */ +static void emit_tex_ms_own_sample(struct gctx *c, nir_tex_instr *t, struct tex_bind *b, bool narrow) +{ + uint8_t raw[G5P_USC_MAX]; + if (!narrow || varying_coord_slot(c, t) >= 0 || c->mrt.targets != 1) { + refuse(c, "the 4x LOAD sample: only Apple's narrow form with computed coordinates is witnessed"); + return; + } + tex_flush(c); + nir_def *cd = nir_get_tex_src(t, nir_tex_src_coord); + struct ref x = resolve_value(c, cd, 0), y = resolve_value(c, cd, 1); + unsigned xr = reg_of(c, x, "texture coordinate"), yr = reg_of(c, y, "texture coordinate"); + const unsigned coord = alloc_run(c, 3, "multisample coordinates and sample operand"); + put(c, raw, g5p_usc_move(raw, 2 * coord, 2 * xr, 0, vgrp(c, x), 1), "coordinate x"); + put(c, raw, g5p_usc_move(raw, 2 * coord + 2, 2 * yr, 0, vgrp(c, y), 1), "coordinate y"); + put(c, raw, g5p_usc_ms_own_sample(raw, 2 * coord + 4), "the task's own sample"); + c->scratch |= BITFIELD64_RANGE(coord, 3); + struct g5p_tex s = {.image_qword = b->image / 2, .sampler_qword = b->sampler / 2, .coord_dword = coord, + .coord_control = true, .dim = G5P_TEX_2D_MS, .narrow = true, .dst_half = 0}; + put(c, raw, g5p_usc_sample(raw, &s), "texture sample (2D multisample)"); + c->tex_coord_pending |= BITFIELD64_RANGE(coord, 3); + c->tex_batch++; + tex_flush(c); + c->p->evidence |= PCO_G5P_EV_TEX_CAPTURE; +} + static void emit_tex(struct gctx *c, nir_tex_instr *t) { if (c->tex_batch == 4 || c->vs || !plain_tex(t) || size_query(t) || c->narrow_tex == t) @@ -4931,6 +5187,10 @@ static void emit_tex(struct gctx *c, nir_tex_instr *t) int slot = varying_coord_slot(c, t); bool narrow = c->narrow_tex == t; unsigned coord; + if (c->key && c->key->ms_own_sample) { + emit_tex_ms_own_sample(c, t, b, narrow); + return; + } if (slot >= 0) { tex_flush(c); coord = narrow ? 0 : alloc_run(c, 2, "texture coordinate"); @@ -5005,6 +5265,14 @@ static void vs_converts(struct gctx *c, struct vs_convert *cv, unsigned n) c->alloc_out = true; /* converted attributes stay in bank 1, where they are fetched */ qsort(cv, n, sizeof(*cv), vs_convert_order); for (unsigned i = 0; i < n; i++) { + if (cv[i].reg >= NUM_DWORDS) { + /* a dword fetched above r31 (sorted last, after the low converts freed their registers): moved into a + * register first (Apple's vertex OR form, g5p_usc_vs_or_wide()), its converts then read it there */ + const unsigned hi = cv[i].reg, lo = alloc_reg(c, -1); + put(c, raw, g5p_usc_vs_or_wide(raw, H(c, lo), vbank(c, lo), hi, 0, hi, 0), "move a vertex input from above r31"); + for (unsigned k = i; k < n && cv[k].reg == hi; k++) + cv[k].reg = lo; + } const unsigned half = 2 * cv[i].reg + (cv[i].byte >> 1); bool last_half = true, last_reg = true; for (unsigned k = i + 1; k < n; k++) { @@ -5046,8 +5314,8 @@ static void setup_inputs(struct gctx *c) !nir_src_is_const(in->src[0]) || nir_src_as_uint(in->src[0]) || in->def.bit_size != 32) refuse(c, "load_input: only direct 32-bit generic attributes"); unsigned l = loc - VERT_ATTRIB_GENERIC0, first = nir_intrinsic_component(in); - if (4 * l + 4 > NUM_DWORDS) - refuse(c, "vertex input location %u: its fetch lands beyond r%u", l, NUM_DWORDS - 1); + if (4 * l + 4 > VS_INPUT_DWORDS) + refuse(c, "vertex input location %u: its fetch lands beyond r%u", l, VS_INPUT_DWORDS - 1); c->p->input_mask |= 1u << l; for (unsigned k = 0; k < in->def.num_components; k++) { unsigned comp = first + k; @@ -5069,7 +5337,9 @@ static void setup_inputs(struct gctx *c) if (fmt >= PCO_G5P_VSF_COUNT || in->def.num_components != 1 || (fmt >= PCO_G5P_VSF_U16 && (byte & 1))) refuse(c, "vertex input location %u: converted load %#x", l, base); - raw_regs |= BITFIELD64_BIT(4 * l + comp); + /* a fetched dword above r31: vs_converts() moves it down first (not in the low-register masks) */ + if (4 * l + comp < NUM_DWORDS) + raw_regs |= BITFIELD64_BIT(4 * l + comp); if (v->uses) { if (ncv == ARRAY_SIZE(cv)) refuse(c, "more than %u converted vertex input components", (unsigned)ARRAY_SIZE(cv)); @@ -5078,6 +5348,13 @@ static void setup_inputs(struct gctx *c) continue; } v->input = true; + if (4 * l + comp >= NUM_DWORDS) { + /* above r31: read through a move at each use */ + v->state = VAL_HOME; + v->reg = 4 * l + comp; + c->p->evidence |= PCO_G5P_EV_VS_ALU; + continue; + } if (v->uses) hold(c, v, 4 * l + comp); } @@ -5111,12 +5388,17 @@ static void setup_inputs(struct gctx *c) for (unsigned i = 0; i < 2; i++) { if (!ids[i]) continue; - if (next >= NUM_DWORDS) + if (next >= VS_INPUT_DWORDS) refuse(c, "%s index: no register after the last attribute", i ? "instance" : "vertex"); struct val *v = V(c, ids[i], 0); - v->state = VAL_REG; v->input = true; - hold(c, v, next); + if (next >= NUM_DWORDS) { /* after attributes above r31: read through a move, as they are */ + v->state = VAL_HOME; + v->reg = next; + } else { + v->state = VAL_REG; + hold(c, v, next); + } c->p->id_mask |= 1u << i; c->p->id_reg[i] = next++; c->p->evidence |= PCO_G5P_EV_VS_IDS; @@ -5244,9 +5526,43 @@ static bool out_const(struct gctx *c, unsigned k, uint16_t *h) return true; } +static bool compact_output(const struct pco_g5p_fs_key *key) +{ + return key->color_output == PCO_G5P_COLOR_PACKED_BGRA8 || + key->color_output == PCO_G5P_COLOR_PACKED_RGBA8; +} + +/* Semantic UNORM8 conversion, the same saturate/round-even rule as + * nir_format_float_to_unorm(), lowered through existing G5P ALU encoders. + * Packing occurs at each output store, so eight targets keep eight dwords + * rather than 32 live float components. */ +static bool lower_compact_store(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + const struct pco_g5p_fs_key *key = data; + if (in->intrinsic != nir_intrinsic_store_output) + return false; + if (nir_intrinsic_component(in) || in->src[0].ssa->num_components != 4 || nir_intrinsic_write_mask(in) != 15) + return false; /* refused in fs_output: no partial packed output contract */ + b->cursor = nir_before_instr(&in->instr); + nir_def *rgba = in->src[0].ssa; + nir_def *packed = nir_imm_int(b, 0); + for (unsigned c = 0; c < 4; c++) { + const unsigned channel = key->color_output == PCO_G5P_COLOR_PACKED_BGRA8 && !(c & 1) ? 2 - c : c; + nir_def *v = nir_fsat(b, nir_channel(b, rgba, channel)); + v = nir_f2u32(b, nir_fround_even(b, nir_fmul_imm(b, v, 255.0f))); + packed = nir_ior(b, packed, nir_ishl_imm(b, v, 8 * c)); + } + nir_src_rewrite(&in->src[0], packed); + nir_intrinsic_set_write_mask(in, 1); + in->num_components = 1; + return true; +} + /* Output-plan decisions that need use counts. */ static void plan_fs_output(struct gctx *c) { + if (compact_output(c->key)) + return; /* g5p_mrt_out_plan_make(): one word per target, width 8 */ if (c->key->color_output != PCO_G5P_COLOR_F16) { /* four raw words per target: 4, or 8 for two targets (emitpix field 0, Apple's 4-target width) */ if (c->mrt.targets > 1) { /* one emitpix writes both targets: both need their stores */ @@ -5345,30 +5661,81 @@ static void fs_prologue(struct gctx *c) /* Discards run after the body (no derivative work follows them): a test * of the 0.0/1.0 condition against 0.0 feeding `04 61`, or `04 68`. */ -static void fs_discards(struct gctx *c) +static void fs_discards(struct gctx *c, bool defer) { tex_flush(c); uint8_t raw[G5P_USC_MAX]; c->p->discard_tokens = c->ndiscards + c->discard_always; + c->deferred_kill = -2; for (unsigned i = 0; i < c->ndiscards; i++) { struct cond k = cond_of(c, c->discards[i], 0); unsigned b = k.zero_b ? reg_of(c, k.a, "discard condition") : select_into(c, k, NULL, K_ONE, NULL, K_ZERO); - unsigned t = alloc_reg(c, -1); + /* held past the output moves (halves 0..7: the colour, then the epilogue's tile read at + * half 4): a register from dword 4 up */ + int t = -1; + /* above the colour halves (dwords 0..1), the tile read (2..3) and dual-source outputs (halves 8..11) */ + for (unsigned r = c->has_secondary ? 6 : 4; defer && t < 0 && r < 32; r++) + if (!((c->busy | c->reserved | c->tex_coord_pending | c->tex_pending) & BITFIELD64_BIT(r))) + t = alloc_reg(c, r); + if (t < 0) + t = alloc_reg(c, -1); + if (defer && t < (c->has_secondary ? 6 : 4)) + refuse(c, "no free register above the outputs for the held discard condition"); + if (c->key && c->key->ms4_kill && c->ndiscards == 1 && !c->discard_always) { + /* 4 samples, one conditional discard: b != 0.0 writes the kill's sample mask 15 (Apple's 4x form, the + * same relation as the one-sample test below). + * Several discards keep the one-sample test; their draws are refused at 4x (pvr_arch_g5p_gfx.c). */ + unsigned gb = k.zero_b ? vgrp(c, k.a) : 1; + put(c, raw, g5p_usc_discard_test_ms4(raw, 2 * t, S(2 * b, gb)), "discard test (4 samples)"); + } else { unsigned z = zero_reg(c); unsigned gb = k.zero_b ? vgrp(c, k.a) : 1, gz = kgrp(c, K_ZERO); put(c, raw, g5p_usc_discard_test(raw, G5P_FEQ, 2 * t, S(2 * b, gb), S(2 * z, gz), 0), "discard test"); - put(c, raw, g5p_usc_discard(raw, 2 * t), "discard"); - c->busy &= ~BITFIELD64_BIT(t); + } + if (defer) { + c->deferred_kill = t; /* stays busy until fs_kill() */ + } else { + put(c, raw, g5p_usc_discard(raw, 2 * t), "discard"); + c->busy &= ~BITFIELD64_BIT(t); + } free_scratch(c); c->p->evidence |= PCO_G5P_EV_DISCARD; } if (c->discard_always) { - put(c, raw, g5p_usc_discard(raw, -1), "discard"); + if (defer) + c->deferred_kill = -1; + else + put(c, raw, g5p_usc_discard(raw, -1), "discard"); c->p->evidence |= PCO_G5P_EV_DISCARD; } } +/* A held kill (fs_discards(defer)), in Apple's place: after the epilogue's tile read is issued, + * before its wait. A9 AGXMetalA9 compiles every destination-reading discarding program that way + * (three Metal probes of discarding programs with a blended or RGB-masked colour output: the tile read, + * then the kill, the wait for the tile read later), PUNCH_THROUGH and FAST_PUNCH_THROUGH alike; ours issued + * the kill first and read the tile after it. @epi: the epilogue's bytes; with no leading tile read + * the kill goes first, the old order. Used by masked and general-blend programs. */ +static void fs_put_epilogue(struct gctx *c, const uint8_t *epi, unsigned n, const char *what) +{ + uint8_t raw[G5P_USC_MAX]; + if (c->deferred_kill == -2) { + put(c, epi, n, what); + return; + } + const unsigned read = n >= 4 && epi[0] == 0x1c && epi[3] == 0x28 ? 4 + : n >= 6 && epi[0] == 0x1c && epi[3] == 0x29 ? 6 : 0; + if (read) + put(c, epi, read, "tile read (before the kill: Apple's order)"); + put(c, raw, g5p_usc_discard(raw, c->deferred_kill < 0 ? -1 : 2 * c->deferred_kill), "discard"); + if (c->deferred_kill >= 0) + c->busy &= ~BITFIELD64_BIT(c->deferred_kill); + c->deferred_kill = -2; + if (n > read) + put(c, epi + read, n - read, what); +} + static void fs_output(struct gctx *c) { tex_flush(c); @@ -5382,8 +5749,43 @@ static void fs_output(struct gctx *c) put(c, raw, g5p_usc_depth_feedback(raw, 2 * reg), "fragment depth feedback"); c->p->depth_feedback = true; } - fs_discards(c); + /* a discarding program whose epilogue reads the tile: the kill waits for that read (fs_put_epilogue()). + * A general-blend epilogue always takes Apple's order (a blending Metal probe: test, tile read, + * kill, blend), the form its default PUNCH_THROUGH runs in; the premultiplied production form keeps its + * bytes. */ + const bool defer = (c->key->mask_upass || c->blend == BLEND_GENERAL) && + c->ndiscards + c->discard_always == 1 && + c->key->color_output == PCO_G5P_COLOR_F16 && !compact_output(c->key) && + !c->key->mrt_blend_state && any_color(c) && + (c->blend == 1 || c->blend == 2 || c->blend == BLEND_GENERAL); + fs_discards(c, defer); c->now = LAST_END; + if (compact_output(c->key)) { + /* target t's packed dword: output word t (c->out[4t], lower_compact_store()) */ + struct pvr_g5p_dgl_source sources[8] = {0}; + for (unsigned t = 0; t < c->mrt.targets; t++) { + if (c->slot_written[t] != 1) + refuse(c, "compact MRT requires a full four-channel store to each target"); + struct ref r = c->out[4 * t]; + sources[t].immediate = is_const(c, r); + if (sources[t].immediate) { + sources[t].value = V(c, r.def, r.comp)->imm; + } else { + sources[t].value = reg_of(c, r, "compact MRT output"); + sources[t].group = vgrp(c, r); + for (unsigned j = t + 1; j < c->mrt.targets; j++) + if (c->out[4 * j].def == r.def && c->out[4 * j].comp == r.comp) + sources[t].group = 0; + } + } + uint8_t code[10 * G5P_USC_MAX]; + size_t size = pvr_g5p_dgl_output_words(code, sizeof(code), sources, c->mrt.words, 0xf); + if (!size) + refuse(c, "compact MRT output source cannot be encoded"); + put(c, code, size, "compact MRT packed output and emitpix"); + free_scratch(c); + return; + } if (c->key->color_output != PCO_G5P_COLOR_F16) { /* target t's channels: output words 4t..4t+3 (c->out[] is indexed 4 * slot + component) */ const unsigned words = c->mrt.words; @@ -5481,7 +5883,7 @@ static void fs_output(struct gctx *c) unsigned n = g5p_blend_epilogue(buf, sizeof(buf)); if (!n) refuse(c, "blend epilogue encoder rejected its fields (%s)", g5p_usc_error ? g5p_usc_error : "?"); - put(c, buf, n, "premultiplied blend epilogue"); + fs_put_epilogue(c, buf, n, "premultiplied blend epilogue"); c->max_dword = MAX2(c->max_dword, 7); /* destination halves 4..7, 1-alpha in half 12 */ c->p->evidence |= PCO_G5P_EV_BLEND; } else if (c->blend == BLEND_GENERAL) { @@ -5498,13 +5900,15 @@ static void fs_output(struct gctx *c) if (c->ndiscards + c->discard_always > 1) refuse(c, "discard in a program with a general blend epilogue needs one merged kill"); /* A kill before a general epilogue runs PUNCH_THROUGH with a general - * blend, base program or punch variant alike. r39-6.1 hard-hung the - * iPad in Aquarium with that form under depth writes (not bisected - * yet), so it is off: such a program is refused at pipeline - * creation, the r39-6 rule. */ - if (c->ndiscards || c->discard_always) - refuse(c, "discard in a program with a general blend epilogue " - "(general-blend punch-through is off)"); + * blend, base program or punch variant alike, in Apple's order: the + * tile read, then the kill, then the blend (fs_output()'s defer; + * Apple's straight-alpha punch-through capture runs its draw in the PUNCH_THROUGH pass). + * Aquarium hard-hung the iPad with the old + * order (the kill before the tile read), which is no longer used. */ + /* Opaque mode retains the masked form for non-discarding programs only. */ + if ((c->ndiscards || c->discard_always) && c->deferred_kill == -2 && c->key->mask_upass != 1) + refuse(c, "discard in a program with a general blend epilogue: no held kill for Apple's order " + "(several discards, MRT, compact or integer output)"); uint8_t buf[512]; const char *err; struct g5p_blend_key bk = general_key(c->key); @@ -5513,11 +5917,13 @@ static void fs_output(struct gctx *c) if (err) refuse(c, "general blend epilogue: %s", err); if (n) { - put(c, buf, n, "general blend epilogue"); + fs_put_epilogue(c, buf, n, "general blend epilogue"); c->max_dword = MAX2(c->max_dword, c->binfo.max_half / 2); } c->p->evidence |= PCO_G5P_EV_BLEND; } + if (c->deferred_kill != -2) /* no epilogue took the held kill (a general epilogue of no bytes) */ + fs_put_epilogue(c, NULL, 0, "no epilogue"); /* One emitpix carries the total word count over all targets (2/4/8). */ put(c, raw, g5p_usc_emit_mrt(raw, c->mrt.words, 0), "emit"); put(c, raw, g5p_usc_special(raw, true), "end"); @@ -5531,13 +5937,18 @@ static void collect_vs_outputs(struct gctx *c) nir_intrinsic_instr *in = nir_instr_as_intrinsic(ins); if (in->intrinsic != nir_intrinsic_store_output) continue; - unsigned loc = nir_intrinsic_io_semantics(in).location; + const unsigned sem_loc = nir_intrinsic_io_semantics(in).location; + const unsigned loc = out_location(in); int base = vs_base(c, loc); if (base == -2) - refuse(c, "store_output to %s (only position and generic varyings have output slots)", + refuse(c, "store_output to %s (only position, generic varyings and clip distances have output slots)", gl_varying_slot_name_for_stage(loc, MESA_SHADER_VERTEX)); - if (!nir_src_is_const(in->src[1]) || nir_src_as_uint(in->src[1])) + if (!nir_src_is_const(in->src[1]) || (loc == sem_loc && nir_src_as_uint(in->src[1]))) refuse(c, "indirect store_output"); + if (base >= 0 && (loc == VARYING_SLOT_CLIP_DIST0 || loc == VARYING_SLOT_CLIP_DIST1) && + 4 * (loc - VARYING_SLOT_CLIP_DIST0) + nir_intrinsic_component(in) + + util_last_bit(nir_intrinsic_write_mask(in)) > c->p->clip_count) + refuse(c, "a clip distance past the %u declared", c->p->clip_count); if (base == -1) continue; /* not read by the fragment program */ u_foreach_bit(k, nir_intrinsic_write_mask(in)) { @@ -5950,9 +6361,134 @@ static bool hoist_branch_derivatives(nir_shader *nir) return nir_progress(progress, impl, nir_metadata_control_flow); } +/* An array sample at a constant layer that clamps to 0 reads the view's + * base layer: Vulkan's layer is clamp(RNE(z), 0, layers - 1). It is drawn + * as a 2D sample of that layer (the upload's PCO_G5P_UPLOAD_IMAGE_AS_2D), + * which also samples a single-layer array view of a linear image: the A9 has + * no array state for a linear image, and its array sample needs one even + * for one layer (pvr_arch_g5p_tex.c). PPSSPP binds every texture and + * framebuffer as a 2D_ARRAY view and samples layer 0.0. */ +static bool layer0_as_2d(nir_builder *b, nir_tex_instr *t, void *data) +{ + if (!t->is_array || t->sampler_dim != GLSL_SAMPLER_DIM_2D || t->coord_components != 3 || + (t->op != nir_texop_tex && t->op != nir_texop_txb && t->op != nir_texop_txl && + t->op != nir_texop_txd && t->op != nir_texop_txf && t->op != nir_texop_tg4)) + return false; + const int ci = nir_tex_instr_src_index(t, nir_tex_src_coord); + if (ci < 0) + return false; + nir_scalar z = nir_scalar_chase_movs(nir_get_scalar(t->src[ci].src.ssa, 2)); + if (!nir_scalar_is_const(z)) + return false; + if (t->op == nir_texop_txf ? nir_scalar_as_int(z) != 0 : !(nir_scalar_as_float(z) <= 0.5)) + return false; + b->cursor = nir_before_instr(&t->instr); + nir_def *c = t->src[ci].src.ssa; + nir_src_rewrite(&t->src[ci].src, nir_channels(b, c, 0x3)); + t->coord_components = 2; + t->is_array = false; + t->backend_flags |= G5P_TEX_LAYER0_AS_2D; + return true; +} + +/* A 4-sample texture (texture_multisampled_2d, sampler2DMS) is sampled as the 2W x 2H 2D + * texture of its samples (pvr_arch_g5p_tex.c): sample s of texel p is texel + * 2p + (s >> 1, s & 1); its size is half that texture's; it has 4 samples. */ +static bool ms_as_2d(nir_builder *b, nir_tex_instr *t, void *data) +{ + if (t->sampler_dim != GLSL_SAMPLER_DIM_MS || t->is_array) + return false; + b->cursor = nir_after_instr(&t->instr); + if (t->op == nir_texop_texture_samples) { + nir_def_replace(&t->def, nir_imm_int(b, 4)); + return true; + } + if (t->op == nir_texop_txs) { + t->sampler_dim = GLSL_SAMPLER_DIM_2D; + nir_def *half = nir_ushr_imm(b, &t->def, 1); + nir_def_rewrite_uses_after(&t->def, half); + return true; + } + if (t->op != nir_texop_txf_ms) + return false; + const int ci = nir_tex_instr_src_index(t, nir_tex_src_coord); + const int si = nir_tex_instr_src_index(t, nir_tex_src_ms_index); + if (ci < 0 || si < 0) + return false; + b->cursor = nir_before_instr(&t->instr); + nir_def *c = t->src[ci].src.ssa, *s = t->src[si].src.ssa; + nir_def *x = nir_iadd(b, nir_ishl_imm(b, nir_channel(b, c, 0), 1), nir_ushr_imm(b, s, 1)); + nir_def *y = nir_iadd(b, nir_ishl_imm(b, nir_channel(b, c, 1), 1), nir_iand_imm(b, s, 1)); + nir_tex_instr_remove_src(t, si); + nir_src_rewrite(&t->src[nir_tex_instr_src_index(t, nir_tex_src_coord)].src, nir_vec2(b, x, y)); + if (nir_tex_instr_src_index(t, nir_tex_src_lod) < 0) + nir_tex_instr_add_src(t, nir_tex_src_lod, nir_imm_int(b, 0)); + t->op = nir_texop_txf; + t->sampler_dim = GLSL_SAMPLER_DIM_2D; + return true; +} + +/* A half float's bits (in a 32-bit value) as a 32-bit float, in integer code (G9P's 26b4dcbc83d ("pvr/g9p: packing built-ins")) */ +static nir_def * +half_to_f32(nir_builder *b, nir_def *h) +{ + nir_def *magnitude = nir_iand_imm(b, h, 0x7fff); + nir_def *shifted = nir_ishl_imm(b, magnitude, 13); + /* exponent 31: inf and NaN; exponent 0: zero and subnormals */ + nir_def *bits = nir_bcsel(b, nir_uge_imm(b, magnitude, 0x7c00), nir_ior_imm(b, shifted, 0x7f800000), + nir_iadd_imm(b, shifted, 0x38000000)); + + bits = nir_bcsel(b, nir_ult_imm(b, magnitude, 0x400), + nir_fmul_imm(b, nir_u2f32(b, magnitude), 1.0f / (1 << 24)), bits); + return nir_ior(b, bits, nir_ishl_imm(b, nir_iand_imm(b, h, 0x8000), 16)); +} + +/* A 32-bit float as a half float's bits, rounded to nearest, in integer code */ +static nir_def * +f32_to_half(nir_builder *b, nir_def *f) +{ + nir_def *sign = nir_iand_imm(b, nir_ushr_imm(b, f, 16), 0x8000); + nir_def *abs = nir_iand_imm(b, f, 0x7fffffff); + nir_def *normal = nir_ushr_imm(b, nir_iadd_imm(b, abs, 0x1000 - 0x38000000), 13); + nir_def *shift = nir_isub(b, nir_imm_int(b, 126), nir_ushr_imm(b, abs, 23)); + nir_def *mantissa = nir_ior_imm(b, nir_iand_imm(b, abs, 0x7fffff), 0x800000); + nir_def *rounded = nir_iadd(b, mantissa, nir_ishl(b, nir_imm_int(b, 1), nir_iadd_imm(b, shift, -1))); + nir_def *subnormal = nir_ushr(b, rounded, shift); + nir_def *v = nir_bcsel(b, nir_uge_imm(b, abs, 0x38800000), normal, + nir_bcsel(b, nir_uge_imm(b, abs, 0x33000000), subnormal, nir_imm_int(b, 0))); + + v = nir_bcsel(b, nir_uge_imm(b, abs, 0x47800000), nir_imm_int(b, 0x7c00), v); + v = nir_bcsel(b, nir_ult(b, nir_imm_int(b, 0x7f800000), abs), nir_imm_int(b, 0x7e00), v); + return nir_ior(b, sign, nir_iand_imm(b, v, 0x7fff)); +} + +/* packHalf2x16(), unpackHalf2x16(): no 16-bit values in G5P programs (g9p 26b4dcbc83d) */ +static bool +lower_half_packing(nir_builder *b, nir_instr *instr, void *data) +{ + if (instr->type != nir_instr_type_alu) + return false; + nir_alu_instr *alu = nir_instr_as_alu(instr); + + if (alu->op != nir_op_pack_half_2x16 && alu->op != nir_op_unpack_half_2x16) + return false; + b->cursor = nir_before_instr(instr); + nir_def *src = nir_ssa_for_alu_src(b, alu, 0); + + if (alu->op == nir_op_pack_half_2x16) { + nir_def_replace(&alu->def, nir_ior(b, f32_to_half(b, nir_channel(b, src, 0)), + nir_ishl_imm(b, f32_to_half(b, nir_channel(b, src, 1)), 16))); + } else { + nir_def_replace(&alu->def, nir_vec2(b, half_to_f32(b, nir_iand_imm(b, src, 0xffff)), + half_to_f32(b, nir_ushr_imm(b, src, 16)))); + } + return true; +} + static bool lower(nir_shader *nir, bool preserve_cf) { bool progress; + NIR_PASS(_, nir, nir_shader_instructions_pass, lower_half_packing, nir_metadata_control_flow, NULL); /* Projective sampling (GLSL textureProj/texture2DProj, Mesa's fixed-function * fragment program; Zink passes 2D projectors on as OpImageSampleProj*): * Apple's GPU has no projected sample, Metal divides in the program, so @@ -5971,8 +6507,13 @@ static bool lower(nir_shader *nir, bool preserve_cf) * This adds no TPU instruction or shadow-sampling recipe. */ .lower_txs_lod = nir->info.stage == MESA_SHADER_FRAGMENT || (nir->info.stage == MESA_SHADER_VERTEX && g5p_vtex_size_lod_probe()), + /* Explicit gradients in a vertex program as an explicit level, GL's isotropic formula (G9P's 2e5d5bc3e2f ("pvr/g9p: explicit gradients as explicit levels")): the qualified vertex txl instead of the unqualified gradient recipe. Fragment + * gradients keep their device-qualified recipe (R24). */ + .lower_txd = nir->info.stage == MESA_SHADER_VERTEX && !g5p_vtex_gradient_probe(), }; NIR_PASS(_, nir, nir_lower_tex, &txp); + NIR_PASS(_, nir, nir_shader_tex_pass, ms_as_2d, nir_metadata_control_flow, NULL); + NIR_PASS(_, nir, nir_shader_tex_pass, layer0_as_2d, nir_metadata_control_flow, NULL); pvr_g5p_tbo_lower(nir); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_push_const, nir_address_format_32bit_offset); NIR_PASS(_, nir, nir_lower_flrp, 32, false); @@ -6974,6 +7515,88 @@ static void compact_vertex_bank1(struct gctx *c) } } +/* Last vertex pressure retry (vs_evict). The forms that must use bank 1 (the + * addressed UBO/SSBO loads: data, offset and address pair; the vertex math + * bridge; an output's bank-1 copy) found no room there when the fetched + * attributes fill r0..r31 until their first reads, which come after a UBO + * load (a translated D3D9-style skinning shader: eight vec4 attributes, every + * one masked by a UBO word). Apple's vertex programs hold their temps in + * bank 0 and read each attribute once into one (Apple's offline compiler on + * this shape, 12 float4 attributes and a 4 KiB constant buffer, does the + * same). Before each instruction, keep one + * aligned run of eight bank-1 registers free, which holds every bank-1 demand + * of one instruction in its first-fit order (a memory UBO load's data, offset + * and aligned pair; a register-indexed load's offset, pair and aligned vec4), + * by moving held values into bank 0, the run needing the fewest moves, with the + * class-98 move whose destination is bank 0 and source A bank 1, the move + * cf_operand() and the vertex texture park already emit. */ +static bool evict_movable(struct gctx *c, unsigned r) +{ + if ((c->reserved | c->scratch | c->tex_pending | c->tex_coord_pending) & BITFIELD64_BIT(r)) + return false; + if (!(c->busy & BITFIELD64_BIT(r))) + return true; + bool owned = false; + for (unsigned i = 0; i < c->nvals; i++) + if (c->vals[i].held && c->vals[i].reg == r) { + if (c->vals[i].last <= c->now) + return false; /* an operand of this instruction stays where it is */ + owned = true; + } + return owned; /* a busy register with no value (a pair, a persistent constant) stays */ +} + +static int evict_last(struct gctx *c, unsigned r) +{ + int last = -1; + for (unsigned i = 0; i < c->nvals; i++) + if (c->vals[i].held && c->vals[i].reg == r) + last = MAX2(last, c->vals[i].last); + return last; +} + +/* The aligned run of n bank-1 registers that is cheapest to free (fewest + * moves, then the latest nearest read), or -1. */ +static int evict_run(struct gctx *c, unsigned n) +{ + int best = -1, best_cost = INT_MAX, best_near = -1; + for (unsigned r = 0; r + n <= NUM_DWORDS; r += n) { + int cost = 0, near = INT_MAX; + bool ok = true; + for (unsigned k = r; k < r + n && ok; k++) { + ok = evict_movable(c, k); + if (ok && (c->busy & BITFIELD64_BIT(k))) { + cost++; + near = MIN2(near, evict_last(c, k)); + } + } + if (ok && (cost < best_cost || (cost == best_cost && near > best_near))) { + best = r; + best_cost = cost; + best_near = near; + } + } + return best; +} + +static void evict_vertex_bank1(struct gctx *c) +{ + int run = evict_run(c, 8); + if (run < 0) + return; + uint8_t raw[G5P_USC_MAX]; + u_foreach_bit64(src, BITFIELD64_RANGE(run, 8) & c->busy) { + int dst = alloc_bank0(c); + if (dst < 0) + return; /* bank 0 is full too: the instruction refuses as before */ + put(c, raw, g5p_usc_move(raw, H(c, dst), H(c, src), 0, 1, 0x21), "attribute bank value into bank 0"); + for (unsigned i = 0; i < c->nvals; i++) + if (c->vals[i].held && c->vals[i].reg == src) + c->vals[i].reg = dst; + c->busy &= ~BITFIELD64_BIT(src); + } +} + static void emit_block(struct gctx *c, nir_block *block) { nir_foreach_instr(ins, block) { @@ -6984,6 +7607,8 @@ static void emit_block(struct gctx *c, nir_block *block) decl_alloc(c, c->now); if (c->vs && c->vs_bank0 && c->compact_vs && !c->cf) compact_vertex_bank1(c); + if (c->vs && c->vs_bank0 && c->vs_evict && !c->cf) + evict_vertex_bank1(c); switch (ins->type) { case nir_instr_type_alu: emit_alu(c, nir_instr_as_alu(ins)); @@ -7625,7 +8250,7 @@ bool pco_g5p_compile_general_key(nir_shader *nir, bool half_output, const uint8_ static bool compile_general_link(nir_shader *nir, bool half_output, const uint8_t *attrib_components, const struct pco_g5p_fs_key *key, const struct pco_g5p_vs_link *link, - struct pco_g5p_program *out, bool preserve_cf) + struct pco_g5p_program *out, bool preserve_cf, bool evict) { static const struct pco_g5p_fs_key default_key = {.write_mask = 0xf}; memset(out, 0, sizeof(*out)); @@ -7659,7 +8284,15 @@ static bool compile_general_link(nir_shader *nir, bool half_output, const uint8_ mr = g5p_mrt_check_key(&c->mrt, c->key->blend, c->key->write_mask); if (mr != G5P_MRT_OK) refuse(c, "%s", g5p_mrt_refusal_str(mr)); - if (c->key->color_output != PCO_G5P_COLOR_F16) { + if (compact_output(c->key) != (c->mrt.targets > 4)) + refuse(c, "five to eight colour targets take the compact UNORM8 output, fewer do not"); + if (compact_output(c->key)) { + if (c->key->blend || c->key->write_mask != 15 || c->key->mrt_blend_state) + refuse(c, "five to eight colour targets: blending and partial write masks are not built " + "(compact MRT is full-write and unblended)"); + NIR_PASS(_, nir, nir_shader_intrinsics_pass, lower_compact_store, nir_metadata_control_flow, + (void *)c->key); + } else if (c->key->color_output != PCO_G5P_COLOR_F16) { if (c->key->mrt_blend_state) refuse(c, "raw32 MRT blend or masks are not qualified"); if (c->key->color_output != PCO_G5P_COLOR_UINT32 && @@ -7740,6 +8373,16 @@ static bool compile_general_link(nir_shader *nir, bool half_output, const uint8_ refuse(c, "out of memory"); memset(c->home, -1, c->impl->ssa_alloc * 4); schedule_texture_batches(c); + if (evict) { + /* pco_g5p_compile_general_link()'s last vertex retry: the program's own order, temps in bank 0, attributes + * moved out of bank 1 where a bank-1-only form needs room (evict_vertex_bank1()) */ + /* as the other between-instruction mover (compact_vs): straight-line programs without stores */ + if (!c->vs || c->cf || impl_has_side_effects(c->impl)) + refuse(c, "register pressure: bank-1 eviction is a straight-line vertex retry without stores"); + c->vs_bank0 = c->remat = c->eager_vout = c->vs_evict = true; + ok = two_pass(c); + goto done; + } ok = two_pass(c); if (!ok && !c->vs && strstr(out->refusal, "register pressure")) { /* Retry: constants and uniforms re-read per use (round 19). */ @@ -7872,11 +8515,22 @@ bool pco_g5p_compile_general_link(nir_shader *nir, bool half_output, const uint8 * together. Retry the original fragment control flow only on pressure; * the existing scalar register/mask-stack lowering handles the branches. * Keep all phis scalar in that retry, including texture-result phis. */ - nir_shader *original = nir->info.stage == MESA_SHADER_FRAGMENT && cf_left(nir) + /* A vertex program every pressure retry refused is compiled once more + * from its own order, before the scheduling retries reordered it, with the + * attributes moved out of bank 1 where a bank-1-only form needs room + * (evict_vertex_bank1()). Programs that compile today never get there. */ + nir_shader *original = (nir->info.stage == MESA_SHADER_FRAGMENT && cf_left(nir)) || + nir->info.stage == MESA_SHADER_VERTEX ? nir_shader_clone(NULL, nir) : NULL; - bool ok = compile_general_link(nir, half_output, attrib_components, key, link, out, false); - if (!ok && original && strstr(out->refusal, "register pressure")) - ok = compile_general_link(original, half_output, attrib_components, key, link, out, true); + bool ok = compile_general_link(nir, half_output, attrib_components, key, link, out, false, false); + if (!ok && original && strstr(out->refusal, "register pressure")) { + char first[sizeof(out->refusal)]; + memcpy(first, out->refusal, sizeof(first)); + const bool vs = nir->info.stage == MESA_SHADER_VERTEX; + ok = compile_general_link(original, half_output, attrib_components, key, link, out, !vs, vs); + if (!ok && vs) /* the ladder's own reason stays the one reported */ + memcpy(out->refusal, first, sizeof(first)); + } ralloc_free(original); return ok; } diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index 6515bcefa7b..284ab420ab6 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -16,6 +16,8 @@ * ICD's job arena is what bounds a draw's programs, see pco_g5p_general.c). */ #define PCO_G5P_PROGRAM_MAX (1u << 20) #define PCO_G5P_MAX_UPLOADS 40 +/* storage buffers per stage (maxPerStageDescriptorStorageBuffers) */ +#define PCO_G5P_MAX_SSBOS 16 #define PCO_G5P_MAX_ATTRIBS 16 #define PCO_G5P_COMMON_DWORDS 64 /* Every vertex program that has flown needs <= 12 temps; 16 and 20 wedged @@ -80,6 +82,10 @@ enum pco_g5p_color_output { PCO_G5P_COLOR_F16 = 0, PCO_G5P_COLOR_UINT32, PCO_G5P_COLOR_SINT32, + /* 5..8 UNORM8 targets: each target's RGBA packed into one dword (BGRA8: + * bytes B,G,R,A), stored through a physical R32UI PBE */ + PCO_G5P_COLOR_PACKED_BGRA8, + PCO_G5P_COLOR_PACKED_RGBA8, }; struct pco_g5p_rt_blend { @@ -114,7 +120,22 @@ struct pco_g5p_fs_key { bool packed_sint; /* saturate signed channels, then preserve low bits */ bool packed_scalar8; /* R8: one scalar source; never pack unused G/B/A */ uint8_t packed_uint_bits; /* 8/10/16: packed output, scalar8 is one word; full mask only */ + /* Apple's masked form (enabled): no blending, write mask RGB, one 4-byte UNORM8 target, one + * sample. AGXMetalA9 runs it OPAQUE (FAST_PUNCH_THROUGH with a discard) with ISPCTL upass 7, the program + * keeping the destination alpha from a tile read issued before its kill. 1: the program may discard; + * The qualified masked form is selected at pipeline creation. */ + uint8_t mask_upass; + /* The 4x LOAD background only (pvr_g5p_compile_load_ms()): its one plain 2D sample reads the target's + * 2D-multisample state at the task's own sample, as Apple's loader (an iOS capture of a + * 4x colour load): the operand g5p_usc_ms_own_sample() in the register after the coordinates, then the + * narrow sample with dimension code f9. False everywhere else. */ + bool ms_own_sample; + /* 4 raster samples: a program's one conditional discard kills all four samples (g5p_usc_discard_test_ms4(), + * Apple's 4x form); several discards keep the one-sample test (Apple's several-discard 4x form is not + * ported: such draws are refused at 4x) */ + bool ms4_kill; }; +#define PCO_G5P_HAS_MASK_UPASS 1 #define PCO_G5P_FS_KEY_DEFAULT ((struct pco_g5p_fs_key){.write_mask = 0xf}) enum pco_g5p_upload_kind { @@ -132,17 +153,30 @@ enum pco_g5p_upload_kind { PCO_G5P_UPLOAD_BLEND_CONSTANTS, /* 4 dwords: the dynamic blend constants (f32) */ PCO_G5P_UPLOAD_FETCH_SAMPLER, /* two generated samplerless-read policy words */ }; +/* PCO_G5P_UPLOAD_IMAGE's offset (the state is always descriptor +0): how the + * program samples it. AS_2D: an array sample at the constant layer 0, drawn + * as a 2D sample, so the driver uploads the view's base layer as a 2D state + * (a 2D_ARRAY state converted, pvr_g5p_tex_state_as_2d()); ARRAY: an array + * sample, which needs a 2D_ARRAY state (the A9's array sample reads its + * layer through the array descriptor even for one layer). */ +/* PCO_G5P_UPLOAD_SSBO_ADDR at this offset: not the buffer's address but its + * bound size in bytes (one dword; get_ssbo_size, WGSL arrayLength) */ +#define PCO_G5P_SSBO_SIZE_OFFSET 0xfffcu +#define PCO_G5P_UPLOAD_IMAGE_AS_2D 1u +#define PCO_G5P_UPLOAD_IMAGE_ARRAY 2u + /* Varyings (docs/gpu/experiments/varyings.md). Apple's A9 compiler takes at * most 60 interpolated components ("Number of varying components(64) * exceeds the limit (60)"): 15 vec4 fragment inputs. */ -/* Plan row R2 (desktop-gl-varyings.md section 5): 16 vec4, the 64 components - * the driver reports (maxVertexOutputComponents, maxFragmentInputComponents) - * and stock Zink's IMG GL limit at 64 (16 varyings). Beyond Apple's per-target - * 60, inside the hardware-proven range (varyings-capacity-1: exact to 124 on - * the A9, flat included). With the position, a point size and 8 clip - * distances the vertex is 4 + 64 + 1 + 8 = 77 dwords, inside the TA's 128. */ -#define PCO_G5P_MAX_FS_VARYINGS 16 +/* 28 vec4, the 112 components the driver reports (maxVertexOutputComponents, + * maxFragmentInputComponents): WebGPU core wants 16 inter-stage locations + * (wgpu: components / 4 - 1). With the position, a point size + * and 8 clip distances the vertex is 4 + 112 + 1 + 8 = 125 dwords, inside the + * TA's 128. Beyond Apple's per-target 60, inside the hardware-proven range + * (varying-capacity and geometry-shader probes: exact to 124 on the A9, flat + * included). Before: 16 vec4, 64 components. */ +#define PCO_G5P_MAX_FS_VARYINGS 28 #define PCO_G5P_VARYING_SLOTS 32 /* VARYING_SLOT_VAR0..VAR31 */ enum pco_g5p_interp { @@ -171,6 +205,9 @@ struct pco_g5p_vs_link { /* a point pipeline: gl_PointSize is the last output dword, after the * varyings (TA_OUTPUT_SEL psprite_size_pres; Apple's g3b-points) */ bool point_size; + /* user clip distances admitted (the enabled user-plane path): their + * dwords follow the point size, the vertex's last (Apple's applegpu_g5p) */ + bool clip_distance; }; /* Per-draw transform feedback block (PCO_G5P_UPLOAD_XFB): the VA each @@ -233,6 +270,7 @@ struct pco_g5p_program { /* VS of a non-point pipeline: it wrote gl_PointSize, which was dropped * (set by pvr_g5p_nir.c after compilation; polygon mode POINT refuses it) */ bool psize_dropped; + uint8_t clip_count; /* VS: user clip distances, the last output dwords (TA user planes 0..n-1) */ char refusal[512]; }; diff --git a/src/imagination/g9p/g9p_a9.h b/src/imagination/g9p/g9p_a9.h index 1bb9a7235b6..d7b27d97d04 100644 --- a/src/imagination/g9p/g9p_a9.h +++ b/src/imagination/g9p/g9p_a9.h @@ -1,7 +1,25 @@ /* SPDX-License-Identifier: MIT */ #ifndef G9P_A9_H #define G9P_A9_H -#define G9P_A9_FIELDS bool a9; bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; +/* Compute texture words the dispatch puts in the common store (g9p_a9_tex.h) */ +#define G9P_A9_MAX_UPLOADS 10 +enum g9p_a9_upload_kind { G9P_A9_UP_IMAGE = 1, G9P_A9_UP_SAMPLER, G9P_A9_UP_FETCH_SAMPLER, G9P_A9_UP_SIZE, G9P_A9_UP_LOD }; +struct g9p_a9_upload { + unsigned char kind, set, binding, element; + unsigned char combined; /* a sampler in a combined image sampler: its words at +32 */ + unsigned char level; /* a size block's level */ + unsigned char common, dwords; + uint32_t value; /* literal common-store LOD */ +}; +/* g9p_shader::a9_bank1_regs: the kernel record's register field allocates bank 1 (g9p_cdm.h) */ +#define G9P_A9_HAS_BANK1_REGS 1 +/* A9 threadgroup word of the workgroup ID z (g9p_shader::a9_wgid_z: the kernel reads it). Words 0-1 hold x and y, + * word 2 the software-barrier counter (g5p_barrier.h, k = 2), shared variables start at word 4 (G9P_SHARED_BASE). + * The kernel PDS writes z here from temps q1's upper half, as AGXMetalA9's PDSEncoderGen1::EncodeComputePDSProgram + * (0x207d1ea4c) does after x and y when the kernel uses it (pvr_arch_g5p_compute.c). */ +#define G9P_A9_TG_WGID_Z 3 +#define G9P_A9_FIELDS bool a9; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; \ + unsigned a9_num_uploads; struct g9p_a9_upload a9_uploads[G9P_A9_MAX_UPLOADS]; char a9_refusal[192]; #define G9P_A9_END_PAD(ctx) do { if ((ctx)->out->a9 && (2*(ctx)->c->n+6)%64==0) G9P_EMIT((ctx)->c,0xc000); } while(0) /* Keep main's G9P numbering; only the A9 backend uses NIR-order breaks. */ #define G9P_A9_EMPTY_BREAK_BEFORE(ctx, side) do { if (!(ctx)->out->a9 || !(side)) (ctx)->ip++; } while (0) @@ -12,6 +30,7 @@ if ((ctx)->out->a9 && (ctx)->stage == MESA_SHADER_COMPUTE) \ (ctx)->out->a9_bodyless = !(n); \ } while (0) +const char *g9p_a9_last_refusal(void); struct g9p_shader; bool g9p_a9_compile_compute(nir_shader *, struct g9p_shader *, void *); #endif diff --git a/src/imagination/g9p/g9p_a9_alu.h b/src/imagination/g9p/g9p_a9_alu.h index 70706829cbf..3d653bb8ff9 100644 --- a/src/imagination/g9p/g9p_a9_alu.h +++ b/src/imagination/g9p/g9p_a9_alu.h @@ -70,6 +70,31 @@ a9_alu_fma_literals(struct g9p_ctx *ctx, nir_alu_instr *alu, nir_scalar s[3], bo ctx->reg[key(&alu->def, 0)] = dst; return true; } +/* wa_h/wa_f8: Apple's f32-to-f16 conversion writes one half of a bank-1 + * dword. Convert both scalar sources without a software float pack. */ +static bool +a9_alu_pack_half(struct g9p_ctx *ctx, nir_alu_instr *alu, nir_scalar s[3]) +{ + int dst = alloc_reg(ctx); + if (dst < 0) return false; + ctx->temps |= BITFIELD64_BIT(dst); + for (unsigned i = 0; i < 2; i++) { + int r = src_reg(ctx, s[i]); + if (r < 0 || (r = one_src(ctx, r)) < 0) return false; + unsigned at = ctx->c->n; + g9p_f16_of(ctx->c, dst, r); + (*g9p_code_word(ctx->c, at + 1)) |= i; /* the high half: bit 0 of the destination half index, as Apple's half packs */ + } + release(ctx, &alu->instr); + ctx->busy |= BITFIELD64_BIT(dst); + ctx->reg[key(&alu->def, 0)] = dst; + return true; +} +# define G9P_A9_ALU_PACK_HALF() do { \ + if (ctx->out->a9 && ctx->stage == MESA_SHADER_COMPUTE && \ + alu->op == nir_op_pack_half_2x16_split) \ + return a9_alu_pack_half(ctx, alu, s); \ + } while (0) # define G9P_A9_BRANCH_IMMEDIATE() do { \ uint32_t value; \ if (ctx->out->a9 && !is_float && a >= 0 && \ diff --git a/src/imagination/g9p/g9p_a9_compute.h b/src/imagination/g9p/g9p_a9_compute.h index eac2eb87e45..9444c726bcc 100644 --- a/src/imagination/g9p/g9p_a9_compute.h +++ b/src/imagination/g9p/g9p_a9_compute.h @@ -1,20 +1,313 @@ /* SPDX-License-Identifier: MIT */ +/* One invocation is the whole subgroup on G5P. */ +static bool +a9_lower_subgroups_size1(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + b->cursor = nir_before_instr(&in->instr); + nir_def *v, *bit, *parts[NIR_MAX_VEC_COMPONENTS]; + unsigned n = in->def.num_components, bits = in->def.bit_size; + switch (in->intrinsic) { + case nir_intrinsic_load_subgroup_size: + v = nir_imm_int(b, 1); break; + case nir_intrinsic_load_subgroup_invocation: + case nir_intrinsic_first_invocation: + case nir_intrinsic_last_invocation: + v = nir_imm_int(b, 0); break; + case nir_intrinsic_load_num_subgroups: + v = nir_imm_int(b, b->shader->info.workgroup_size[0] * + b->shader->info.workgroup_size[1] * + b->shader->info.workgroup_size[2]); break; + case nir_intrinsic_load_subgroup_id: + v = nir_load_local_invocation_index(b); break; + case nir_intrinsic_load_subgroup_eq_mask: + case nir_intrinsic_load_subgroup_ge_mask: + case nir_intrinsic_load_subgroup_le_mask: + case nir_intrinsic_ballot: + parts[0] = in->intrinsic == nir_intrinsic_ballot ? + nir_b2iN(b, in->src[0].ssa, bits) : nir_imm_intN_t(b, 1, bits); + for (unsigned j = 1; j < n; j++) parts[j] = nir_imm_intN_t(b, 0, bits); + v = nir_vec(b, parts, n); break; + case nir_intrinsic_load_subgroup_gt_mask: + case nir_intrinsic_load_subgroup_lt_mask: + case nir_intrinsic_ballot_bit_count_exclusive: + v = nir_imm_zero(b, n, bits); break; + case nir_intrinsic_vote_ieq: + case nir_intrinsic_vote_feq: + case nir_intrinsic_elect: + v = nir_imm_true(b); break; + case nir_intrinsic_vote_any: + case nir_intrinsic_vote_all: + case nir_intrinsic_read_invocation: + case nir_intrinsic_read_first_invocation: + case nir_intrinsic_shuffle: + case nir_intrinsic_shuffle_xor: + case nir_intrinsic_shuffle_up: + case nir_intrinsic_shuffle_down: + case nir_intrinsic_rotate: + case nir_intrinsic_quad_broadcast: + case nir_intrinsic_quad_swap_horizontal: + case nir_intrinsic_quad_swap_vertical: + case nir_intrinsic_quad_swap_diagonal: + case nir_intrinsic_quad_vote_any: + case nir_intrinsic_quad_vote_all: + case nir_intrinsic_reduce: + case nir_intrinsic_inclusive_scan: + v = in->src[0].ssa; break; + case nir_intrinsic_inverse_ballot: + case nir_intrinsic_ballot_bitfield_extract: + case nir_intrinsic_ballot_bit_count_reduce: + case nir_intrinsic_ballot_bit_count_inclusive: + case nir_intrinsic_ballot_find_lsb: + case nir_intrinsic_ballot_find_msb: + bit = nir_iand_imm(b, nir_channel(b, in->src[0].ssa, 0), 1); + if (in->intrinsic == nir_intrinsic_inverse_ballot) + v = nir_ine_imm(b, bit, 0); + else if (in->intrinsic == nir_intrinsic_ballot_bitfield_extract) + v = nir_iand(b, nir_ieq_imm(b, in->src[1].ssa, 0), nir_ine_imm(b, bit, 0)); + else if (in->intrinsic == nir_intrinsic_ballot_find_lsb || + in->intrinsic == nir_intrinsic_ballot_find_msb) + v = nir_bcsel(b, nir_ine_imm(b, bit, 0), nir_imm_int(b, 0), nir_imm_int(b, -1)); + else + v = nir_u2uN(b, bit, bits); + break; + case nir_intrinsic_exclusive_scan: { + nir_const_value identity = nir_alu_binop_identity(nir_intrinsic_reduction_op(in), bits); + nir_def *scalar = nir_build_imm(b, 1, bits, &identity); + for (unsigned j = 0; j < n; j++) parts[j] = scalar; + v = nir_vec(b, parts, n); break; + } + default: return false; + } + nir_def_replace(&in->def, v); + return true; +} + +/* Apple's shared-memory size codes require size-aligned 1/2/4-dword runs. */ +static nir_mem_access_size_align +a9_shared_size_align(nir_intrinsic_op op, uint8_t bytes, uint8_t bit_size, + uint32_t align_mul, uint32_t align_offset, + bool offset_is_const, enum gl_access_qualifier access, + const void *data) +{ + unsigned align = nir_combined_align(align_mul, align_offset); + if (bit_size != 32 || align < 4) + return (nir_mem_access_size_align) { 1, bit_size, bit_size / 8 }; + unsigned comps = bytes / 4; + unsigned n = comps >= 4 && align >= 16 ? 4 : comps >= 2 && align >= 8 ? 2 : 1; + return (nir_mem_access_size_align) { n, 32, 4 * n, nir_mem_access_shift_method_scalar }; +} + +/* Specialise storage writes before ALU lowering. Two data components mark + * Apple's packed 0x18 form; raw 0x1a always has four dwords. Cube image + * coordinates are already x,y,face (or 6*cube+face) in SPIR-V/NIR. */ +static bool +a9_pack_image_store(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + if (in->intrinsic != nir_intrinsic_image_deref_store) + return false; + nir_variable *v = nir_deref_instr_get_variable(nir_src_as_deref(in->src[0])); + if (!v || nir_src_bit_size(in->src[3]) != 32) + return false; + b->cursor = nir_before_instr(&in->instr); + nir_def *value = in->src[3].ssa, *lo, *hi = nir_imm_int(b, 0); + switch (v->data.image.format) { + case PIPE_FORMAT_R32_UINT: case PIPE_FORMAT_R32_SINT: case PIPE_FORMAT_R32_FLOAT: + value = nir_vec4(b, nir_channel(b, value, 0), hi, hi, hi); + break; + case PIPE_FORMAT_R32G32B32A32_UINT: case PIPE_FORMAT_R32G32B32A32_SINT: + case PIPE_FORMAT_R32G32B32A32_FLOAT: + return false; + case PIPE_FORMAT_R8G8B8A8_UNORM: + lo = nir_pack_unorm_4x8(b, value); + value = nir_vec2(b, lo, hi); + break; + case PIPE_FORMAT_R16G16B16A16_FLOAT: + hi = nir_pack_half_2x16(b, nir_channels(b, value, 0xc)); + FALLTHROUGH; + case PIPE_FORMAT_R16G16_FLOAT: + lo = nir_pack_half_2x16(b, nir_channels(b, value, 0x3)); + value = nir_vec2(b, lo, hi); + break; + case PIPE_FORMAT_R16_FLOAT: + lo = nir_pack_half_2x16(b, nir_vec2(b, nir_channel(b, value, 0), nir_imm_float(b, 0))); + value = nir_vec2(b, lo, hi); + break; + default: + return false; /* a9_image() records the format by name */ + } + nir_src_rewrite(&in->src[3], value); + in->num_components = value->num_components; + return true; +} + +/* The texture unit takes an integer array layer even for filtered samples. */ +static bool +a9_lower_array_layer(nir_builder *b, nir_tex_instr *tex, void *data) +{ + if (tex->op != nir_texop_txl || tex->sampler_dim != GLSL_SAMPLER_DIM_2D || !tex->is_array) + return false; + int ci = nir_tex_instr_src_index(tex, nir_tex_src_coord); + if (ci < 0) return false; + b->cursor = nir_before_instr(&tex->instr); + nir_def *co = tex->src[ci].src.ssa; + nir_def *layer = nir_f2u32(b, nir_fround_even(b, nir_fmax(b, nir_channel(b, co, 2), + nir_imm_float(b, 0)))); + nir_src_rewrite(&tex->src[ci].src, + nir_vec3(b, nir_channel(b, co, 0), nir_channel(b, co, 1), layer)); + return true; +} + +/* The grid owns disjoint stride-sized regions. Keep this before shared + * vectorisation: the ordinary SSBO lowering handles vectors and atomics. */ +static bool +a9_spill_shared(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + if (in->intrinsic == nir_intrinsic_barrier) { + nir_variable_mode modes = nir_intrinsic_memory_modes(in); + if (!(modes & nir_var_mem_shared)) return false; + nir_intrinsic_set_memory_modes(in, modes | nir_var_mem_ssbo); + return true; + } + nir_intrinsic_op op; + unsigned off_src = 0; + switch (in->intrinsic) { + case nir_intrinsic_load_shared: op = nir_intrinsic_load_ssbo; break; + case nir_intrinsic_store_shared: op = nir_intrinsic_store_ssbo; off_src = 1; break; + case nir_intrinsic_shared_atomic: op = nir_intrinsic_ssbo_atomic; break; + case nir_intrinsic_shared_atomic_swap: op = nir_intrinsic_ssbo_atomic_swap; break; + default: return false; + } + b->cursor = nir_before_instr(&in->instr); + nir_def *wg = nir_load_workgroup_id(b), *nwg = nir_load_num_workgroups(b); + nir_def *linear = nir_iadd(b, nir_channel(b, wg, 0), + nir_imul(b, nir_channel(b, nwg, 0), nir_iadd(b, nir_channel(b, wg, 1), + nir_imul(b, nir_channel(b, nwg, 1), nir_channel(b, wg, 2))))); + nir_def *offset = nir_iadd(b, nir_imul_imm(b, linear, *(unsigned *)data), + nir_iadd_imm(b, in->src[off_src].ssa, nir_intrinsic_base(in))); + nir_def *index = nir_vulkan_resource_index(b, 2, 32, nir_imm_int(b, 0), + .desc_set = G9P_SHARED_SPILL, .binding = 0, + .desc_type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER); + nir_intrinsic_instr *spill = nir_intrinsic_instr_create(b->shader, op); + spill->num_components = in->num_components; + nir_intrinsic_set_access(spill, nir_intrinsic_access(in)); + if (op == nir_intrinsic_store_ssbo) { + spill->src[0] = nir_src_for_ssa(in->src[0].ssa); + spill->src[1] = nir_src_for_ssa(index); + spill->src[2] = nir_src_for_ssa(offset); + nir_intrinsic_set_write_mask(spill, nir_intrinsic_write_mask(in)); + } else { + spill->src[0] = nir_src_for_ssa(index); + spill->src[1] = nir_src_for_ssa(offset); + if (op != nir_intrinsic_load_ssbo) { + nir_intrinsic_set_atomic_op(spill, nir_intrinsic_atomic_op(in)); + spill->src[2] = nir_src_for_ssa(in->src[1].ssa); + if (op == nir_intrinsic_ssbo_atomic_swap) + spill->src[3] = nir_src_for_ssa(in->src[2].ssa); + } + nir_def_init(&spill->instr, &spill->def, in->def.num_components, in->def.bit_size); + } + if (op == nir_intrinsic_load_ssbo || op == nir_intrinsic_store_ssbo) { + nir_intrinsic_set_align_mul(spill, nir_intrinsic_align_mul(in)); + nir_intrinsic_set_align_offset(spill, + (nir_intrinsic_align_offset(in) + nir_intrinsic_base(in)) % nir_intrinsic_align_mul(in)); + } + nir_builder_instr_insert(b, &spill->instr); + if (op != nir_intrinsic_store_ssbo) + nir_def_rewrite_uses(&in->def, &spill->def); + nir_instr_remove(&in->instr); + return true; +} + + bool g9p_a9_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_ctx) { + nir->options = &nir_options_a9; const nir_lower_compute_system_values_options csv = { .lower_cs_local_id_to_index = true, .global_id_is_32bit = true, }; memset(out, 0, sizeof(*out)); out->a9 = true; + a9_last_refusal[0] = 0; lower_front(nir); - NIR_PASS(_, nir, nir_lower_vars_to_explicit_types, nir_var_mem_shared, glsl_get_natural_size_align_bytes); - NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_shared, nir_address_format_32bit_offset); + NIR_PASS(_, nir, nir_shader_intrinsics_pass, a9_lower_subgroups_size1, + nir_metadata_control_flow, NULL); + NIR_PASS(_, nir, nir_shader_intrinsics_pass, a9_pack_image_store, + nir_metadata_control_flow, NULL); + NIR_PASS(_, nir, nir_lower_alu); + /* lower_front() already allocated and lowered shared variables. Doing + * it again double-counts the allocation after addresses became explicit. */ if (nir->info.shared_size > 16384) { - fprintf(stderr, "g5p: shared allocation exceeds16384 bytes\n"); - return false; + out->shared_spill_stride = ALIGN_POT(nir->info.shared_size, 64); + NIR_PASS(_, nir, nir_shader_intrinsics_pass, a9_spill_shared, + nir_metadata_control_flow, &out->shared_spill_stride); + nir->info.shared_size = 0; } + const nir_lower_mem_access_bit_sizes_options shared_opts = { + .modes = nir_var_mem_shared, .callback = a9_shared_size_align, + }; + NIR_PASS(_, nir, nir_lower_mem_access_bit_sizes, &shared_opts); out->a9_local_dwords = 4 + DIV_ROUND_UP(nir->info.shared_size, 4); + NIR_PASS(_, nir, nir_shader_tex_pass, a9_lower_array_layer, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_lower_compute_system_values, &csv); NIR_PASS(_, nir, nir_lower_undef_to_zero, NULL); - return compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx); + /* A kernel that ran out of registers is compiled again in pressure mode + * (g9p_compile.c "pressure"). Clone before compile_back mutates nir. */ + nir_shader *again = nir_shader_clone(mem_ctx, nir); + const struct g9p_shader saved = *out; + /* The first attempt is provisional: an admitted retry is not a refusal. */ + + const bool compiled = compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, + false, false, mem_ctx, false, false); + + if (!compiled) { + if (!again || strncmp(out->a9_refusal, "out of registers", 16)) { + + return false; + } + /* Preserve the existing pressure retry and its 64-home ceiling. Only a second + * register refusal gets Apple's larger budget, using the same pressure + * mode on a fresh clone of the prepared NIR. */ + nir_shader *budget = nir_shader_clone(mem_ctx, again); + *out = saved; + + const bool retried = compile_back(again, out, MESA_SHADER_COMPUTE, NULL, 0, + false, false, mem_ctx, true, false); + + if (!retried) { + if (strncmp(out->a9_refusal, "out of registers", 16)) { + + return false; + } + *out = saved; + if (!compile_back(budget, out, MESA_SHADER_COMPUTE, NULL, 0, + false, false, mem_ctx, true, true)) + return false; + } + } + /* the fixed common-store places of texture words (g9p_a9_tex.h) */ + bool sizes = false, lods = false; + for (unsigned i = 0; i < out->a9_num_uploads; i++) { + sizes |= out->a9_uploads[i].kind == G9P_A9_UP_SIZE; + lods |= out->a9_uploads[i].kind == G9P_A9_UP_LOD; + } + if (lods && out->num_buffers > 13) { + if (!out->a9_refusal[0]) + snprintf(out->a9_refusal, sizeof(out->a9_refusal), + "more than 13 buffers with shared texture levels (common 28-30)"); + return backend_failure(out, &(struct g9p_ctx){0}); + } + if (out->a9_num_uploads && out->num_buffers > 15) { + if (!out->a9_refusal[0]) + snprintf(out->a9_refusal, sizeof(out->a9_refusal), + "%u buffers with sampled textures (buffer words reach the texture words at common 32)", + out->num_buffers); + return backend_failure(out, &(struct g9p_ctx){0}); + } + if (sizes && out->num_textures) { + if (!out->a9_refusal[0]) + snprintf(out->a9_refusal, sizeof(out->a9_refusal), + "a texture size query together with a storage image (both at common 40-47)"); + return backend_failure(out, &(struct g9p_ctx){0}); + } + return true; } diff --git a/src/imagination/g9p/g9p_a9_intrinsics.h b/src/imagination/g9p/g9p_a9_intrinsics.h index e73a3f4ac10..9c1388baee9 100644 --- a/src/imagination/g9p/g9p_a9_intrinsics.h +++ b/src/imagination/g9p/g9p_a9_intrinsics.h @@ -1,9 +1,10 @@ /* SPDX-License-Identifier: MIT */ #include "../g5p/g5p_barrier.h" #include "../g5p/g5p_atomic.h" +#include "../g5p/g5p_shared_atomic.h" #include "../g5p/g5p_image.h" /* Static threadgroup operands: A9 host-compiler forms (compute.md ยง4). - * Reserve TG0/1 for IDs and TG2/3 for the future software-barrier counters. + * TG0/1 hold the workgroup IDs x/y, TG2 the software-barrier counter, TG3 the ID z (G9P_A9_TG_WGID_Z). * CDM allocation covers system words plus the native shared block. */ /* S and RI address forms from compute-isa-shared-atomic.md ยง1.3. * RI reads a temporary half index for the last time (short parcel8800). @@ -14,37 +15,50 @@ static bool a9_shared(struct g9p_ctx *ctx, nir_intrinsic_instr *in) nir_src offset = in->src[load ? 0 : 1]; unsigned bits = load ? in->def.bit_size : nir_src_bit_size(in->src[0]); unsigned comps = load ? in->def.num_components : nir_src_num_components(in->src[0]); - if (!ctx->out->a9 || bits != 32 || comps != 1 || (nir_intrinsic_base(in) & 3)) + unsigned base = 16 + nir_intrinsic_base(in); + unsigned unit = 4 * comps, shift_bits = util_logbase2(unit); + unsigned size = comps == 1 ? G9P_SIZE_U32 : comps == 2 ? G9P_SIZE_2X32 : G9P_SIZE_4X32; + if (!ctx->out->a9 || bits != 32 || (comps != 1 && comps != 2 && comps != 4) || + base % unit || (!load && nir_intrinsic_write_mask(in) != BITFIELD_MASK(comps))) return false; - unsigned word = 4 + nir_intrinsic_base(in) / 4; + unsigned word = base / unit; bool constant = nir_src_is_const(offset); int index = -1; if (constant) { unsigned byte = nir_src_as_uint(offset); - if (byte & 3) return false; - word += byte / 4; + if (byte % unit) return false; + word += byte / unit; } else { int src = src_reg(ctx, nir_get_scalar(offset.ssa, 0)); - int shift = const_reg(ctx, 2); + int shift = const_reg(ctx, shift_bits); index = alloc_temp(ctx); if (src < 0 || shift < 0 || index < 0) return false; g9p_shift(ctx->c, index, src, shift, true); - G9P_EMIT(ctx->c, 0x1189, 0x0300 | 2 * index, - 0x0101 | index << 10, 0x0120, 0x0fa1, 0x4000); + g9p_shared_index(ctx->c, index, index, shift_bits); } - int reg = load ? alloc_reg(ctx) : src_reg(ctx, nir_get_scalar(in->src[0].ssa, 0)); - if (reg < 0 || word >= 65536) return false; - unsigned w1 = 0x0100 | (!load ? 0x80 : 0) | 2 * reg; - if (constant) { - G9P_EMIT(ctx->c, 0x2314, w1, 0x8100 | (word & 255), 0x0200 | (word >> 8)); + int reg; + if (load) { + reg = comps == 1 ? alloc_reg(ctx) : alloc_range(ctx, comps); + } else if (comps == 1) { + reg = src_reg(ctx, nir_get_scalar(in->src[0].ssa, 0)); } else { - w1 |= (2 * index) << 9; - if (word < 256) G9P_EMIT(ctx->c, 0x2314, w1, 0x8800 | word); - else G9P_EMIT(ctx->c, 0x2314, w1, 0x8900 | (word & 255), word >> 8); + reg = alloc_range(ctx, comps); + if (reg < 0) return false; + ctx->temps |= BITFIELD64_RANGE(reg, comps); + for (unsigned j = 0; j < comps; j++) { + int r = src_reg(ctx, nir_get_scalar(in->src[0].ssa, j)); + if (r < 0) return false; + g9p_alu2(ctx->c, G9P_OR, reg + j, r, r, false); + } } + if (reg < 0 || 2 * reg + 2 * comps - 1 >= 128 || word >= 4096) return false; + g9p_shared_access(ctx->c, size, !load, reg, word, constant ? -1 : 2 * index); g9p_wait(ctx->c); release(ctx, &in->instr); - if (load) ctx->reg[key(&in->def, 0)] = reg; + if (load) for (unsigned j = 0; j < comps; j++) { + ctx->reg[key(&in->def, j)] = reg + j; + if (!ctx->last_use[key(&in->def, j)]) ctx->busy &= ~BITFIELD64_BIT(reg + j); + } return true; } /* Keep the election's local-index source live for every task barrier. */ @@ -80,7 +94,7 @@ static bool a9_barrier(struct g9p_ctx *ctx, nir_intrinsic_instr *in) bool device=modes & (nir_var_mem_ssbo | nir_var_mem_global | nir_var_image); if ((modes & ~(nir_var_mem_shared | nir_var_mem_ssbo | nir_var_mem_global | nir_var_image)) || (scope != SCOPE_NONE && scope != SCOPE_INVOCATION && scope != SCOPE_WORKGROUP)) { - fprintf(stderr, "g5p: unsupported barrier memory mode or execution scope\n"); + mesa_logw("g5p: unsupported barrier memory mode or execution scope"); return false; } int fence_pair=-1; @@ -94,8 +108,10 @@ static bool a9_barrier(struct g9p_ctx *ctx, nir_intrinsic_instr *in) g9p_mov_uniform(ctx->c,fence_pair+1,57,zero); } if (scope == SCOPE_WORKGROUP) { - unsigned N = ctx->a9_barriers_remaining--; - if (!N || N > 31) return false; + /* Apple's A9 b30/b32/b40/b70 (disassembled barrier kernels): both + * X2f and X0f count down and wrap in 1..15, not 1..31. */ + unsigned rem = ctx->a9_barriers_remaining--; + unsigned N = ((rem - 1) % 15) + 1; if (n > 32) { int r = alloc_temp(ctx), t = alloc_temp(ctx); if (r < 0 || t < 0) return false; @@ -114,6 +130,51 @@ static bool a9_barrier(struct g9p_ctx *ctx, nir_intrinsic_instr *in) g9p_wait(ctx->c); return true; } +static bool a9_shared_atomic(struct g9p_ctx *ctx, nir_intrinsic_instr *in) +{ + if (!ctx->out->a9 || ctx->stage != MESA_SHADER_COMPUTE || + in->def.bit_size != 32 || in->def.num_components != 1 || + (nir_intrinsic_base(in) & 3)) return false; + nir_atomic_op op = nir_intrinsic_atomic_op(in); + bool cas = in->intrinsic == nir_intrinsic_shared_atomic_swap; + if (cas && op != nir_atomic_op_cmpxchg) return false; + switch (op) { + case nir_atomic_op_iadd: + case nir_atomic_op_imin: case nir_atomic_op_umin: + case nir_atomic_op_imax: case nir_atomic_op_umax: + case nir_atomic_op_iand: case nir_atomic_op_ior: case nir_atomic_op_ixor: + case nir_atomic_op_xchg: break; + case nir_atomic_op_cmpxchg: if (!cas) return false; break; + default: + mesa_logw("g5p: unsupported shared atomic op %u", op); + return false; + } + unsigned word = 4 + nir_intrinsic_base(in) / 4; + int index = -1; + if (nir_src_is_const(in->src[0])) { + unsigned byte = nir_src_as_uint(in->src[0]); + if (byte & 3) return false; + word += byte / 4; + } else { + int src = src_reg(ctx, nir_get_scalar(in->src[0].ssa, 0)); + int shift = const_reg(ctx, 2); + index = alloc_temp(ctx); + if (src < 0 || shift < 0 || index < 0) return false; + g9p_shift(ctx->c, index, src, shift, true); + g9p_shared_index(ctx->c, index, index, 2); /* a dword index */ + } + int val = src_reg(ctx, nir_get_scalar(in->src[cas ? 2 : 1].ssa, 0)); + int cmp = cas ? src_reg(ctx, nir_get_scalar(in->src[1].ssa, 0)) : 0; + int old = alloc_reg(ctx), next = alloc_temp(ctx); + if (val < 0 || cmp < 0 || old < 0 || next < 0 || word >= 4096) return false; + g5p_shared_atomic(ctx->c, op, old, val, cmp, next, word, + index < 0 ? -1 : 2 * index, ctx->st); + release(ctx, &in->instr); + ctx->reg[key(&in->def, 0)] = old; + if (!ctx->last_use[key(&in->def, 0)]) ctx->busy &= ~BITFIELD64_BIT(old); + return true; +} + static bool a9_ssbo_atomic(struct g9p_ctx *ctx, nir_intrinsic_instr *in) { if (!ctx->out->a9 || ctx->stage != MESA_SHADER_COMPUTE || @@ -167,48 +228,95 @@ static bool a9_ssbo_cas(struct g9p_ctx *ctx, nir_intrinsic_instr *in) } static bool a9_image(struct g9p_ctx *ctx, nir_intrinsic_instr *in) { - bool store=in->intrinsic==nir_intrinsic_image_deref_store; - if(!ctx->out->a9 || ctx->stage!=MESA_SHADER_COMPUTE || - nir_intrinsic_image_dim(in)!=GLSL_SAMPLER_DIM_2D || nir_intrinsic_image_array(in))return false; - nir_deref_instr *deref=nir_src_as_deref(in->src[0]); - nir_variable *v=nir_deref_instr_get_variable(deref); - if(!v || deref->deref_type!=nir_deref_type_var || v->data.image.format!=PIPE_FORMAT_R32_UINT || - !nir_src_is_const(in->src[store?4:3]) || nir_src_as_uint(in->src[store?4:3])!=0)return false; - if(ctx->out->num_textures && (ctx->out->textures[0].set!=v->data.descriptor_set || - ctx->out->textures[0].binding!=v->data.binding))return false; - ctx->out->num_textures=1; - ctx->out->textures[0].set=v->data.descriptor_set; - ctx->out->textures[0].binding=v->data.binding; - int co=alloc_pair(ctx); - if(co<0 || co>30)return false; - ctx->temps |= BITFIELD64_RANGE(co,2); - for(unsigned j=0;j<2;j++) { - int r=src_reg(ctx,nir_get_scalar(in->src[1].ssa,j)); - if(r<0)return false; - g9p_alu2(ctx->c,G9P_OR,co+j,r,r,false); - } - int data=alloc_range(ctx,4); - if(data<0 || data>28)return false; - if(store) { - if(nir_src_bit_size(in->src[3])!=32 || nir_src_num_components(in->src[3])!=4)return false; - ctx->temps |= BITFIELD64_RANGE(data,4); - for(unsigned j=0;j<4;j++) { - int r=src_reg(ctx,nir_get_scalar(in->src[3].ssa,j)); - if(r<0)return false; - g9p_alu2(ctx->c,G9P_OR,data+j,r,r,false); + bool store = in->intrinsic == nir_intrinsic_image_deref_store; + enum glsl_sampler_dim dim = nir_intrinsic_image_dim(in); + bool array = nir_intrinsic_image_array(in); + if (!ctx->out->a9 || ctx->stage != MESA_SHADER_COMPUTE) + return false; + if ((dim != GLSL_SAMPLER_DIM_2D && !(store && dim == GLSL_SAMPLER_DIM_CUBE)) || + (!store && array)) { + char why[128]; + snprintf(why, sizeof(why), "image %s dimension %s is_array=%u unsupported", + store ? "store" : "load", (dim == GLSL_SAMPLER_DIM_CUBE ? "cube" : dim == GLSL_SAMPLER_DIM_3D ? "3D" : + dim == GLSL_SAMPLER_DIM_2D ? "2D" : "other"), array); + a9_tex_refuse(ctx, why); + return false; + } + nir_deref_instr *deref = nir_src_as_deref(in->src[0]); + nir_variable *v = nir_deref_instr_get_variable(deref); + if (!v || deref->deref_type != nir_deref_type_var) { + a9_tex_refuse(ctx, "image access without a constant storage binding"); + return false; + } + enum pipe_format format = v->data.image.format; + bool packed = false, supported = format == PIPE_FORMAT_R32_UINT; + if (store) switch (format) { + case PIPE_FORMAT_R32_UINT: case PIPE_FORMAT_R32_SINT: case PIPE_FORMAT_R32_FLOAT: + case PIPE_FORMAT_R32G32B32A32_UINT: case PIPE_FORMAT_R32G32B32A32_SINT: + case PIPE_FORMAT_R32G32B32A32_FLOAT: + supported = true; break; + case PIPE_FORMAT_R8G8B8A8_UNORM: case PIPE_FORMAT_R16G16B16A16_FLOAT: + case PIPE_FORMAT_R16G16_FLOAT: case PIPE_FORMAT_R16_FLOAT: + supported = packed = true; break; + default: break; + } + if (!supported) { + char why[160]; + snprintf(why, sizeof(why), "image %s format %s (%s)", store ? "store" : "load", + util_format_short_name(format), store ? "no A9 pack path yet" : "R32_UINT only"); + a9_tex_refuse(ctx, why); + return false; + } + if (!nir_src_is_const(in->src[store ? 4 : 3]) || + nir_src_as_uint(in->src[store ? 4 : 3]) != 0) { + a9_tex_refuse(ctx, "image access at a nonzero or dynamic level"); + return false; + } + if (ctx->out->num_textures && (ctx->out->textures[0].set != v->data.descriptor_set || + ctx->out->textures[0].binding != v->data.binding)) { + a9_tex_refuse(ctx, "more than one storage image binding"); + return false; + } + ctx->out->num_textures = 1; + ctx->out->textures[0].set = v->data.descriptor_set; + ctx->out->textures[0].binding = v->data.binding; + unsigned coords = array || dim == GLSL_SAMPLER_DIM_CUBE ? 3 : 2; + int co = alloc_range(ctx, coords); + if (co < 0 || co + coords > 32) return false; + ctx->temps |= BITFIELD64_RANGE(co, coords); + for (unsigned j = 0; j < coords; j++) { + int r = src_reg(ctx, nir_get_scalar(in->src[1].ssa, j)); + if (r < 0) return false; + g9p_alu2(ctx->c, G9P_OR, co + j, r, r, false); + } + unsigned words = packed ? 2 : 4; + int data = alloc_range(ctx, words); + if (data < 0 || data + words > 32) return false; + if (store) { + if (nir_src_bit_size(in->src[3]) != 32 || nir_src_num_components(in->src[3]) != words) + return false; + ctx->temps |= BITFIELD64_RANGE(data, words); + for (unsigned j = 0; j < words; j++) { + int r = src_reg(ctx, nir_get_scalar(in->src[3].ssa, j)); + if (r < 0) return false; + g9p_alu2(ctx->c, G9P_OR, data + j, r, r, false); } - } else if(in->def.bit_size!=32 || in->def.num_components>4)return false; - g5p_image_raw32(ctx->c,store,data,co,40,31); - release(ctx,&in->instr); - if(!store) for(unsigned j=0;j<4;j++) { - ctx->reg[key(&in->def,j)]=data+j; - if(j>=in->def.num_components || !ctx->last_use[key(&in->def,j)])ctx->busy &= ~BITFIELD64_BIT(data+j); + } else if (in->def.bit_size != 32 || in->def.num_components > 4) return false; + g5p_image_access(ctx->c, store, packed, coords == 3, data, co, 40, 31); + release(ctx, &in->instr); + if (!store) for (unsigned j = 0; j < 4; j++) { + ctx->reg[key(&in->def, j)] = data + j; + if (j >= in->def.num_components || !ctx->last_use[key(&in->def, j)]) + ctx->busy &= ~BITFIELD64_BIT(data + j); } return true; } #define G9P_A9_INTRINSICS \ if (ctx->out->a9) { switch (intr->intrinsic) { \ + case nir_intrinsic_image_deref_atomic: case nir_intrinsic_image_deref_atomic_swap: \ + a9_tex_refuse(ctx, "image atomics unsupported on A9"); return false; \ case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_load: return a9_image(ctx, intr); \ + case nir_intrinsic_shared_atomic: case nir_intrinsic_shared_atomic_swap: return a9_shared_atomic(ctx, intr); \ case nir_intrinsic_ssbo_atomic_swap: return a9_ssbo_cas(ctx, intr); \ case nir_intrinsic_ssbo_atomic: return a9_ssbo_atomic(ctx, intr); \ case nir_intrinsic_load_shared: case nir_intrinsic_store_shared: return a9_shared(ctx, intr); \ diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h new file mode 100644 index 00000000000..0069a620322 --- /dev/null +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -0,0 +1,366 @@ +/* SPDX-License-Identifier: MIT */ +#include "../g5p/g5p_usc.h" +#include "util/half_float.h" +/* + * A9 compute samples (WGSL textureSampleLevel/Grad/Load/Dimensions in a + * compute shader). Apple's A9 kernels (sample(s, uv, level(l)), + * gradient2d(), read(g, lod)) use the + * coupled operand tuples of the qualified vertex sample (g5p_usc_sample_ext() + * G5P_TEX_TUPLE_COUPLED_*), byte for byte: + * level(l) G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL, 2D, four channels, f16 level in r2 + * gradient G5P_TEX_TUPLE_COUPLED_GRAD, 2D, four channels, [dx.x dy.x dx.y dy.y] f32 + * The f16 level is Apple's f32-to-f16 conversion (g9p_f16_of(), the vertex + * level form without a last-use flag). Every sample here is followed by a + * wait, so each one issues in the state of Apple's single-sample kernels. + * + * Descriptor words go to fixed common-store dwords, inside the 64 the + * dispatch's DMA task already loads (pvr_arch_g5p_compute.c): texture + * states at 32 and 48 (four-qword aligned), sampler words at 58, 60, 62, + * size blocks (textureDimensions, CPU-computed as the fragment path's) at 40 + * and 44 when no storage image uses 40-47. Buffers keep dwords 0-2n, the + * device fence 56-57, a storage image 40-47 and its level 31. + */ +static const uint8_t a9_tex_image_dw[] = { 32, 48 }; +static const uint8_t a9_tex_sampler_dw[] = { 58, 60, 62 }; +static const uint8_t a9_tex_size_dw[] = { 40, 44 }; + +static void +a9_tex_refuse(struct g9p_ctx *ctx, const char *why) +{ + g9p_fail(ctx->out, "%s", why); +} + +/* The common dword of an upload, added if new; -1 when its slots are full */ +static int +a9_tex_upload(struct g9p_ctx *ctx, unsigned kind, unsigned set, unsigned binding, unsigned element, + unsigned combined, unsigned level) +{ + struct g9p_shader *out = ctx->out; + const uint8_t *slots = kind == G9P_A9_UP_IMAGE ? a9_tex_image_dw : + kind == G9P_A9_UP_SIZE ? a9_tex_size_dw : a9_tex_sampler_dw; + const unsigned nslots = kind == G9P_A9_UP_IMAGE ? ARRAY_SIZE(a9_tex_image_dw) : + kind == G9P_A9_UP_SIZE ? ARRAY_SIZE(a9_tex_size_dw) : ARRAY_SIZE(a9_tex_sampler_dw); + unsigned used = 0; + + for (unsigned i = 0; i < out->a9_num_uploads; i++) { + const struct g9p_a9_upload *u = &out->a9_uploads[i]; + const bool same_class = u->kind == kind || + ((u->kind == G9P_A9_UP_SAMPLER || u->kind == G9P_A9_UP_FETCH_SAMPLER) && + (kind == G9P_A9_UP_SAMPLER || kind == G9P_A9_UP_FETCH_SAMPLER)); + if (u->kind == kind && u->set == set && u->binding == binding && u->element == element && + u->combined == combined && u->level == level) + return u->common; + used += same_class; + } + if (used == nslots || out->a9_num_uploads == G9P_A9_MAX_UPLOADS || set > 255 || binding > 255 || + element > 255 || level > 255) { + a9_tex_refuse(ctx, kind == G9P_A9_UP_IMAGE ? "more than 2 sampled textures in a compute shader" : + kind == G9P_A9_UP_SIZE ? "more than 2 texture size queries in a compute shader" : + "more than 3 samplers in a compute shader"); + return -1; + } + out->a9_uploads[out->a9_num_uploads++] = (struct g9p_a9_upload){ + .kind = kind, .set = set, .binding = binding, .element = element, .combined = combined, + .level = level, .common = slots[used], + .dwords = kind == G9P_A9_UP_IMAGE ? 8 : kind == G9P_A9_UP_SIZE ? 4 : 2, + }; + return slots[used]; +} + +/* ra_f/sla_f/slc_f use common-store integer/f16 levels, not the + * temporary continuation of the older 2D samples. Three literal slots + * precede the storage-image level at 31; buffer pointers must end at 27. */ +static int +a9_tex_shared_lod(struct g9p_ctx *ctx, nir_tex_instr *tex, bool read) +{ + int li = nir_tex_instr_src_index(tex, nir_tex_src_lod); + assert(li < 0 || nir_src_is_const(tex->src[li].src)); + uint32_t value = li < 0 ? 0 : read ? nir_src_as_uint(tex->src[li].src) : + _mesa_float_to_half(nir_src_as_float(tex->src[li].src)); + unsigned used = 0; + for (unsigned i = 0; i < ctx->out->a9_num_uploads; i++) { + const struct g9p_a9_upload *u = &ctx->out->a9_uploads[i]; + if (u->kind != G9P_A9_UP_LOD) continue; + if (u->value == value) return u->common; + used++; + } + if (used == 3 || ctx->out->a9_num_uploads == G9P_A9_MAX_UPLOADS) { + a9_tex_refuse(ctx, "more than 3 shared texture levels in a compute shader"); + return -1; + } + ctx->out->a9_uploads[ctx->out->a9_num_uploads++] = (struct g9p_a9_upload) { + .kind = G9P_A9_UP_LOD, .common = 28 + used, .dwords = 1, .value = value, + }; + return 28 + used; +} + +static bool +a9_tex_var(nir_tex_instr *tex, nir_tex_src_type type, unsigned *set, unsigned *binding, unsigned *element) +{ + const int i = nir_tex_instr_src_index(tex, type); + if (i < 0) + return false; + nir_deref_instr *deref = nir_src_as_deref(tex->src[i].src); + nir_variable *var = nir_deref_instr_get_variable(deref); + if (!var) + return false; + *set = var->data.descriptor_set; + *binding = var->data.binding; + *element = 0; + if (deref->deref_type == nir_deref_type_array) { + if (!nir_src_is_const(deref->arr.index)) + return false; + *element = nir_src_as_uint(deref->arr.index); + } else if (deref->deref_type != nir_deref_type_var) { + return false; + } + return true; +} + +static void +a9_tex_emit_raw(struct g9p_ctx *ctx, const uint8_t *raw, unsigned n) +{ + for (unsigned i = 0; i + 1 < n; i += 2) + G9P_EMIT(ctx->c, raw[i] | raw[i + 1] << 8); +} + +/* A value copied into register @dst (OR copies any bit pattern) */ +static bool +a9_tex_copy(struct g9p_ctx *ctx, unsigned dst, nir_def *def, unsigned comp) +{ + const int r = src_reg(ctx, nir_get_scalar(def, comp)); + if (r < 0) + return false; + g9p_alu2(ctx->c, G9P_OR, dst, r, r, false); + return true; +} + +/* textureDimensions / textureNumLevels at a constant level: words of the + * CPU-computed size block (pvr_g5p_tex_size(): w, h, depth/layers, levels) */ +static bool +a9_txs(struct g9p_ctx *ctx, nir_tex_instr *tex) +{ + unsigned set, binding, element, level = 0; + const int li = nir_tex_instr_src_index(tex, nir_tex_src_lod); + + if (!a9_tex_var(tex, nir_tex_src_texture_deref, &set, &binding, &element)) { + a9_tex_refuse(ctx, "texture size query of a texture without a constant binding"); + return false; + } + if (tex->op == nir_texop_txs && li >= 0) { + if (!nir_src_is_const(tex->src[li].src)) { + a9_tex_refuse(ctx, "textureDimensions at a non-constant level in a compute shader"); + return false; + } + level = nir_src_as_uint(tex->src[li].src); + } + const int common = a9_tex_upload(ctx, G9P_A9_UP_SIZE, set, binding, element, 0, level); + if (common < 0) + return false; + int zero = const_reg(ctx, 0); + if (zero < 0 || (zero = low_reg(ctx, zero)) < 0) + return false; + const unsigned n = tex->op == nir_texop_query_levels ? 1 : tex->def.num_components; + int dst[4]; + for (unsigned c = 0; c < n; c++) { + if (!ctx->last_use[key(&tex->def, c)]) { + dst[c] = -1; + continue; + } + dst[c] = alloc_reg(ctx); + if (dst[c] < 0) + return false; + g9p_mov_uniform(ctx->c, dst[c], common + (tex->op == nir_texop_query_levels ? 3 : c), zero); + } + release(ctx, &tex->instr); + for (unsigned c = 0; c < n; c++) + ctx->reg[key(&tex->def, c)] = dst[c]; + return true; +} + +static bool +a9_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) +{ + if (tex->op == nir_texop_txs || tex->op == nir_texop_query_levels) + return a9_txs(ctx, tex); + + const bool read = tex->op == nir_texop_txf, grad = tex->op == nir_texop_txd; + const int ci = nir_tex_instr_src_index(tex, nir_tex_src_coord); + unsigned set, binding, element, sset, sbinding, selement, combined = 0; + + if (tex->op == nir_texop_tex || tex->op == nir_texop_txb) { + a9_tex_refuse(ctx, "an implicit-LOD or bias sample in a compute shader (no derivatives)"); + return false; + } + if (tex->op != nir_texop_txl && !grad && !read) { + a9_tex_refuse(ctx, "texture operation not supported in compute shaders yet"); + return false; + } + const bool array = tex->sampler_dim == GLSL_SAMPLER_DIM_2D && tex->is_array; + const bool cube = tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE; + const bool layered = array || cube; + if (tex->is_shadow || + (tex->sampler_dim != GLSL_SAMPLER_DIM_2D && !cube) || + (cube && (tex->is_array || tex->op != nir_texop_txl)) || (array && grad)) { + char why[160]; + const char *dim = cube ? "cube" : tex->sampler_dim == GLSL_SAMPLER_DIM_3D ? "3D" : + array ? "2D array" : tex->sampler_dim == GLSL_SAMPLER_DIM_2D ? "2D" : "other"; + snprintf(why, sizeof(why), "compute %s on %s is_array=%u is_shadow=%u unsupported", + (read ? "txf" : grad ? "txd" : "txl"), dim, tex->is_array, tex->is_shadow); + a9_tex_refuse(ctx, why); + return false; + } + if (nir_alu_type_get_base_type(tex->dest_type) != nir_type_float) { + a9_tex_refuse(ctx, "an integer-texture sample or load in a compute shader"); + return false; + } + for (unsigned i = 0; i < tex->num_srcs; i++) { + switch (tex->src[i].src_type) { + case nir_tex_src_coord: case nir_tex_src_lod: case nir_tex_src_ddx: case nir_tex_src_ddy: + case nir_tex_src_texture_deref: case nir_tex_src_sampler_deref: + break; + default: + a9_tex_refuse(ctx, "a texel offset or other extra texture source in a compute shader"); + return false; + } + } + if (ci < 0 || !a9_tex_var(tex, nir_tex_src_texture_deref, &set, &binding, &element)) { + a9_tex_refuse(ctx, "a compute sample without a constant texture binding"); + return false; + } + if (read) { + sset = sbinding = selement = 0; + } else if (!a9_tex_var(tex, nir_tex_src_sampler_deref, &sset, &sbinding, &selement)) { + /* a combined image sampler: its sampler words are at +32 of the same descriptor */ + sset = set, sbinding = binding, selement = element, combined = 1; + } + const int image = a9_tex_upload(ctx, G9P_A9_UP_IMAGE, set, binding, element, 0, 0); + const int sampler = image < 0 ? -1 : + read ? a9_tex_upload(ctx, G9P_A9_UP_FETCH_SAMPLER, 0, 0, 0, 0, 0) : + a9_tex_upload(ctx, G9P_A9_UP_SAMPLER, sset, sbinding, selement, combined, 0); + if (sampler < 0) + return false; + + /* coordinates: a fresh block, consumed by the sample */ + const unsigned coords = layered ? 3 : 2; + const int block = alloc_range(ctx, coords); + if (block < 0 || block + coords > 32) + return false; + ctx->temps |= BITFIELD64_RANGE(block, coords); + for (unsigned c = 0; c < coords; c++) + if (!a9_tex_copy(ctx, block + c, tex->src[ci].src.ssa, c)) + return false; + + /* level (f16 for samples, u32 for reads) or gradients */ + int lod; + const int li = nir_tex_instr_src_index(tex, nir_tex_src_lod); + const bool register_lod = layered && li >= 0 && !nir_src_is_const(tex->src[li].src); + if (register_lod) { + /* Apple's A9 witnesses sla_d/slc_d convert the level to the low f16 of + * a register with the source in slot A, its last read marked; the + * older plain-2D g9p_f16_of() reads slot B; leave that proven path + * alone. Copy into a private low temporary: the witness marks the + * source's last read, so it cannot be a still-live NIR value. */ + int src = src_reg(ctx, nir_get_scalar(tex->src[li].src.ssa, 0)); + if (src < 0) return false; + int copy = alloc_low_temp(ctx); + if (copy < 0) return false; + g9p_alu2(ctx->c, G9P_OR, copy, src, src, false); + lod = read ? copy : alloc_low_temp(ctx); + if (lod < 0) return false; + if (!read) { + const unsigned at = ctx->c->n; + g9p_f16_of(ctx->c, lod, 0); + (*g9p_code_word(ctx->c, at + 2)) |= copy << G9P_SRC_A_SHIFT | G9P_SRC_A_LAST_READ; + } + /* ra_d loads its uint LOD directly into r0: no f16 conversion. */ + } else if (layered) { + lod = a9_tex_shared_lod(ctx, tex, read); + if (lod < 0) return false; + } else if (grad) { + const int dxi = nir_tex_instr_src_index(tex, nir_tex_src_ddx); + const int dyi = nir_tex_instr_src_index(tex, nir_tex_src_ddy); + lod = alloc_range(ctx, 4); + if (dxi < 0 || dyi < 0 || lod < 0 || lod > 28) + return false; + ctx->temps |= BITFIELD64_RANGE(lod, 4); + for (unsigned c = 0; c < 2; c++) { + if (!a9_tex_copy(ctx, lod + 2 * c, tex->src[dxi].src.ssa, c) || + !a9_tex_copy(ctx, lod + 2 * c + 1, tex->src[dyi].src.ssa, c)) + return false; + } + } else { + const int li = nir_tex_instr_src_index(tex, nir_tex_src_lod); + int r = li >= 0 ? src_reg(ctx, nir_get_scalar(tex->src[li].src.ssa, 0)) : const_reg(ctx, 0); + if (r < 0) + return false; + if (read) { + lod = alloc_temp(ctx); + if (lod < 0 || lod > 31) + return false; + g9p_alu2(ctx->c, G9P_OR, lod, r, r, false); + } else { + r = one_src(ctx, r); + lod = r < 0 ? r : alloc_low_temp(ctx); + if (lod < 0) + return false; + g9p_f16_of(ctx->c, lod, r); + } + } + if (cube) { + /* slc_f's cube coordinates (g9p_cube_coords()): bank0 r0..2 directions -> bank1 coords. + * Bank0 homes start at r32, so this scratch cannot overwrite them. */ + for (unsigned c = 0; c < 3; c++) + g9p_mov_to_high(ctx->c, c, block + c); + g9p_cube_coords(ctx->c, 2 * block, 0); + } + release(ctx, &tex->instr); + const int dst = alloc_range(ctx, 4); + if (dst < 0 || dst > 28) + return false; + + if (register_lod) { + /* Apple's A9 register-LOD array or cube sample (slc_d/sla_d) and fetch + * (ra_d, coordinate r3; its preceding ops produce x,y,layer in r3,r4,r5): + * g5p_usc_sample_register_lod(), the coupled temporary-level tuple + * with the LOD register. Keep this local to these witnessed compute + * cases; g5p_usc_sample_ext() and all graphics encodings are unchanged. */ + const struct g5p_tex_ext x = { + .dst_half = 2 * dst, .image_qword = image / 2, .sampler_qword = sampler / 2, .coord_dword = block, + .dim = cube ? G5P_TEX_CUBE : G5P_TEX_2D_ARRAY, .mask = 0xf, .read = read, + .lod = G5P_TEX_LOD_LEVEL, .lod_dword = lod, .tuple = G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL, + }; + uint8_t raw[G5P_USC_MAX]; + const unsigned n = g5p_usc_sample_register_lod(raw, &x); + if (!n) { + a9_tex_refuse(ctx, "the register-LOD sample was refused by the encoder"); + return false; + } + a9_tex_emit_raw(ctx, raw, n); + } else { + struct g5p_tex_ext x = { + .dst_half = 2 * dst, .image_qword = image / 2, .sampler_qword = sampler / 2, + .coord_dword = block, .dim = cube ? G5P_TEX_CUBE : array ? G5P_TEX_2D_ARRAY : G5P_TEX_2D, .mask = 0xf, .read = read, + .lod = grad ? G5P_TEX_LOD_GRAD : G5P_TEX_LOD_LEVEL, .lod_dword = lod, + .lod_shared = layered, + .tuple = layered ? G5P_TEX_TUPLE_COUPLED_SHARED_LEVEL : + grad ? G5P_TEX_TUPLE_COUPLED_GRAD : G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL, + }; + uint8_t raw[G5P_USC_MAX]; + const unsigned n = g5p_usc_sample_ext(raw, &x); + if (!n) { + a9_tex_refuse(ctx, "the compute sample recipe was refused by the encoder"); + return false; + } + a9_tex_emit_raw(ctx, raw, n); + } + g9p_wait(ctx->c); + for (unsigned c = 0; c < 4; c++) { + ctx->reg[key(&tex->def, c)] = dst + c; + if (c >= tex->def.num_components || !ctx->last_use[key(&tex->def, c)]) + ctx->busy &= ~BITFIELD64_BIT(dst + c); + } + return true; +} +#define G9P_A9_TEX(ctx, tex) \ + do { if ((ctx)->out->a9 && (ctx)->stage == MESA_SHADER_COMPUTE) return a9_tex(ctx, tex); } while (0) diff --git a/src/imagination/g9p/g9p_cdm.h b/src/imagination/g9p/g9p_cdm.h index 09776d96a33..97d2d6b7c88 100644 --- a/src/imagination/g9p/g9p_cdm.h +++ b/src/imagination/g9p/g9p_cdm.h @@ -78,6 +78,13 @@ static inline uint32_t g9p_pds_bra_not_if1(uint32_t target) * COMPUTE.md "Register allocation"): field 2n holds 8n registers (field 1-2 * covers r0-r8, 3-4 r0-r16, 5-6 r0-r24, 8 r0-r32). Bit 8 observed set on * every iPadOS kernel record, meaning open. iPadOS's 0x2100 is field 2. + * + * On A9 this field allocates register bank 1 only (destination W1 bit 9 set: the bank G9P's compute code + * uses), and the DOUTU temps allocate bank 0. AGXMetalA9's ComputeProgramVariant (0x207d2c4b0) builds the + * record's word 0 (+0x9b8) from __GPU_METADATA r.3.0, which is 8 in all 164 examined A9 kernels + * (field 2, iPadOS's 0x2100), while EncodeComputePDSProgram (0x207d1ea4c) takes + * the DOUTU temps from r.0.0 (8..64, bank-0 values to r60). Fields 12 and 18 (48 and + * 72 registers, tested with bank-0 register parking) lost the GPU before any store (no CDM completion), so an A9 kernel passes its bank-1 count (at most 32: field 8, the largest measured). */ #define G9P_CDM_KERNEL_REGS(n) (0x100u | (2u * (((n) + 7) / 8)) << 12) #define G9P_CDM_KERNEL_SIZES G9P_CDM_KERNEL_REGS(8) diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index 322ea8c9ff1..2513097ac6c 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -18,6 +18,9 @@ #include #include #include +#include "c11/threads.h" +#include +#include "util/log.h" #include #include @@ -40,6 +43,45 @@ #include "util/format/u_format.h" #include "g9p_compile.h" #include "g9p_isa.h" +#include "util/u_debug.h" +#include "util/log.h" + +DEBUG_GET_ONCE_BOOL_OPTION(g9p_pressure, "G9P_PRESSURE", false) + +/* Pipeline compilation may run on several threads. Keep the fallback for + * the current compile, never a different pipeline's last diagnostic. */ +static thread_local char a9_last_refusal[192]; +const char *g9p_a9_last_refusal(void) +{ + return a9_last_refusal[0] ? a9_last_refusal : "A9 backend failed without a diagnostic"; +} + +static bool +g9p_fail(struct g9p_shader *out, const char *fmt, ...) +{ + char why[192]; + va_list ap; + va_start(ap, fmt); + vsnprintf(why, sizeof(why), fmt, ap); + va_end(ap); + if (out->a9) { + if (!out->a9_refusal[0]) + snprintf(out->a9_refusal, sizeof(out->a9_refusal), "%s", why); + snprintf(a9_last_refusal, sizeof(a9_last_refusal), "%s", out->a9_refusal); + } + if (!out->a9) { + /* Preserve main's non-A9 diagnostics. */ + if (!strcmp(why, "shader has no entry point")) + return false; + if (!strncmp(why, "too many loops", 14)) + snprintf(why, sizeof(why), "too many loops"); + char *suffix = strstr(why, " (no parking victim)"); + if (!suffix) suffix = strstr(why, " (bank-0 homes exhausted)"); + if (suffix) *suffix = 0; + fprintf(stderr, "g9p: %s\n", why); + } + return false; +} /* r0-r29 for values, the mask stack after them (G9P_REGS: debug override) */ static unsigned @@ -50,13 +92,56 @@ g9p_num_regs(void) return e ? MIN2(atoi(e), 63) : 30; } #define G9P_NUM_REGS g9p_num_regs() + +/* + * A9 compute: the bank-1 registers per lane (values and the mask stack: the kernel record's register field) + * that a workgroup of @threads lanes may use. Apple's limit for a bank-1 allocation of f registers is + * 393216 / ((64f + 448) & ~511) lanes (AGXMetalA9 ComputeProgramVariant 0x207d2c4b0 -> +0xb08; its bank-0 + * term reserves one task's allocation). Apple's kernels always ask for 8 there, and put the rest in bank 0. + * Ours took up to 32: at 256 lanes and 24 (field 6), one group fills the whole bank-1 capacity, and 64 such + * groups lost the GPU before their stores; one group alone, or 16 registers, ran successfully. + * So a group keeps one task (32 lanes) of that capacity free, as Apple's bank-0 + * term does: 16 at 256 lanes, 24 up to 224, 32 up to 160. What does not fit goes to bank 0 (homes). + */ +/* + * A9 compute: the homes (bank-0 registers above r31) a workgroup of @threads lanes may use. Apple's bank-0 + * limit for t DOUTU temps is 32 * floor((3329 - a) / a) lanes, a = (4t + 28) & ~31 (the same +0xb08 value); + * the temps are 32 + homes rounded up to 8. Apple's A9 allocator declares up to 240 temps at 64 lanes without + * spilling (loop144 240, p120 reads r238). Homes run r32..r239, fewer as the workgroup grows (256 lanes: 56). + * A kernel that needs more is refused by name here, instead of at pipeline creation. + */ +static unsigned +g9p_a9_max_homes(unsigned threads) +{ + unsigned homes = 208; + while (homes > 0) { + const unsigned t = 8 * DIV_ROUND_UP(32 + homes, 8), a = (4 * t + 28) & ~31u; + if (32 * ((3329 - a) / a) >= threads) + break; + homes -= 8; + } + return homes; +} + +static unsigned +g9p_a9_bank1_limit(unsigned threads) +{ + for (unsigned f = 32; f > 8; f -= 8) { + if (393216 / ((64 * f + 448) & ~511u) >= threads + 32) + return f; + } + return 8; +} /* * Registers for values: vertex shaders keep the mask stack in r15 (compile_back()), so * r30-r31 hold values too there (G9P_REGS still overrides, for tests) */ #define VALUE_REGS(ctx) ((ctx)->stage == MESA_SHADER_VERTEX && (ctx)->st == 15 && !getenv("G9P_REGS") \ - ? 32u : G9P_NUM_REGS) -#define G9P_MAX_LOOPS 64 + ? 32u : (ctx)->value_cap ? MIN2(G9P_NUM_REGS, (ctx)->value_cap) : G9P_NUM_REGS) +#define LOW_REGS(ctx) MIN2(16u, VALUE_REGS(ctx)) +/* The immediate source of a break's stack word is six bits (W2[6:1]); + * bit 7 is last-read, bit 8 continuation. It encodes if_depth + 3. */ +#define G9P_MAX_IF_DEPTH (63 - 3) /* nir_tex_instr.backend_flags: a seamful cube's gradients, for the hardware */ #define G9P_TEX_NATIVE_GRADIENTS 1 @@ -88,6 +173,50 @@ static const nir_shader_compiler_options nir_options = { .lower_fround_even = true, .lower_flrp32 = true, .lower_fsat = true, + .lower_bit_count = true, + .lower_bitfield_reverse = true, + .lower_find_lsb = true, + .lower_ifind_msb = true, + .lower_iadd_sat = true, + .lower_mul_2x32_64 = true, + .lower_device_index_to_zero = true, + .lower_bitfield_insert = true, + .lower_bitfield_extract = true, + .lower_pack_unorm_4x8 = true, + .lower_unpack_unorm_4x8 = true, + .lower_pack_snorm_4x8 = true, + .lower_unpack_snorm_4x8 = true, + .lower_pack_snorm_2x16 = true, + .lower_unpack_snorm_2x16 = true, + .lower_pack_unorm_2x16 = true, + .lower_unpack_unorm_2x16 = true, + .lower_extract_byte = true, + .lower_extract_word = true, + .lower_insert_byte = true, + .lower_insert_word = true, + .max_unroll_iterations = 0, +}; + +/* A9 compute options. */ +static const nir_shader_compiler_options nir_options_a9 = { + /* multiply-adds as one instruction (g9p_ffma()), not a mul and an add */ + .float_mul_add32 = nir_float_muladd_support_has_ffma | nir_float_muladd_support_fuse, + .lower_int64_options = ~0U, + .lower_fdiv = true, + .lower_iabs = true, + .lower_mul_high = true, + .lower_isign = true, + .lower_fmod = true, + .lower_fpow = true, + .lower_fsqrt = true, + .lower_ffract = true, + .lower_fceil = true, + .lower_ftrunc = true, + .lower_fround_even = true, + .lower_flrp32 = true, + .lower_fsat = true, + .lower_bit_count = true, + .lower_bitfield_reverse = true, .lower_find_lsb = true, .lower_ifind_msb = true, .lower_iadd_sat = true, @@ -95,6 +224,10 @@ static const nir_shader_compiler_options nir_options = { .lower_device_index_to_zero = true, .lower_bitfield_insert = true, .lower_bitfield_extract = true, + .lower_pack_unorm_4x8 = true, + .lower_unpack_unorm_4x8 = true, + .lower_pack_snorm_4x8 = true, + .lower_unpack_snorm_4x8 = true, .lower_pack_snorm_2x16 = true, .lower_unpack_snorm_2x16 = true, .lower_pack_unorm_2x16 = true, @@ -125,6 +258,8 @@ struct g9p_vs_store { }; struct g9p_ctx { + /* A9 compute: at most this many value registers in bank 1 (0: no cap; g9p_a9_bank1_limit()) */ + unsigned value_cap; /* a sample iterated straight to the sampler has been emitted */ bool straight_done; /* @@ -177,13 +312,19 @@ struct g9p_ctx { /* The decl_reg defs */ nir_def **decls; unsigned num_decls; + bool pressure; + /* Keys grouped by last use, and ownership released in this generation. */ + unsigned *dies_at, *dies_keys; + uint8_t *released, *is_decl; + unsigned emit_loop_index; /* * Fragment NIR registers parked above r31 (per decl_reg key, -1: none), * chosen across generate() attempts when registers run out */ - int8_t *home; + int16_t *home; unsigned num_homes; + bool oom_low; /* the failed allocation needs r0-r15 */ bool oom; /* the last generate() ran out of registers */ unsigned oom_ip; @@ -215,7 +356,7 @@ struct g9p_ctx { struct { unsigned start, end; - } loops[G9P_MAX_LOOPS]; + } *loops; unsigned num_loops, loop_index; /* Jumps to the innermost loop's exit (after a break), patched at its end */ @@ -223,6 +364,28 @@ struct g9p_ctx { unsigned num_exit_jumps, max_exit_jumps; }; +/* Every compile_back failure reaches this helper, including control flow + * failures outside emit_instr(). An instruction's precise reason wins. */ +static bool +backend_failure(struct g9p_shader *out, const struct g9p_ctx *ctx) +{ + if (ctx->c && debug_get_option_g9p_pressure()) + mesa_logi("g9p: failed attempt reached %u bytes of code, %u loop records, instruction %u\n", + 2 * ctx->c->n, ctx->num_loops, ctx->ip); + if (ctx->c && ctx->c->error) + return g9p_fail(out, "%s", ctx->c->error); + if (out->a9) { + if (out->a9_refusal[0]) { + char why[sizeof(out->a9_refusal)]; + memcpy(why, out->a9_refusal, sizeof(why)); + return g9p_fail(out, "%s", why); + } + return ctx->oom ? g9p_fail(out, "out of registers at instruction %u", ctx->oom_ip) : + g9p_fail(out, "backend control flow or register allocation failed at instruction %u", ctx->ip); + } + return false; +} + static unsigned key(const nir_def *def, unsigned comp) { @@ -258,9 +421,9 @@ helper_samples_end(struct g9p_ctx *ctx) const unsigned at = ctx->frag_samples[i]; /* Apple's word pair for a sample before the last: W3 0600, W4 044d */ - if (at != last && ctx->c->w[at + 4] == 0x0200) { - ctx->c->w[at + 3] &= ~0x0100; - ctx->c->w[at + 4] = 0x044d; + if (at != last && (*g9p_code_word(ctx->c, at + 4)) == 0x0200) { + (*g9p_code_word(ctx->c, at + 3)) &= ~0x0100; + (*g9p_code_word(ctx->c, at + 4)) = 0x044d; } } } @@ -340,6 +503,7 @@ alloc_reg(struct g9p_ctx *ctx) flush_loads(ctx); return alloc_reg(ctx); } + ctx->oom_low = false; dump_busy(ctx); return -1; } @@ -375,7 +539,7 @@ reads_def(nir_src *src, void *data) static bool evict_low(struct g9p_ctx *ctx, unsigned lo, unsigned n) { - uint64_t high_free = ~ctx->busy & BITFIELD64_RANGE(16, VALUE_REGS(ctx) - 16); + uint64_t high_free = VALUE_REGS(ctx) > 16 ? ~ctx->busy & BITFIELD64_RANGE(16, VALUE_REGS(ctx) - 16) : 0; int keys[16]; if (ctx->c->fragment || lo + n > 16 || getenv("G9P_NO_EVICT")) @@ -419,7 +583,7 @@ evict_low(struct g9p_ctx *ctx, unsigned lo, unsigned n) return false; high_free &= high_free - 1; } - high_free = ~ctx->busy & BITFIELD64_RANGE(16, VALUE_REGS(ctx) - 16); + high_free = VALUE_REGS(ctx) > 16 ? ~ctx->busy & BITFIELD64_RANGE(16, VALUE_REGS(ctx) - 16) : 0; for (unsigned r = lo; r < lo + n; r++) { if (keys[r - lo] < 0) continue; @@ -441,7 +605,9 @@ evict_low(struct g9p_ctx *ctx, unsigned lo, unsigned n) static int evict_low_window(struct g9p_ctx *ctx, unsigned n) { - for (int r = 16 - n; r >= 0; r--) { + if (n > LOW_REGS(ctx)) + return -1; + for (int r = LOW_REGS(ctx) - n; r >= 0; r--) { if (evict_low(ctx, r, n)) return r; } @@ -452,7 +618,7 @@ evict_low_window(struct g9p_ctx *ctx, unsigned n) static int alloc_low_temp(struct g9p_ctx *ctx) { - for (unsigned r = 0; r < MIN2(16, G9P_NUM_REGS); r++) { + for (unsigned r = 0; r < LOW_REGS(ctx); r++) { if (!(ctx->busy & BITFIELD64_BIT(r))) { ctx->busy |= BITFIELD64_BIT(r); ctx->temps |= BITFIELD64_BIT(r); @@ -474,6 +640,7 @@ alloc_low_temp(struct g9p_ctx *ctx) return r; } } + ctx->oom_low = true; dump_busy(ctx); return -1; } @@ -482,7 +649,7 @@ alloc_low_temp(struct g9p_ctx *ctx) static int alloc_low_range(struct g9p_ctx *ctx, unsigned n) { - for (unsigned r = 0; r + n <= MIN2(16, G9P_NUM_REGS); r++) { + for (unsigned r = 0; r + n <= LOW_REGS(ctx); r++) { if (!(ctx->busy & BITFIELD64_RANGE(r, n))) { ctx->busy |= BITFIELD64_RANGE(r, n); ctx->max_reg = MAX2(ctx->max_reg, r + n - 1); @@ -502,6 +669,7 @@ alloc_low_range(struct g9p_ctx *ctx, unsigned n) return r; } } + ctx->oom_low = true; dump_busy(ctx); return -1; } @@ -538,7 +706,7 @@ one_src(struct g9p_ctx *ctx, int r) static int alloc_pair(struct g9p_ctx *ctx) { - for (unsigned r = 0; r + 1 < MIN2(16, G9P_NUM_REGS); r++) { + for (unsigned r = 0; r + 1 < LOW_REGS(ctx); r++) { if (!(ctx->busy & BITFIELD64_RANGE(r, 2))) { ctx->busy |= BITFIELD64_RANGE(r, 2); ctx->max_reg = MAX2(ctx->max_reg, r + 1); @@ -558,6 +726,7 @@ alloc_pair(struct g9p_ctx *ctx) return r; } } + ctx->oom_low = true; dump_busy(ctx); return -1; } @@ -573,6 +742,7 @@ alloc_range(struct g9p_ctx *ctx, unsigned n) return r; } } + ctx->oom_low = false; dump_busy(ctx); return -1; } @@ -675,6 +845,8 @@ release_scalar(struct g9p_ctx *ctx, nir_def *def, unsigned comp) if (ctx->last_use[k] > ctx->ip || ctx->reg[k] < 0) return; + if (ctx->released) + ctx->released[k] = 1; /* a NIR register's last use: unpinned and free */ if (nir_def_is_intrinsic(def) && nir_def_as_intrinsic(def)->intrinsic == nir_intrinsic_decl_reg) { @@ -936,12 +1108,11 @@ walk_liveness(struct g9p_ctx *ctx, struct exec_list *list, bool uses) nir_loop *loop = nir_cf_node_as_loop(node); unsigned l = ctx->loop_index++; - if (!uses && l < G9P_MAX_LOOPS) { + if (!uses && l < ctx->num_loops) { ctx->loops[l].start = ctx->ip + 1; - ctx->num_loops = MAX2(ctx->num_loops, l + 1); } walk_liveness(ctx, &loop->body, uses); - if (!uses && l < G9P_MAX_LOOPS) + if (!uses && l < ctx->num_loops) ctx->loops[l].end = ctx->ip; break; } @@ -951,6 +1122,177 @@ walk_liveness(struct g9p_ctx *ctx, struct exec_list *list, bool uses) } } +/* Register ranges from CFG liveness, intersected with reaching writes. SSA + * ranges stay as recorded by walk_liveness(). */ +struct reg_block_range { unsigned first, last; }; + +static void +number_reg_blocks(struct exec_list *list, unsigned *ip, struct reg_block_range *blk, unsigned *instr_ip) +{ + foreach_list_typed(nir_cf_node, node, node, list) { + switch (node->type) { + case nir_cf_node_block: { + nir_block *b = nir_cf_node_as_block(node); + blk[b->index].first = *ip + 1; + nir_foreach_instr(instr, b) { + (*ip)++; + instr_ip[instr->index] = *ip; + } + blk[b->index].last = *ip; + if (blk[b->index].first > blk[b->index].last) + blk[b->index].first = blk[b->index].last; /* empty block */ + break; + } + case nir_cf_node_if: { + nir_if *nif = nir_cf_node_as_if(node); + (*ip)++; + number_reg_blocks(&nif->then_list, ip, blk, instr_ip); + number_reg_blocks(&nif->else_list, ip, blk, instr_ip); + break; + } + case nir_cf_node_loop: + number_reg_blocks(&nir_cf_node_as_loop(node)->body, ip, blk, instr_ip); + break; + default: + break; + } + } +} + +static void +reg_liveness(struct g9p_ctx *ctx, nir_function_impl *impl, void *mem_ctx) +{ + const unsigned nd = ctx->num_decls, words = BITSET_WORDS(MAX2(nd, 1)); + unsigned *decl_of = rzalloc_array(mem_ctx, unsigned, impl->ssa_alloc); /* decl def index -> d + 1 */ + + for (unsigned d = 0; d < nd; d++) + decl_of[ctx->decls[d]->index] = d + 1; + nir_metadata_require(impl, nir_metadata_block_index); + const unsigned ni = nir_index_instrs(impl); + const unsigned nb = impl->num_blocks; + struct reg_block_range *blk = rzalloc_array(mem_ctx, struct reg_block_range, nb); + unsigned *instr_ip = rzalloc_array(mem_ctx, unsigned, MAX2(ni, 1)); /* by instr index */ + BITSET_WORD *gen = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + BITSET_WORD *kill = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + BITSET_WORD *in = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + BITSET_WORD *out = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + unsigned ip = 0; + + number_reg_blocks(&impl->body, &ip, blk, instr_ip); + nir_foreach_block(b, impl) { + BITSET_WORD *g = gen + b->index * words, *k = kill + b->index * words; + nir_foreach_instr(instr, b) { + if (instr->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + if (intr->intrinsic == nir_intrinsic_load_reg) { + const unsigned d = decl_of[intr->src[0].ssa->index] - 1; + if (!BITSET_TEST(k, d)) + BITSET_SET(g, d); + } else if (intr->intrinsic == nir_intrinsic_store_reg) { + BITSET_SET(k, decl_of[intr->src[1].ssa->index] - 1); + } + } + } + for (bool progress = true; progress;) { + progress = false; + nir_foreach_block_reverse(b, impl) { + BITSET_WORD *o = out + b->index * words, *i = in + b->index * words; + for (unsigned s = 0; s < 2; s++) { + if (!b->successors[s] || b->successors[s] == impl->end_block) + continue; + const BITSET_WORD *si = in + b->successors[s]->index * words; + for (unsigned w = 0; w < words; w++) + o[w] |= si[w]; + } + for (unsigned w = 0; w < words; w++) { + const BITSET_WORD n = gen[b->index * words + w] | (o[w] & ~kill[b->index * words + w]); + if (n != i[w]) { + i[w] = n; + progress = true; + } + } + } + } + /* forward: a write reaches the block (an undefined value needs no register: uninitialised paths) */ + BITSET_WORD *din = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + BITSET_WORD *dout = rzalloc_array(mem_ctx, BITSET_WORD, nb * words); + for (bool progress = true; progress;) { + progress = false; + nir_foreach_block(b, impl) { + BITSET_WORD *di = din + b->index * words, *dd = dout + b->index * words; + nir_foreach_pred(p, b) { + if (p->index >= nb) + continue; + for (unsigned w = 0; w < words; w++) + di[w] |= dout[p->index * words + w]; + } + for (unsigned w = 0; w < words; w++) { + const BITSET_WORD n = di[w] | kill[b->index * words + w]; + if (n != dd[w]) { + dd[w] = n; + progress = true; + } + } + } + } + for (unsigned i = 0; i < nb * words; i++) { + in[i] &= din[i]; + out[i] &= dout[i]; + } + unsigned *lo = ralloc_array(mem_ctx, unsigned, MAX2(nd, 1)), *hi = rzalloc_array(mem_ctx, unsigned, MAX2(nd, 1)); + memset(lo, 0xff, MAX2(nd, 1) * sizeof(unsigned)); +#define REG_SPAN(d, a, z) do { lo[d] = MIN2(lo[d], (a)); hi[d] = MAX2(hi[d], (z)); } while (0) + nir_foreach_block(b, impl) { + for (unsigned d = 0; d < nd; d++) { + if (BITSET_TEST(in + b->index * words, d)) + REG_SPAN(d, blk[b->index].first, blk[b->index].first); + if (BITSET_TEST(out + b->index * words, d)) + REG_SPAN(d, blk[b->index].last, blk[b->index].last); + } + nir_foreach_instr(instr, b) { + if (instr->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + const unsigned at = instr_ip[instr->index]; + if (intr->intrinsic == nir_intrinsic_load_reg) { + const unsigned d = decl_of[intr->src[0].ssa->index] - 1; + REG_SPAN(d, at, MAX2(at, ctx->last_use[key(&intr->def, 0)])); + } else if (intr->intrinsic == nir_intrinsic_store_reg) { + const unsigned d = decl_of[intr->src[1].ssa->index] - 1; + REG_SPAN(d, at, at); + } + } + } + /* in+out over a loop: a register live into a block of a loop is live over the whole block range + * between; the hull covers it. Replace the walk's stretched ranges. */ + for (unsigned d = 0; d < nd; d++) { + const unsigned k = key(ctx->decls[d], 0); + if (lo[d] == UINT_MAX) + continue; + ctx->reg_first[ctx->decls[d]->index] = lo[d]; + ctx->last_use[k] = hi[d]; + } +} +#undef REG_SPAN + +/* Exactly the loop records visited by walk_liveness(), including nested + * loops in either arm. Allocate before its two traversals. */ +static unsigned +count_loop_records(struct exec_list *list) +{ + unsigned count = 0; + foreach_list_typed(nir_cf_node, node, node, list) { + if (node->type == nir_cf_node_loop) { + count += 1 + count_loop_records(&nir_cf_node_as_loop(node)->body); + } else if (node->type == nir_cf_node_if) { + nir_if *nif = nir_cf_node_as_if(node); + count += count_loop_records(&nif->then_list) + count_loop_records(&nif->else_list); + } + } + return count; +} + /* --- Code ---------------------------------------------------------------- */ /* The common-store slot of the storage buffer @index refers to (@flags on the binding) */ @@ -1120,6 +1462,7 @@ emit_alu(struct g9p_ctx *ctx, nir_alu_instr *alu) } for (unsigned i = 0; i < num_inputs; i++) s[i] = nir_get_scalar(alu->src[i].src.ssa, alu->src[i].swizzle[0]); + G9P_A9_ALU_PACK_HALF(); G9P_A9_ALU_LITERAL_FMA(); /* compared where it is used (emit_branch_cmp()) */ if (branch_compare(&alu->def)) @@ -2207,6 +2550,7 @@ emit_fb_fetch(struct g9p_ctx *ctx, nir_intrinsic_instr *intr) } } if (pair < 0) { + ctx->oom_low = false; dump_busy(ctx); return false; } @@ -2474,9 +2818,9 @@ sample_lod(struct g9p_ctx *ctx, nir_tex_instr *tex, unsigned at, int lod) { if (lod < 0) return; - ctx->c->w[at + 3] = (ctx->c->w[at + 3] & 0xff00) | 0x80 | + (*g9p_code_word(ctx->c, at + 3)) = ((*g9p_code_word(ctx->c, at + 3)) & 0xff00) | 0x80 | (tex->op == nir_texop_txl ? 0x40 : 0x20) | lod; - ctx->c->w[at + 4] = (ctx->c->w[at + 4] & 0xff00) | 0x80; + (*g9p_code_word(ctx->c, at + 4)) = ((*g9p_code_word(ctx->c, at + 4)) & 0xff00) | 0x80; } /* @@ -2620,11 +2964,11 @@ extras_patch(struct g9p_ctx *ctx, unsigned at, const struct tex_extras *e) { if (e->half < 0) return; - ctx->c->w[at + 4] |= 0x40 | e->half; + (*g9p_code_word(ctx->c, at + 4)) |= 0x40 | e->half; if (e->compare) - ctx->c->w[at + 3] |= 0x8000; + (*g9p_code_word(ctx->c, at + 3)) |= 0x8000; if (e->offset) - ctx->c->w[at + 2] |= 0x4000; + (*g9p_code_word(ctx->c, at + 2)) |= 0x4000; } /* @@ -2682,7 +3026,7 @@ emit_tex_layered(struct g9p_ctx *ctx, nir_tex_instr *tex) return false; at = ctx->c->n; g9p_sample_regs(ctx->c, dst, tex_state_dword(ctx, slot), tex_sampler_dword(ctx, slot)); - ctx->c->w[at] = (ctx->c->w[at] & ~0x0f00) | + (*g9p_code_word(ctx->c, at)) = ((*g9p_code_word(ctx->c, at)) & ~0x0f00) | (cube ? G9P_TEX_CUBE : is3d ? G9P_TEX_3D : G9P_TEX_2D_ARRAY) << 8; sample_lod(ctx, tex, at, lod); extras_patch(ctx, at, &extras); @@ -2798,18 +3142,18 @@ emit_tex_vertex(struct g9p_ctx *ctx, nir_tex_instr *tex) g9p_sample_vertex(ctx->c, dst, tex_state_dword(ctx, slot), tex_sampler_dword(ctx, slot), coord, lod); if (cube) { - ctx->c->w[at] = (ctx->c->w[at] & ~0x0f00) | G9P_TEX_CUBE << 8; - ctx->c->w[at + 2] |= 0x8000; + (*g9p_code_word(ctx->c, at)) = ((*g9p_code_word(ctx->c, at)) & ~0x0f00) | G9P_TEX_CUBE << 8; + (*g9p_code_word(ctx->c, at + 2)) |= 0x8000; } else if (is3d) { - ctx->c->w[at] = (ctx->c->w[at] & ~0x0f00) | G9P_TEX_3D << 8; + (*g9p_code_word(ctx->c, at)) = ((*g9p_code_word(ctx->c, at)) & ~0x0f00) | G9P_TEX_3D << 8; } else if (tex->is_array) { - ctx->c->w[at] = (ctx->c->w[at] & ~0x0f00) | G9P_TEX_2D_ARRAY << 8; - ctx->c->w[at + 2] |= 0x8000; + (*g9p_code_word(ctx->c, at)) = ((*g9p_code_word(ctx->c, at)) & ~0x0f00) | G9P_TEX_2D_ARRAY << 8; + (*g9p_code_word(ctx->c, at + 2)) |= 0x8000; } extras_patch(ctx, at, &extras); /* the vertex form's compare also sets bit 15 of W6 (Apple's sample_compare at a level) */ if (extras.compare) - ctx->c->w[at + 6] |= 0x8000; + (*g9p_code_word(ctx->c, at + 6)) |= 0x8000; g9p_wait(ctx->c); release(ctx, &tex->instr); for (unsigned c = 0; c < 4; c++) { @@ -2827,9 +3171,11 @@ emit_tex_vertex(struct g9p_ctx *ctx, nir_tex_instr *tex) * where the sample reads them (as Apple's code for t.sample(s, uv * 2.0 + * 0.25) does; r0-r1 are free until the colour output). */ +#include "g9p_a9_tex.h" static bool emit_tex(struct g9p_ctx *ctx, nir_tex_instr *tex) { + G9P_A9_TEX(ctx, tex); int ci = nir_tex_instr_src_index(tex, nir_tex_src_coord); int slot = texture_slot(ctx, tex); int first, second, dst; @@ -2930,7 +3276,11 @@ emit_intrinsic(struct g9p_ctx *ctx, nir_intrinsic_instr *intr) r = alloc_reg(ctx); if (r < 0) return false; - g9p_tg_load(ctx->c, r, c); + /* A9: z is in its own word after the barrier counter (g9p_a9.h G9P_A9_TG_WGID_Z) */ + const bool a9_z = ctx->out->a9 && c == 2; + g9p_tg_load(ctx->c, r, a9_z ? G9P_A9_TG_WGID_Z : c); + if (a9_z) + ctx->out->a9_wgid_z = true; ctx->reg[key(&intr->def, c)] = r; } g9p_wait(ctx->c); @@ -3480,8 +3830,13 @@ emit_break(struct g9p_ctx *ctx, bool when_true) } g9p_loop_break(ctx->c, ctx->st, when_true); if (ctx->num_exit_jumps == ctx->max_exit_jumps) { - fprintf(stderr, "g9p: too many breaks\n"); - return false; + unsigned capacity = ctx->max_exit_jumps * 2; + unsigned *jumps = capacity > ctx->max_exit_jumps ? + reralloc(ctx->c, ctx->exit_jumps, unsigned, capacity) : NULL; + if (!jumps) + return g9p_fail(ctx->out, "allocation failure growing loop exits"); + ctx->exit_jumps = jumps; + ctx->max_exit_jumps = capacity; } ctx->exit_jumps[ctx->num_exit_jumps++] = g9p_jump(ctx->c, G9P_JMP_NONE, 0); return true; @@ -3497,6 +3852,8 @@ emit_if_else(struct g9p_ctx *ctx, nir_if *nif, bool sense) { unsigned to_else, to_pop; + if (ctx->if_depth >= G9P_MAX_IF_DEPTH) + return g9p_fail(ctx->out, "control flow nested deeper than %u", G9P_MAX_IF_DEPTH); /* keep the lanes for which the condition holds */ g9p_push_if(ctx->c, ctx->st, !sense); ctx->if_depth++; @@ -3564,10 +3921,33 @@ emit_if(struct g9p_ctx *ctx, nir_if *nif) return true; } +/* A stretched range may end without an instruction reading its key. */ +static void +expire_keys(struct g9p_ctx *ctx, unsigned end, bool decls_only) +{ + if (!ctx->pressure) + return; + for (unsigned i = ctx->dies_at[end]; i < ctx->dies_at[end + 1]; i++) { + const unsigned k = ctx->dies_keys[i]; + const int r = ctx->reg[k]; + if (r < 0 || ctx->released[k] || + (decls_only && (!ctx->is_decl[k] || r >= 32))) + continue; + ctx->released[k] = 1; + if (ctx->is_decl[k]) { + ctx->pinned &= ~BITFIELD64_BIT(r); + ctx->busy &= ~BITFIELD64_BIT(r); + } else if (!(ctx->pinned & BITFIELD64_BIT(r))) { + ctx->busy &= ~BITFIELD64_BIT(r); + } + } +} + static bool emit_loop(struct g9p_ctx *ctx, nir_loop *loop) { unsigned body; + const unsigned lidx = ctx->emit_loop_index++; unsigned first_exit = ctx->num_exit_jumps; const unsigned outer_depth = ctx->if_depth; @@ -3585,6 +3965,8 @@ emit_loop(struct g9p_ctx *ctx, nir_loop *loop) 2 * (int)(ctx->c->n - ctx->exit_jumps[j])); ctx->num_exit_jumps = first_exit; g9p_pop_levels(ctx->c, ctx->st, 2); + if (ctx->pressure) + expire_keys(ctx, ctx->loops[lidx].end, false); return true; } @@ -3595,6 +3977,7 @@ emit_cf_list(struct g9p_ctx *ctx, struct exec_list *list) switch (node->type) { case nir_cf_node_block: nir_foreach_instr(instr, nir_cf_node_as_block(node)) { + if (ctx->c->error) return false; ctx->ip++; /* NIR registers whose live range starts here (a loop's start) */ for (unsigned d = 0; d < ctx->num_decls; d++) { @@ -3622,9 +4005,21 @@ emit_cf_list(struct g9p_ctx *ctx, struct exec_list *list) nir_print_instr(instr, stderr); fprintf(stderr, "\n"); } + /* the refusal for the shader dump (pvr_g5p_shader_dump.h): the backend's own reason + * if it gave one, then the instruction */ + if (ctx->out->a9) { + const size_t used = strnlen(ctx->out->a9_refusal, sizeof(ctx->out->a9_refusal)); + FILE *f = fmemopen(ctx->out->a9_refusal + used, sizeof(ctx->out->a9_refusal) - used, "w"); + if (f) { + fprintf(f, "%s%s", used ? ": " : "", ctx->oom ? "out of registers at " : "cannot compile "); + nir_print_instr(instr, f); + fclose(f); + } + } return false; } nir_foreach_def(instr, park_def, ctx); + expire_keys(ctx, ctx->ip, true); } break; case nir_cf_node_if: @@ -3649,6 +4044,10 @@ emit_cf_list(struct g9p_ctx *ctx, struct exec_list *list) static bool generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) { + if (ctx->out->a9) { + ctx->out->a9_refusal[0] = a9_last_refusal[0] = 0; + ctx->out->a9_num_uploads = 0; + } uint16_t *body; unsigned body_n; @@ -3656,6 +4055,9 @@ generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) memset(ctx->reg, -1, impl->ssa_alloc * 4); memset(ctx->uni, -1, impl->ssa_alloc * 4 * sizeof(*ctx->uni)); + if (ctx->released) + memset(ctx->released, 0, ctx->num_keys); + ctx->emit_loop_index = 0; ctx->c->n = 0; ctx->c->uses_stack = false; ctx->busy = ctx->pinned = ctx->temps = 0; @@ -3735,11 +4137,12 @@ generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) * write is the end. */ stack = ctx->stage == MESA_SHADER_COMPUTE || ctx->c->uses_stack; + if (ctx->c->error) return g9p_fail(ctx->out, "%s", ctx->c->error); body_n = ctx->c->n; G9P_A9_SET_BODYLESS(ctx, body_n); body = malloc(body_n * sizeof(uint16_t) + 1); if (!body) - return false; + return g9p_fail(ctx->out, "allocation failure framing program code"); memcpy(body, ctx->c->w, body_n * sizeof(uint16_t)); ctx->c->n = 0; if (ctx->stage == MESA_SHADER_COMPUTE) @@ -3852,7 +4255,7 @@ generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) g9p_mov_from_high(ctx->c, data, st->data); } g9p_mem_at(ctx->c, true, G9P_SIZE_U32, data, addrs[k].pair); - ctx->c->w[ctx->c->n - 1] |= st->dword; + (*g9p_code_word(ctx->c, ctx->c->n - 1)) |= st->dword; if (++batch == 8 && i + 1 < ctx->num_vs_stores) { g9p_wait(ctx->c); batch = 0; @@ -3904,7 +4307,7 @@ generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) G9P_A9_END_PAD(ctx); g9p_end(ctx->c); break; } - return true; + return !ctx->c->error || g9p_fail(ctx->out, "%s", ctx->c->error); } int @@ -3934,6 +4337,27 @@ phis_scalar(const nir_instr *instr, const void *data) return 1; } +/* No native find-MSB emitter yet: branchless binary search, including zero + * and GLSL's signed negative-input convention. */ +static bool +lower_find_msb(nir_builder *b, nir_alu_instr *alu, void *data) +{ + if (alu->op != nir_op_ufind_msb && alu->op != nir_op_ifind_msb) + return false; + b->cursor = nir_before_instr(&alu->instr); + nir_def *x = nir_ssa_for_alu_src(b, alu, 0); + if (alu->op == nir_op_ifind_msb) + x = nir_bcsel(b, nir_ilt_imm(b, x, 0), nir_inot(b, x), x); + nir_def *t = x, *r = nir_imm_int(b, 0); + for (unsigned shift = x->bit_size / 2; shift; shift /= 2) { + nir_def *upper = nir_ugt_imm(b, t, (1ull << shift) - 1); + t = nir_bcsel(b, upper, nir_ushr_imm(b, t, shift), t); + r = nir_iadd(b, r, nir_bcsel(b, upper, nir_imm_int(b, shift), nir_imm_int(b, 0))); + } + nir_def_replace(&alu->def, nir_bcsel(b, nir_ieq_imm(b, x, 0), nir_imm_int(b, -1), r)); + return true; +} + static void optimize(nir_shader *nir) { @@ -3949,6 +4373,11 @@ optimize(nir_shader *nir) NIR_PASS(progress, nir, nir_opt_cse); NIR_PASS(progress, nir, nir_opt_constant_folding); NIR_PASS(progress, nir, nir_opt_algebraic); + /* Algebraic lowering of find_lsb/ifind_msb introduces ufind_msb. */ + NIR_PASS(progress, nir, nir_shader_alu_pass, lower_find_msb, + nir_metadata_control_flow, NULL); + /* Algebraic lowering of extended multiplies can introduce mul_high. */ + NIR_PASS(progress, nir, nir_lower_alu); NIR_PASS(progress, nir, nir_opt_dead_cf); NIR_PASS(progress, nir, nir_opt_remove_phis); } while (progress); @@ -4160,6 +4589,8 @@ lower_front(nir_shader *nir) glsl_get_natural_size_align_bytes); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_shared, nir_address_format_32bit_offset); } + NIR_PASS(_, nir, nir_shader_alu_pass, lower_find_msb, + nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_shader_intrinsics_pass, zero_descriptor_offset, nir_metadata_control_flow, NULL); } @@ -4247,15 +4678,15 @@ lower_coverage(nir_shader *nir, const struct g9p_coverage *coverage) } /* The last attempt at a vertex shader: in program order, every load at its use */ -static _Thread_local bool g9p_no_schedule; +static thread_local bool g9p_no_schedule; /* * Registers above r31 only once every attempt without them failed: a shader that fits * reloading its values runs far more instances at once (the aquarium's fish vertex shader: * 100 registers with homes on the first attempt, 4 instances, geometry 10x slower) */ -static _Thread_local bool g9p_no_homes; +static thread_local bool g9p_no_homes; /* vertex shaders: attribute loads moved next to their uses (g9p_compile_vertex() tries both) */ -static _Thread_local bool g9p_move_inputs; +static thread_local bool g9p_move_inputs; /* * Vertex outputs right after their values, with emit_output() writing each @@ -4318,7 +4749,7 @@ hoist_vertex_outputs(nir_shader *nir) return progress; } /* the homes an attempt may use: few first (r32-r63), all of them last */ -static _Thread_local unsigned g9p_max_homes = 92; +static thread_local unsigned g9p_max_homes = 92; /* * Loads that are cheap to repeat (uniform buffers, push constants, @@ -4685,12 +5116,14 @@ size_of_sampled_level(nir_builder *b, nir_tex_instr *tex, void *data) static bool compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, const struct g9p_vertex_attrib *attribs, unsigned frag_coord_slot, - bool repeat_loads, bool point_size, void *mem_ctx) + bool repeat_loads, bool point_size, void *mem_ctx, bool pressure, bool apple_budget) { /* G9P_LATE_OUTPUTS: outputs held to the end unless out of registers, as before */ /* not with transform feedback: its capture reads the outputs at the end */ + apple_budget &= stage == MESA_SHADER_COMPUTE && out->a9; const bool xfb = nir->xfb_info && nir->xfb_info->buffers_written; - struct g9p_ctx ctx = { .point_size = point_size, .repeat_loads = repeat_loads, + struct g9p_ctx ctx = { .pressure = pressure && stage == MESA_SHADER_COMPUTE && out->a9, + .point_size = point_size, .repeat_loads = repeat_loads, .flush_outputs = repeat_loads || (!xfb && !getenv("G9P_LATE_OUTPUTS")) }; nir_function_impl *impl; @@ -4717,8 +5150,11 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, NIR_PASS(_, nir, nir_lower_tex, &offsets); NIR_PASS(_, nir, nir_lower_tex, &levels); } + /* A9 compute reads texels with the sampler's read form and the fetch sampler (g9p_a9_tex.h) */ + if (!(out->a9 && stage == MESA_SHADER_COMPUTE)) NIR_PASS(_, nir, nir_shader_tex_pass, lower_txf, nir_metadata_control_flow, NULL); - NIR_PASS(_, nir, nir_shader_instructions_pass, lower_half_packing, nir_metadata_control_flow, NULL); + NIR_PASS(_, nir, nir_shader_instructions_pass, lower_half_packing, nir_metadata_control_flow, + (void *)(uintptr_t)(out->a9 && stage == MESA_SHADER_COMPUTE)); } /* integer division through floats: no such instruction (GLSL ES 1.00 int and ivec /) */ { @@ -4735,6 +5171,13 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, NULL, NULL); NIR_PASS(_, nir, nir_opt_algebraic_late); optimize(nir); + if (ctx.pressure) { + /* Fold constant shared addresses into a9_shared()'s word immediate; + * CSE otherwise keeps an iadd per inlined copy live. */ + const nir_opt_offsets_options off = { .shared_max = 16336 }; + NIR_PASS(_, nir, nir_opt_offsets, &off); + optimize(nir); + } if (stage == MESA_SHADER_FRAGMENT) { NIR_PASS(_, nir, nir_shader_intrinsics_pass, lower_dynamic_ubo_instr, nir_metadata_control_flow, NULL); @@ -4776,7 +5219,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, impl = nir_shader_get_entrypoint(nir); if (!impl) - return false; + return g9p_fail(out, "shader has no entry point"); nir_index_ssa_defs(impl); ctx.tex_total = count_textures(impl); if (getenv("G9P_DEBUG")) @@ -4794,7 +5237,18 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.attribs = attribs; ctx.frag_coord_slot = frag_coord_slot; ctx.c = rzalloc(mem_ctx, struct g9p_code); + if (!ctx.c) return g9p_fail(out, "allocation failure allocating program code"); + ctx.c->alloc_ctx = ctx.c; + ctx.c->w = ctx.c->initial_w; + ctx.c->capacity = G9P_INITIAL_CODE_WORDS; ctx.c->fragment = stage == MESA_SHADER_FRAGMENT; + ctx.c->high_bank0 = stage == MESA_SHADER_COMPUTE && out->a9; + ctx.c->a9_stack = stage == MESA_SHADER_COMPUTE && out->a9; + /* A9 compute: values in bank 1 up to the workgroup's limit, the mask stack after them; the rest in + * bank-0 homes (g9p_a9_bank1_limit()) */ + if (stage == MESA_SHADER_COMPUTE && out->a9) + ctx.value_cap = g9p_a9_bank1_limit(MAX2(out->local_size[0], 1) * MAX2(out->local_size[1], 1) * + MAX2(out->local_size[2], 1)) - 1; ctx.reg = ralloc_array(mem_ctx, int8_t, impl->ssa_alloc * 4); ctx.uni = ralloc_array(mem_ctx, int16_t, impl->ssa_alloc * 4); memset(ctx.uni, -1, impl->ssa_alloc * 4 * sizeof(*ctx.uni)); @@ -4805,20 +5259,43 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, memset(ctx.reg_first, 0xff, impl->ssa_alloc * sizeof(unsigned)); ctx.decls = ralloc_array(mem_ctx, nir_def *, impl->ssa_alloc); ctx.max_exit_jumps = 256; - ctx.exit_jumps = ralloc_array(mem_ctx, unsigned, ctx.max_exit_jumps); + ctx.exit_jumps = ralloc_array(ctx.c, unsigned, ctx.max_exit_jumps); + ctx.num_loops = count_loop_records(&impl->body); + ctx.loops = ralloc_array(ctx.c, __typeof__(*ctx.loops), MAX2(ctx.num_loops, 1)); + if (!ctx.loops || !ctx.exit_jumps) + return g9p_fail(out, "allocation failure allocating loop records or exits"); ctx.ip = 0; walk_liveness(&ctx, &impl->body, false); - if (ctx.loop_index > G9P_MAX_LOOPS) { - fprintf(stderr, "g9p: too many loops\n"); - return false; - } ctx.ip = 0; ctx.loop_index = 0; walk_liveness(&ctx, &impl->body, true); + if (ctx.pressure) { + reg_liveness(&ctx, impl, mem_ctx); + const unsigned nip = ctx.ip + 2; + unsigned total = 0; + ctx.dies_at = rzalloc_array(mem_ctx, unsigned, nip + 1); + ctx.released = rzalloc_array(mem_ctx, uint8_t, ctx.num_keys); + ctx.is_decl = rzalloc_array(mem_ctx, uint8_t, ctx.num_keys); + for (unsigned d = 0; d < ctx.num_decls; d++) + ctx.is_decl[key(ctx.decls[d], 0)] = 1; + for (unsigned k = 0; k < ctx.num_keys; k++) + if (ctx.last_use[k] && ctx.last_use[k] < nip) + ctx.dies_at[ctx.last_use[k] + 1]++, total++; + for (unsigned i = 1; i <= nip; i++) + ctx.dies_at[i] += ctx.dies_at[i - 1]; + ctx.dies_keys = ralloc_array(mem_ctx, unsigned, MAX2(total, 1)); + unsigned *fill = ralloc_array(mem_ctx, unsigned, nip); + memcpy(fill, ctx.dies_at, nip * sizeof(unsigned)); + for (unsigned k = 0; k < ctx.num_keys; k++) + if (ctx.last_use[k] && ctx.last_use[k] < nip) + ctx.dies_keys[fill[ctx.last_use[k]]++] = k; + } + + /* G9P_PRESSURE: the most scalars live at once, from the liveness above */ - if (getenv("G9P_PRESSURE")) { + if (debug_get_option_g9p_pressure()) { unsigned *delta = rzalloc_array(mem_ctx, unsigned, ctx.ip + 2), *dec = rzalloc_array(mem_ctx, unsigned, ctx.ip + 2); unsigned live = 0, peak = 0, peak_ip = 0, decls = 0; @@ -4849,8 +5326,18 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, if (live > peak) peak = live, peak_ip = i; } - fprintf(stderr, "g9p: pressure peak %u at instruction %u of %u (%u NIR register scalars)\n", + mesa_logi( "g9p: pressure peak %u at instruction %u of %u (%u NIR register scalars)\n", peak, peak_ip, ctx.ip, decls); + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + nir_def *def = nir_instr_def(instr); + if (def && ctx.def_ip[def->index] == peak_ip) { + mesa_logi( "g9p: pressure peak instruction: "); + nir_print_instr(instr, stderr); + fputc('\n', stderr); + } + } + } } /* @@ -4858,17 +5345,19 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, * Fragment shaders that run out of registers park the NIR register live * at that point with the furthest last use above r31 and try again. */ - ctx.home = ralloc_array(mem_ctx, int8_t, impl->ssa_alloc * 4); - memset(ctx.home, -1, impl->ssa_alloc * 4); + ctx.home = ralloc_array(mem_ctx, int16_t, impl->ssa_alloc * 4); + memset(ctx.home, -1, impl->ssa_alloc * 4 * sizeof(*ctx.home)); /* * Homes are shared: a value parks in the first one no other parked value is live in. * One home each ran out at 96 in Firefox's shaders. */ struct { unsigned start, end; - int8_t home; - } *homed = ralloc_array(mem_ctx, __typeof__(*homed), 1024); - unsigned num_homed = 0; + int16_t home; + } *homed = NULL; + unsigned num_homed = 0, homed_capacity = 1024; + homed = ralloc_array(mem_ctx, __typeof__(*homed), homed_capacity); + if (!homed) return g9p_fail(out, "allocation failure allocating parked live ranges"); /* retries that ran out at the same instruction: parking more does not help there */ unsigned stuck = 0, last_oom_ip = UINT_MAX; @@ -4883,7 +5372,7 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, bool victim_decl = false; ctx.oom = false; - if (generate(&ctx, impl, ctx.reserve_st_low ? 15 : G9P_NUM_REGS)) + if (generate(&ctx, impl, ctx.reserve_st_low ? 15 : out->a9 ? VALUE_REGS(&ctx) : G9P_NUM_REGS)) break; if (ctx.oom && stage == MESA_SHADER_VERTEX && (ctx.reserve_low < 2 || ctx.reserve_low == G9P_RESERVE_LOW_PAIR)) { @@ -4891,77 +5380,135 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, ctx.reserve_st_low = true; continue; } - if (!ctx.oom || stage == MESA_SHADER_COMPUTE || num_homed >= 1024 || g9p_no_homes) - return false; + /* A9 compute parks values above its bank-1 limit in bank 0. */ + if (!ctx.oom || (stage == MESA_SHADER_COMPUTE && !(out->a9)) || + g9p_no_homes) + return backend_failure(out, &ctx); stuck = ctx.oom_ip == last_oom_ip ? stuck + 1 : 0; last_oom_ip = ctx.oom_ip; /* each retry parks one more value; 40 live scalars need more than 16 (drawbench fs 2) */ - if (stuck >= 64) { - fprintf(stderr, "g9p: out of registers at instruction %u (parking does not help)\n", - ctx.oom_ip); - return false; + if (stuck >= (apple_budget ? 208 : 64)) { + g9p_fail(out, "out of registers at instruction %u (parking does not help)", ctx.oom_ip); + return backend_failure(out, &ctx); } - for (unsigned d = 0; d < ctx.num_decls; d++) { - const unsigned k = key(ctx.decls[d], 0); - - if (ctx.home[k] < 0 && ctx.reg_first[ctx.decls[d]->index] <= ctx.oom_ip && - ctx.last_use[k] >= ctx.oom_ip && ctx.last_use[k] > best) { - best = ctx.last_use[k]; - victim = k; - victim_decl = true; + for (unsigned batch = 0; batch < (ctx.pressure ? 4 : 1); batch++) { + best = 0; + victim = -1; + victim_decl = false; + if (ctx.pressure) { + const int lim = ctx.oom_low ? 16 : 32; + for (unsigned d = 0; d < ctx.num_decls; d++) { + const unsigned k = key(ctx.decls[d], 0); + const int r = ctx.reg[k]; + if (ctx.home[k] < 0 && r >= 0 && r < lim && + (ctx.pinned & BITFIELD64_BIT(r)) && + ctx.reg_first[ctx.decls[d]->index] <= ctx.oom_ip && + ctx.last_use[k] >= ctx.oom_ip && ctx.last_use[k] > best) { + best = ctx.last_use[k]; + victim = k; + victim_decl = true; + } + } + if (victim < 0) { + for (unsigned k = 0; k < ctx.num_keys; k++) { + const int r = ctx.reg[k]; + if (!ctx.is_decl[k] && r >= 0 && r < lim && ctx.home[k] < 0 && + ctx.last_use[k] > ctx.oom_ip && ctx.last_use[k] > best && + !((ctx.pinned | ctx.temps) & BITFIELD64_BIT(r))) { + best = ctx.last_use[k]; + victim = k; + } + } + } } - } - /* else any NIR register, even past its last use: they stay pinned (dEQP tmp_array) */ - for (unsigned d = 0; victim < 0 && d < ctx.num_decls; d++) { - const unsigned k = key(ctx.decls[d], 0); + if (victim < 0) { + for (unsigned d = 0; d < ctx.num_decls; d++) { + const unsigned k = key(ctx.decls[d], 0); + + if (ctx.home[k] < 0 && ctx.reg_first[ctx.decls[d]->index] <= ctx.oom_ip && + ctx.last_use[k] >= ctx.oom_ip && ctx.last_use[k] > best) { + best = ctx.last_use[k]; + victim = k; + victim_decl = true; + } + } + /* else any NIR register, even past its last use: they stay pinned (dEQP tmp_array) */ + for (unsigned d = 0; victim < 0 && d < ctx.num_decls; d++) { + const unsigned k = key(ctx.decls[d], 0); - if (ctx.home[k] < 0 && ctx.reg_first[ctx.decls[d]->index] <= ctx.oom_ip) { - victim = k; - victim_decl = true; - } - } - /* else the live value (not a NIR register's) with the furthest last use */ - for (unsigned k = 0, ssa_best = 0; victim < 0 && k < ctx.num_keys; k++) { - const int r = ctx.reg[k]; + if (ctx.home[k] < 0 && ctx.reg_first[ctx.decls[d]->index] <= ctx.oom_ip) { + victim = k; + victim_decl = true; + } + } + /* else the live value (not a NIR register's) with the furthest last use */ + for (unsigned k = 0, ssa_best = 0; victim < 0 && k < ctx.num_keys; k++) { + const int r = ctx.reg[k]; - if (r < 0 || r >= 32 || ctx.home[k] >= 0 || ctx.last_use[k] <= ctx.oom_ip || - (ctx.pinned | ctx.temps) & BITFIELD64_BIT(r) || ctx.last_use[k] <= ssa_best) - continue; - ssa_best = ctx.last_use[k]; - best = k + 1; - } - if (victim < 0 && best && best <= ctx.num_keys && ctx.home[best - 1] < 0 && - ctx.reg[best - 1] >= 0) - victim = best - 1; - if (victim < 0) { - fprintf(stderr, "g9p: out of registers at instruction %u\n", ctx.oom_ip); - return false; - } - { - const unsigned start = victim_decl ? ctx.reg_first[victim / 4] : ctx.def_ip[victim / 4]; - /* a NIR register's range takes in the loops it is used in (use_scalar()) */ - const unsigned end = ctx.last_use[victim]; - const unsigned max_homes = g9p_max_homes; - int home = -1; - - for (unsigned h = 0; home < 0 && h < max_homes; h++) { - bool free = true; - - for (unsigned i = 0; free && i < num_homed; i++) - free = homed[i].home != 32 + (int)h + (getenv("G9P_HOME_SKIP") ? atoi(getenv("G9P_HOME_SKIP")) : 0) || homed[i].end < start || end < homed[i].start; - if (free) - home = 32 + h + (getenv("G9P_HOME_SKIP") ? atoi(getenv("G9P_HOME_SKIP")) : 0); + if (r < 0 || r >= 32 || ctx.home[k] >= 0 || ctx.last_use[k] <= ctx.oom_ip || + (ctx.pinned | ctx.temps) & BITFIELD64_BIT(r) || ctx.last_use[k] <= ssa_best) + continue; + ssa_best = ctx.last_use[k]; + best = k + 1; + } + if (victim < 0 && best && best <= ctx.num_keys && ctx.home[best - 1] < 0 && + ctx.reg[best - 1] >= 0) + victim = best - 1; } - if (home < 0) { - fprintf(stderr, "g9p: out of registers above r31 at instruction %u\n", ctx.oom_ip); - return false; + if (victim < 0 && batch) + break; + if (victim < 0) { + g9p_fail(out, "out of registers at instruction %u (no parking victim)", ctx.oom_ip); + return backend_failure(out, &ctx); + } + { + const unsigned start = victim_decl ? ctx.reg_first[victim / 4] : ctx.def_ip[victim / 4]; + /* a NIR register's range takes in the loops it is used in (use_scalar()) */ + const unsigned end = ctx.last_use[victim]; + /* Only the third A9 compute attempt gets Apple's full per-lane budget. */ + const unsigned max_homes = + stage != MESA_SHADER_COMPUTE ? g9p_max_homes + : out->a9 ? MIN2(apple_budget ? 208 : 64, + g9p_a9_max_homes(MAX2(out->local_size[0], 1) * + MAX2(out->local_size[1], 1) * + MAX2(out->local_size[2], 1))) + : MIN2(g9p_max_homes, 64); + int home = -1; + + for (unsigned h = 0; home < 0 && h < max_homes; h++) { + bool free = !(out->a9 && 32 + (int)h > 239); + + for (unsigned i = 0; free && i < num_homed; i++) + free = homed[i].home != 32 + (int)h || homed[i].end < start || end < homed[i].start; + if (free) + home = 32 + h; + } + if (home < 0) { + if (batch) + break; + g9p_fail(out, "out of registers above r31 at instruction %u (bank-0 homes exhausted)", ctx.oom_ip); + return backend_failure(out, &ctx); + } + if (num_homed == homed_capacity) { + unsigned capacity = homed_capacity * 2; + __typeof__(homed) ranges = capacity > homed_capacity ? + reralloc(mem_ctx, homed, __typeof__(*homed), capacity) : NULL; + if (!ranges) + return g9p_fail(out, "allocation failure growing parked live ranges"); + homed = ranges; + homed_capacity = capacity; + } + homed[num_homed++] = (__typeof__(*homed)){ start, end, (int16_t)home }; + ctx.home[victim] = home; + ctx.num_homes = MAX2(ctx.num_homes, (unsigned)home - 31); } - homed[num_homed++] = (__typeof__(*homed)){ start, end, (int8_t)home }; - ctx.home[victim] = home; - ctx.num_homes = MAX2(ctx.num_homes, (unsigned)home - 31); } } st = ctx.max_reg + 1; + if (ctx.c->high_bank0 && ctx.value_cap && st + 1 > ctx.value_cap + 1) { + g9p_fail(out, "bank-1 cap exceeded"); + return backend_failure(out, &ctx); + } /* * Vertex shaders: the mask stack below r16, selects use it too (dEQP vertex * shaders of 17 registers and more drew wrong values or hung the geometry @@ -4978,22 +5525,22 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, for (unsigned k = 0; k < ctx.out->num_buffers; k++) changed |= ctx.out->buffers[k].dwords != ctx.prev_dwords[k]; if (changed && !generate(&ctx, impl, 15)) - return false; + return backend_failure(out, &ctx); st = MAX2(ctx.max_reg, 15); } else if (stage == MESA_SHADER_VERTEX && st > 15) { ctx.reserve_st_low = true; if (!generate(&ctx, impl, 15)) { /* without r15, address work can run out of low registers */ if (!ctx.oom || ctx.reserve_low) - return false; + return backend_failure(out, &ctx); ctx.reserve_low = true; ctx.oom = false; if (!generate(&ctx, impl, 15)) - return false; + return backend_failure(out, &ctx); } st = MAX2(ctx.max_reg, 15); } else if (!generate(&ctx, impl, st)) - return false; + return backend_failure(out, &ctx); /* * Buffers over the uniform operands' 512 dwords: only the dwords it reads, in as few runs @@ -5043,8 +5590,8 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, runs++; } if (span > 512) { - fprintf(stderr, "g9p: fragment shader reads %u dwords of buffers in %u runs\n", span, runs); - return false; + g9p_fail(out, "fragment shader reads %u dwords of buffers in %u runs", span, runs); + return backend_failure(out, &ctx); } unsigned next = g9p_fs_buf_base(out->num_textures); @@ -5073,18 +5620,24 @@ compile_back(nir_shader *nir, struct g9p_shader *out, mesa_shader_stage stage, } } if (!generate(&ctx, impl, st)) - return false; + return backend_failure(out, &ctx); } } out->num_regs = MAX2(st + 1, ctx.num_homes || ctx.num_vs_high ? 32 + ctx.num_homes + ctx.num_vs_high : 0); + /* A9 compute: the code's own registers (bank 1, r0..r@st) are what the CDM record's register field + * allocates; its homes are bank 0's, which the DOUTU temps (num_regs) allocate (g9p_mov_to_high()) */ + if (ctx.c->high_bank0) { + assert(st + 1 <= 32); + out->a9_bank1_regs = st + 1; + } out->code = ctx.c->w; out->code_size = 2 * ctx.c->n; if (stage != MESA_SHADER_COMPUTE && st + 1 > 32) { - fprintf(stderr, "g9p: %u registers, a graphics task has 32\n", out->num_regs); - return false; + g9p_fail(out, "%u registers, a graphics task has 32", out->num_regs); + return backend_failure(out, &ctx); } if (getenv("G9P_DEBUG")) { fprintf(stderr, "g9p: %u registers, %u bytes of code\n", out->num_regs, @@ -5106,7 +5659,7 @@ g9p_compile_compute(nir_shader *nir, struct g9p_shader *out, void *mem_ctx) memset(out, 0, sizeof(*out)); lower_front(nir); NIR_PASS(_, nir, nir_lower_compute_system_values, &csv); - return compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx); + return compile_back(nir, out, MESA_SHADER_COMPUTE, NULL, 0, false, false, mem_ctx, false, false); } #include "g9p_a9_compute.h" /* @@ -5217,6 +5770,11 @@ lower_half_packing(nir_builder *b, nir_instr *instr, void *data) nir_def *src = nir_ssa_for_alu_src(b, alu, 0); if (alu->op == nir_op_pack_half_2x16) { + if (data) { + nir_def_replace(&alu->def, nir_pack_half_2x16_split(b, nir_channel(b, src, 0), + nir_channel(b, src, 1))); + return true; + } nir_def_replace(&alu->def, nir_ior(b, f32_to_half(b, nir_channel(b, src, 0)), nir_ishl_imm(b, f32_to_half(b, nir_channel(b, src, 1)), 16))); } else { @@ -5490,7 +6048,7 @@ g9p_compile_vertex_attempts(nir_shader *nir, const struct g9p_vertex_attrib *att nir_shader *n = nir_shader_clone(mem_ctx, attempt[i]); g9p_no_schedule = i == 2; - if (compile_back(n, out, MESA_SHADER_VERTEX, attribs, 0, i > 0, point_size, mem_ctx)) { + if (compile_back(n, out, MESA_SHADER_VERTEX, attribs, 0, i > 0, point_size, mem_ctx, false, false)) { g9p_no_schedule = g9p_no_homes = false; g9p_max_homes = 92; return true; @@ -5882,7 +6440,7 @@ g9p_compile_fragment(nir_shader *nir, unsigned frag_coord_slot, nir_shader *n = nir_shader_clone(mem_ctx, attempt[i]); g9p_no_schedule = i == 2; - if (compile_back(n, out, MESA_SHADER_FRAGMENT, NULL, frag_coord_slot, i > 0, false, mem_ctx)) { + if (compile_back(n, out, MESA_SHADER_FRAGMENT, NULL, frag_coord_slot, i > 0, false, mem_ctx, false, false)) { g9p_no_schedule = g9p_no_homes = false; g9p_max_homes = 92; return true; diff --git a/src/imagination/g9p/g9p_compile.h b/src/imagination/g9p/g9p_compile.h index 062a0f2abbb..e97bc6583dc 100644 --- a/src/imagination/g9p/g9p_compile.h +++ b/src/imagination/g9p/g9p_compile.h @@ -89,6 +89,8 @@ g9p_fs_buf_base(unsigned n) #define G9P_SHARED_BASE 16 /* Compute: gl_NumWorkGroups, x and y in binding 0's slot, z in binding 1's (filled per dispatch) */ #define G9P_NUM_WORKGROUPS (~22u) +/* Device-backed workgroup memory, binding 0. */ +#define G9P_SHARED_SPILL (~24u) /* A draw's parameters: binding 0's slot holds firstInstance (filled per draw) */ #define G9P_DRAW_PARAMS (~23u) /* The set of the push-constant slot (its address is loaded like a buffer's) */ @@ -193,6 +195,7 @@ struct g9p_shader { unsigned local_size[3]; /* compute: bytes of shared variables (after G9P_SHARED_BASE) */ unsigned shared_size; + unsigned shared_spill_stride; /* 0: hardware threadgroup memory */ /* * Vertex: varyings written (VARn, each 4 components after the position); diff --git a/src/imagination/g9p/g9p_isa.h b/src/imagination/g9p/g9p_isa.h index af114d3ec4a..9eef9e1535e 100644 --- a/src/imagination/g9p/g9p_isa.h +++ b/src/imagination/g9p/g9p_isa.h @@ -16,14 +16,31 @@ #include #include #include +#include +#include +#include "util/ralloc.h" -#define G9P_MAX_CODE_WORDS 65536 +#define G9P_INITIAL_CODE_WORDS 65536 struct g9p_code { - uint16_t w[G9P_MAX_CODE_WORDS]; + uint16_t *w; + unsigned capacity; + void *alloc_ctx; + const char *error; + /* Small generated graphics helpers are calloc/free objects, not ralloc + * objects. Preserve their inline storage and lifetime. Compiler programs + * grow beyond this initial storage under alloc_ctx. */ + uint16_t initial_w[G9P_INITIAL_CODE_WORDS]; + uint16_t failed_patch; unsigned n; /* a fragment shader: registers in the other bank, see g9p_w0() */ bool fragment; + /* registers above r31 ("homes") in the fragment bank (bank 0) while the code's own stay in bank 1: A9 compute + * (g9p_mov_to_high()) */ + bool high_bank0; + /* A9 compute: Apple's A9 mask-stack rule, the stack register's last-read bit only on the program's + * last stack operation (g9p_pop()), not on every pop (g9p_pop_levels()) */ + bool a9_stack; /* the code pushes onto the mask stack (needs g9p_push_init()) */ bool uses_stack; }; @@ -51,9 +68,51 @@ g9p_dst(const struct g9p_code *c, unsigned r) static inline void g9p_emit(struct g9p_code *c, const uint16_t *w, unsigned count) { - assert(c->n + count <= G9P_MAX_CODE_WORDS); - for (unsigned i = 0; i < count; i++) - c->w[c->n++] = w[i]; + if (c->error) return; + if (!c->w) { + c->w = c->initial_w; + c->capacity = G9P_INITIAL_CODE_WORDS; + } + /* Signed 32-bit byte-relative jumps and unsigned byte code_size. */ + if (count > INT_MAX / 2u - c->n) { + c->error = "program exceeds the 32-bit jump-offset range"; + return; + } + if (count > c->capacity - c->n) { + unsigned capacity = c->capacity; + while (capacity < c->n + count) + capacity = capacity > INT_MAX / 4u ? INT_MAX / 2u : capacity * 2; + if (!c->alloc_ctx) { + c->error = "generated helper exceeds inline code storage"; + return; + } + uint16_t *w = reralloc(c->alloc_ctx, c->w == c->initial_w ? NULL : c->w, + uint16_t, capacity); + if (!w) { + c->error = "allocation failure growing program code"; + return; + } + if (c->w == c->initial_w) + memcpy(w, c->w, c->n * sizeof(*w)); + c->w = w; + c->capacity = capacity; + } + memcpy(c->w + c->n, w, count * sizeof(*w)); + c->n += count; +} + +/* Patches retain word indices, never pointers across an emit/reallocation. + * A failed append must not turn its following patch into an out-of-bounds + * write. The compile reports error before exporting code. */ +static inline uint16_t * +g9p_code_word(struct g9p_code *c, unsigned at) +{ + if (c->error) return &c->failed_patch; + if (at >= c->n) { + c->error = "program back-patch outside emitted code"; + return &c->failed_patch; + } + return &c->w[at]; } #define G9P_EMIT(c, ...) \ @@ -130,10 +189,10 @@ g9p_end(struct g9p_code *c) unsigned end = (jump + 6 + 63) / 64 * 64; G9P_EMIT(c, 0x0100, 0x0100 | (end - jump), 0x0000); - while (2 * c->n < end) + while (!c->error && 2 * c->n < end) G9P_EMIT(c, 0xc000); G9P_EMIT(c, 0x8000); - while (c->n % 32) + while (!c->error && c->n % 32) G9P_EMIT(c, 0xc000); } @@ -158,6 +217,25 @@ g9p_mov_imm(struct g9p_code *c, unsigned dst, uint32_t imm) } } +/* + * Word fields, named for the encoders below: W0 holds the class byte, the continuation bit 8 and, outside fragment + * code (g9p_w0()), the register-bank bits of sources B (bit 12) and A (bit 13); a destination word (g9p_dst()) + * the half index, the continuation and the bank bit 9; a source word (g9p_srcs()) source A's register at bit 2 + * and B's at bit 10, each with its last-read bit (7 and 15). A register source above r31 sets the source word's + * bit 0 and puts r >> 5 in an extension word (A at bit 1, B at bit 9). + */ +#define G9P_CONT 0x0100 +#define G9P_W0_SRC_B_BANK1 0x1000 +#define G9P_W0_SRC_A_BANK1 0x2000 +#define G9P_W1_BANK1 0x0200 +#define G9P_CLASS_ALU2 0x99 /* g9p_alu2()'s class: two register sources */ +#define G9P_W0_ALU2 (G9P_CLASS_ALU2 | G9P_CONT | G9P_W0_SRC_A_BANK1 | G9P_W0_SRC_B_BANK1) +#define G9P_SRC_A_SHIFT 2 +#define G9P_SRC_B_SHIFT 10 +#define G9P_SRCS_EXT 0x0001 +#define G9P_EXT_A_SHIFT 1 +#define G9P_EXT_B_SHIFT 9 + /* Two register sources; @kill marks both as last use */ static inline uint16_t g9p_srcs(unsigned a, unsigned b, bool kill) @@ -182,19 +260,40 @@ g9p_alu2(struct g9p_code *c, unsigned op, unsigned dst, unsigned a, unsigned b, * r31"). The compiler parks NIR registers there ("homes") and moves them * with OR, which copies any bit pattern. */ +/* + * A9 compute (c->high_bank0): its registers r0-r31 are in bank 1 (W0 bits 11-13, W1 bit 9), but the + * registers above r31 are bank 0's, as in Apple's A9 kernels: + * examined kernels write bank 0 up to r60 and bank 1 up to r2 or r7, and declare 64 + * registers (__GPU_METADATA r.0.0), so the count covers bank 0. The kernel's register allocation comes + * from it (CDM K0 bits 12+ = ceil(regs / 4), AGXMetalA9 ComputeProgramVariant 0x207d2c4b0 -> +0x9b8; + * DOUTU temps (regs & ~3) << 1, EncodeComputePDSProgram 0x207d1ea4c). Homes in bank 1 above r31 + * read wrong values on every lane of a 69-register kernel and lost the GPU; that bank 1 + * holds only 32, as in vertex code, is inferred. A home move keeps its low side in the code's bank and + * its high side in bank 0, the fragment form above (validated at r40/r77/r100 on the A9): an operand's + * header bit and the destination's W1 bit 9 name each side's bank. + */ +/* A9 compute (high_bank0): bank-0 homes up to r239. Above r127 the destination's half-index bit 8 is W0 bit 9 + * (header bit 1), as in Apple's A9 code (moves to r127 and r128 differ in that bit and the half index; a + * move to r218 with bank-1 sources sets it too). */ +#define G9P_W0_DST_HALF_BIT8 0x0200 /* the destination half index's bit 8 */ static inline void g9p_mov_to_high(struct g9p_code *c, unsigned hi, unsigned src) { - assert(src < 32 && hi < 128); - G9P_EMIT(c, g9p_w0(c, 0x3199), g9p_dst(c, hi), 0x0100 | src << 2 | src << 10, 0x00ce); + assert(src < 32 && hi < (c->high_bank0 ? 240u : 128u)); + G9P_EMIT(c, g9p_w0(c, G9P_W0_ALU2) | (c->high_bank0 && hi >= 128 ? G9P_W0_DST_HALF_BIT8 : 0), + c->high_bank0 ? G9P_CONT | ((2 * hi) & 0xff) : g9p_dst(c, hi), + g9p_srcs(src, src, false), G9P_OR); } static inline void g9p_mov_from_high(struct g9p_code *c, unsigned dst, unsigned hi) { - assert(dst < 32 && hi < 128); - G9P_EMIT(c, g9p_w0(c, 0x3199), g9p_dst(c, dst), 0x0101 | (hi & 31) << 2 | (hi & 31) << 10, - 0x0100 | (hi >> 5) << 1 | (hi >> 5) << 9, 0x00ce); + assert(dst < 32 && hi < (c->high_bank0 ? 240u : 128u)); + /* Three-bit source extensions (hi >> 5 in the fourth word), as Apple's A9 compute reads r182; + * on A9 compute (high_bank0) both sources are bank 0's. */ + G9P_EMIT(c, g9p_w0(c, G9P_W0_ALU2) & ~(c->high_bank0 ? G9P_W0_SRC_A_BANK1 | G9P_W0_SRC_B_BANK1 : 0), + g9p_dst(c, dst), g9p_srcs(hi & 31, hi & 31, false) | G9P_SRCS_EXT, + G9P_CONT | (hi >> 5) << G9P_EXT_A_SHIFT | (hi >> 5) << G9P_EXT_B_SHIFT, G9P_OR); } /* dst = src + imm, 0 <= imm < 64 */ @@ -223,6 +322,58 @@ g9p_tg_load(struct g9p_code *c, unsigned dst, unsigned word) 0x0200 | (word >> 8)); } +/* + * Threadgroup-memory access (g9p_tg_load()'s class), Apple's A9 compiler's forms: W0 the class byte, the load's + * bank bits (g9p_mem_at()) and the access size at bit 11 (G9P_SIZE_*); W1 the data register's half, the store + * bit and the index register's half at bit 9; then the word offset. With no index it takes two words, W2 with + * G9P_SHARED_OFFSET and the offset's low byte, W3 with G9P_SHARED_OFFSET_HIGH and its high bits; after an index, + * W2 adds G9P_SHARED_INDEXED and holds an offset below 256 alone, a larger one in the same two-word layout without + * G9P_SHARED_OFFSET_HIGH. Those three bits are named by their observed roles. + */ +#define G9P_CLASS_SHARED 0x14 +#define G9P_MEM_W0_BANKS 0x0300 +#define G9P_SIZE_SHIFT 11 +#define G9P_SIZE_4X32 7 +#define G9P_MEM_STORE 0x0080 +#define G9P_MEM_INDEX_SHIFT 9 +#define G9P_SHARED_OFFSET 0x8000 +#define G9P_SHARED_INDEXED 0x0800 +#define G9P_SHARED_OFFSET_HIGH 0x0200 +static inline void +g9p_shared_access(struct g9p_code *c, unsigned size, bool store, unsigned reg, unsigned word, int index_half) +{ + const uint16_t w0 = size << G9P_SIZE_SHIFT | G9P_MEM_W0_BANKS | G9P_CLASS_SHARED; + uint16_t w1 = G9P_CONT | (store ? G9P_MEM_STORE : 0) | 2 * reg; + + if (index_half < 0) { + G9P_EMIT(c, w0, w1, G9P_SHARED_OFFSET | G9P_CONT | (word & 255), G9P_SHARED_OFFSET_HIGH | (word >> 8)); + return; + } + w1 |= (unsigned)index_half << G9P_MEM_INDEX_SHIFT; + if (word < 256) + G9P_EMIT(c, w0, w1, G9P_SHARED_OFFSET | G9P_SHARED_INDEXED | word); + else + G9P_EMIT(c, w0, w1, G9P_SHARED_OFFSET | G9P_SHARED_INDEXED | G9P_CONT | (word & 255), word >> 8); +} + +/* + * A shared-memory index after its shift, as Apple's A9 compiler emits it: r[@dst] = 0 + r[@src], an integer add + * of the literal 0 (g9p_add_imm()'s literal form) in g9p_addr()'s class, with @shift in g9p_addr()'s shift field. + * G9P_INDEX_SHIFT_CTL is named by its observed role (seen in this form only). + */ +#define G9P_CLASS_ADDR 0x89 /* g9p_addr()'s class */ +#define G9P_SRC_A_IMM 0x0001 /* source A is a literal, its value from bit 1 (g9p_add_imm()) */ +#define G9P_IMM_SHORT 0x0020 /* g9p_add_imm()'s short-literal word */ +#define G9P_INDEX_SHIFT_CTL 0x0e00 +#define G9P_ADDR_SHIFT_BIT 13 /* g9p_addr()'s shift field */ +static inline void +g9p_shared_index(struct g9p_code *c, unsigned dst, unsigned src, unsigned shift) +{ + G9P_EMIT(c, G9P_CLASS_ADDR | G9P_CONT | G9P_W0_SRC_B_BANK1, G9P_CONT | G9P_W1_BANK1 | 2 * dst, + G9P_SRC_A_IMM | G9P_CONT | src << G9P_SRC_B_SHIFT, G9P_CONT | G9P_IMM_SHORT, + G9P_CONT | G9P_INDEX_SHIFT_CTL | G9P_IADD, shift << G9P_ADDR_SHIFT_BIT); +} + /* * dst = flat varying component @comp (4 per varying in the DOUTI program's * order), the provoking vertex's value bit for bit; follow with g9p_wait(). @@ -425,10 +576,27 @@ g9p_else(struct g9p_code *c, unsigned st) G9P_EMIT(c, g9p_w0(c, 0x21c8), g9p_st_dst(c, st), g9p_st_src(c, st), 0x46d4); } +/* + * Pop @levels. Bit 7 of the source word is the stack register's last-read bit. G9P (A10) sets it on every + * pop; Apple's A9 compiler sets it only on the program's last stack operation (G5P's fragment port + * g5p_usc_stack() does the same, and so does Apple's captured divergent loop: the loop's pop leaves + * the bit clear, the last pop sets it). An intermediate pop that marks the register read for the last time + * lets the counter go while lanes still wait on it: on A9 compute a loop whose lanes leave at different + * iterations ((g & 3) + 2 rounds) lost the GPU before any store, while uniform loops ran. + */ +#define G9P_SRC_A_LAST_READ 0x0080 +#define G9P_CLASS_STACK 0xc8 /* the mask stack: its register as destination and as source A */ +#define G9P_W0_STACK (G9P_CLASS_STACK | G9P_CONT | G9P_W0_SRC_A_BANK1) +#define G9P_STACK_PUSH 0xd1 +#define G9P_STACK_POP 0xd2 +#define G9P_STACK_CTL 0x0600 /* set in every stack operation's last word (observed role) */ +#define G9P_STACK_LEVELS_SHIFT 14 static inline void g9p_pop_levels(struct g9p_code *c, unsigned st, unsigned levels) { - G9P_EMIT(c, g9p_w0(c, 0x21c8), g9p_st_dst(c, st), 0x0080 | g9p_st_src(c, st), 0x06d2 | levels << 14); + G9P_EMIT(c, g9p_w0(c, G9P_W0_STACK), g9p_st_dst(c, st), + (c->a9_stack ? 0 : G9P_SRC_A_LAST_READ) | g9p_st_src(c, st), + G9P_STACK_CTL | G9P_STACK_POP | levels << G9P_STACK_LEVELS_SHIFT); } /* Loops: start (two levels), break the lanes whose compare passed/failed */ @@ -459,8 +627,8 @@ g9p_jump_patch(struct g9p_code *c, unsigned at, int offset) { uint32_t off = (uint32_t)offset; - c->w[at + 1] = (off & 0xfefe) | 0x0100 | ((off >> 8) & 1); - c->w[at + 2] = ((off >> 16) & 0xfefe) | ((off >> 24) & 1); + *g9p_code_word(c, at + 1) = (off & 0xfefe) | 0x0100 | ((off >> 8) & 1); + *g9p_code_word(c, at + 2) = ((off >> 16) & 0xfefe) | ((off >> 24) & 1); } static inline unsigned @@ -493,10 +661,10 @@ g9p_end_vertex(struct g9p_code *c, unsigned reg, unsigned slot) unsigned end = (jump + 6 + 63) / 64 * 64; G9P_EMIT(c, 0x0100, 0x0100 | (end - jump), 0x0000); - while (2 * c->n < end) + while (!c->error && 2 * c->n < end) G9P_EMIT(c, 0xc000); G9P_EMIT(c, 0x155e, 0xc100 | 2 * reg, slot); - while (c->n % 32) + while (!c->error && c->n % 32) G9P_EMIT(c, 0x0000); } @@ -508,10 +676,10 @@ g9p_end_vertex_stores(struct g9p_code *c) unsigned end = (jump + 6 + 63) / 64 * 64; G9P_EMIT(c, 0x0100, 0x0100 | (end - jump), 0x0000); - while (2 * c->n < end) + while (!c->error && 2 * c->n < end) G9P_EMIT(c, 0xc000); G9P_EMIT(c, 0x015e, 0xa000); - while (c->n % 32) + while (!c->error && c->n % 32) G9P_EMIT(c, 0x0000); } @@ -726,7 +894,7 @@ g9p_end_fragment(struct g9p_code *c, unsigned words) if (words) G9P_EMIT(c, (words & 7) << 12 | 0x011c, 0x0800); G9P_EMIT(c, 0x8000); - while (c->n % 32) + while (!c->error && c->n % 32) G9P_EMIT(c, 0x0000); } diff --git a/src/imagination/vulkan/meson.build b/src/imagination/vulkan/meson.build index dcf1adf7c8d..1b49d9a70b0 100644 --- a/src/imagination/vulkan/meson.build +++ b/src/imagination/vulkan/meson.build @@ -16,6 +16,14 @@ pvr_entrypoints = custom_target( depend_files : vk_entrypoints_gen_depend_files, ) +pvr_errlog_gen = custom_target( + 'pvr_errlog_gen', + input : [files('pvr_errlog_gen.py'), vk_api_xml], + output : 'pvr_errlog_gen.h', + command : [prog_python, '@INPUT0@', '--xml', '@INPUT1@', '--out', '@OUTPUT@'], + depend_files : vk_entrypoints_gen_depend_files, +) + pvr_drirc = custom_target( 'pvr_drirc', input : [files('pvr_drirc_gen.py'), drirc_gen, '00-pvr-defaults.conf'], @@ -39,6 +47,7 @@ pvr_files = files( 'winsys/pvr_winsys.c', 'winsys/pvr_winsys_helper.c', 'pvr_bo.c', + 'pvr_bo_log.c', 'pvr_csb.c', 'pvr_descriptor_set.c', 'pvr_device.c', @@ -164,6 +173,8 @@ pvr_files += files( 'pvr_g5p_formats.c', 'pvr_g5p_image.c', 'pvr_g5p_nir.c', + 'pvr_g5p_shader_dump.c', + 'pvr_errlog.c', 'pvr_g5p_texel_buffer.c', 'pvr_g5p_xfb_query.c', 'pvr_g5p_ds.c', @@ -218,7 +229,7 @@ endforeach libvulkan_powervr_mesa = shared_library( 'vulkan_powervr_mesa', - [pvr_files, pvr_entrypoints, pvr_drirc, u_format_pack_h], + [pvr_files, pvr_entrypoints, pvr_drirc, pvr_errlog_gen, u_format_pack_h], include_directories : [ pvr_includes, inc_imagination, diff --git a/src/imagination/vulkan/pvr_arch_cmd_buffer.c b/src/imagination/vulkan/pvr_arch_cmd_buffer.c index 99a4ae51560..ab350dff23c 100644 --- a/src/imagination/vulkan/pvr_arch_cmd_buffer.c +++ b/src/imagination/vulkan/pvr_arch_cmd_buffer.c @@ -227,6 +227,11 @@ static void pvr_cmd_buffer_free_sub_cmd(struct pvr_cmd_buffer *cmd_buffer, case PVR_SUB_CMD_TYPE_COMPUTE: case PVR_SUB_CMD_TYPE_QUERY: vk_free(&cmd_buffer->vk.pool->alloc, sub_cmd->compute.g9p_query); + for (struct pvr_g5p_dispatch_log *l = sub_cmd->compute.g5p_log_first, *next; l; l = next) { + next = l->next; + free(l->line); + free(l); + } pvr_csb_finish(&sub_cmd->compute.control_stream); break; @@ -2696,8 +2701,13 @@ VkResult pvr_arch_cmd_buffer_end_sub_cmd(struct pvr_cmd_buffer *cmd_buffer) result = pvr_sub_cmd_gfx_job_init(&device->pdevice->dev_info, cmd_buffer, gfx_sub_cmd); + if (result == VK_ERROR_FEATURE_NOT_PRESENT && + device->pdevice->skip_refused_work) { + gfx_sub_cmd->g5p_refused = true; + break; + } if (result != VK_SUCCESS) - return pvr_cmd_buffer_set_error_unwarned(cmd_buffer, result); + return pvr_cmd_buffer_set_error_unwarned(cmd_buffer, pvr_refusal_to_api(result)); } if (pvr_sub_cmd_gfx_requires_split_submit(gfx_sub_cmd)) { @@ -4335,6 +4345,13 @@ void PVR_PER_ARCH(CmdBeginRenderPass2)( assert(!state->render_pass_info.pass); assert(cmd_buffer->vk.level == VK_COMMAND_BUFFER_LEVEL_PRIMARY); + /* G5P: the pass's EXTERNAL -> subpass dependencies, before it begins (pvr_render_pass::ext_begin) */ + if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g5p) && pass->ext_begin) { + const VkDependencyInfo ext = { .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, + .pMemoryBarriers = &pass->ext_begin_barrier }; + PVR_PER_ARCH(CmdPipelineBarrier2)(commandBuffer, &ext); + } + /* FIXME: Create a separate function for everything using pass->subpasses, * look at cmd_buffer_begin_subpass() for example. */ state->render_pass_info.pass = pass; @@ -7135,6 +7152,7 @@ static VkResult pvr_cmd_upload_push_consts(struct pvr_cmd_buffer *cmd_buffer, static void pvr_cmd_dispatch( struct pvr_cmd_buffer *const cmd_buffer, const pvr_dev_addr_t indirect_addr, + const uint64_t indirect_offset, const uint32_t base_group[static const PVR_WORKGROUP_DIMENSIONS], const uint32_t workgroup_size[static const PVR_WORKGROUP_DIMENSIONS]) { @@ -7212,7 +7230,7 @@ void PVR_PER_ARCH(CmdDispatchBase)(VkCommandBuffer commandBuffer, return; pvr_cmd_dispatch(cmd_buffer, - PVR_DEV_ADDR_INVALID, + PVR_DEV_ADDR_INVALID, 0, (uint32_t[]){ baseGroupX, baseGroupY, baseGroupZ }, (uint32_t[]){ groupCountX, groupCountY, groupCountZ }); } @@ -7227,7 +7245,7 @@ void PVR_PER_ARCH(CmdDispatchIndirect)(VkCommandBuffer commandBuffer, PVR_CHECK_COMMAND_BUFFER_BUILDING_STATE(cmd_buffer); pvr_cmd_dispatch(cmd_buffer, - PVR_DEV_ADDR_OFFSET(buffer->dev_addr, offset), + PVR_DEV_ADDR_OFFSET(buffer->dev_addr, offset), offset, (uint32_t[]){ 0, 0, 0 }, (uint32_t[]){ 1, 1, 1 }); } @@ -9679,6 +9697,18 @@ static void pvr_emit_vdm_index_list(struct pvr_cmd_buffer *cmd_buffer, pvr_csb_clear_relocation_mark(csb); } +/* Refusal reasons are logged by G5P_REFUSE before this boundary. */ +static void +g5p_draw_result(struct pvr_cmd_buffer *cmd_buffer, VkResult result) +{ + if (result == VK_SUCCESS) + return; + if (result == VK_ERROR_FEATURE_NOT_PRESENT && + cmd_buffer->device->pdevice->skip_refused_work) + return; + vk_command_buffer_set_error(&cmd_buffer->vk, pvr_refusal_to_api(result)); +} + void PVR_PER_ARCH(CmdDraw)(VkCommandBuffer commandBuffer, uint32_t vertexCount, uint32_t instanceCount, @@ -9714,8 +9744,7 @@ void PVR_PER_ARCH(CmdDraw)(VkCommandBuffer commandBuffer, result = pvr_flush_push_descriptors(cmd_buffer, &cmd_buffer->state.gfx_desc_state); if (result == VK_SUCCESS) result = pvr_arch_g5p_cmd_draw(cmd_buffer, &g5p_draw); - if (result != VK_SUCCESS) - vk_command_buffer_set_error(&cmd_buffer->vk, result); + g5p_draw_result(cmd_buffer, result); goto end_cmd_draw; } if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g9p)) { @@ -9796,8 +9825,7 @@ void PVR_PER_ARCH(CmdDrawIndexed)(VkCommandBuffer commandBuffer, if (result == VK_SUCCESS) result = pvr_arch_g5p_cmd_draw_indexed(cmd_buffer, indexCount, instanceCount, firstIndex, vertexOffset, firstInstance); - if (result != VK_SUCCESS) - vk_command_buffer_set_error(&cmd_buffer->vk, result); + g5p_draw_result(cmd_buffer, result); goto end_cmd_draw_indexed; } if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g9p)) { @@ -10166,6 +10194,7 @@ void PVR_PER_ARCH(CmdEndRenderPass2)(VkCommandBuffer commandBuffer, assert(state->render_pass_info.pass); assert(state->render_pass_info.framebuffer); + const struct pvr_render_pass *ended_pass = state->render_pass_info.pass; result = pvr_arch_cmd_buffer_end_sub_cmd(cmd_buffer); if (result != VK_SUCCESS) @@ -10184,6 +10213,14 @@ void PVR_PER_ARCH(CmdEndRenderPass2)(VkCommandBuffer commandBuffer, state->render_pass_info.attachments = attachments; state->render_pass_info.clear_values = clear_values; + /* G5P: the pass's subpass -> EXTERNAL dependencies, after it ended: a transfer or + * draw after the pass that the pass declares it must precede (xapi-present V4, arr0: + * a copy of the pass's colour read the memory before the render wrote it) */ + if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g5p) && ended_pass->ext_end) { + const VkDependencyInfo ext = { .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, + .pMemoryBarriers = &ended_pass->ext_end_barrier }; + PVR_PER_ARCH(CmdPipelineBarrier2)(commandBuffer, &ext); + } } static VkResult diff --git a/src/imagination/vulkan/pvr_arch_descriptor_set.c b/src/imagination/vulkan/pvr_arch_descriptor_set.c index 68091566380..9669970f9cb 100644 --- a/src/imagination/vulkan/pvr_arch_descriptor_set.c +++ b/src/imagination/vulkan/pvr_arch_descriptor_set.c @@ -214,7 +214,7 @@ write_storage_image(const struct pvr_descriptor_set *set, struct pvr_image_descriptor storage_image_desc = image_view->image_state[is_cube ? PVR_TEXTURE_STATE_STORAGE : PVR_TEXTURE_STATE_SAMPLE]; - + PVR_G5P_HOOK(storage_image, dev_info, image_view, &storage_image_desc) if (!PVR_G5P_NATIVE_STORAGE(dev_info) && !PVR_HAS_FEATURE(dev_info, tpu_extended_integer_lookup)) { struct ROGUE_TEXSTATE_STRIDE_IMAGE_WORD1 word1; ROGUE_TEXSTATE_STRIDE_IMAGE_WORD1_unpack(&storage_image_desc.words[1], diff --git a/src/imagination/vulkan/pvr_arch_device.c b/src/imagination/vulkan/pvr_arch_device.c index 38e04b8a1b6..9f8a933db92 100644 --- a/src/imagination/vulkan/pvr_arch_device.c +++ b/src/imagination/vulkan/pvr_arch_device.c @@ -10,6 +10,7 @@ * SPDX-License-Identifier: MIT */ +#include "pvr_bo_log.h" #include "pvr_device.h" #include "vk_log.h" @@ -35,6 +36,7 @@ #include "pvr_g9p.h" # include "pvr_g5p.h" #include "pvr_utrace.h" +#include "pvr_errlog.h" #include "util/os_misc.h" #define PVR_GLOBAL_FREE_LIST_INITIAL_SIZE (2U * 1024U * 1024U) @@ -215,6 +217,14 @@ VkResult pvr_arch_pds_compute_shader_create_and_upload( return result; } +static VkResult pvr_device_check_status(struct vk_device *vk) +{ + struct pvr_device *device = container_of(vk, struct pvr_device, vk); + if (device->ws->ops->is_lost(device->ws)) + return vk_device_set_lost(vk, "device lost (%s)", device->ws->loss_source ? device->ws->loss_source : "kernel job error"); + return VK_SUCCESS; +} + static VkResult pvr_device_init_compute_fence_program(struct pvr_device *device) { struct pvr_pds_compute_shader_program program; @@ -821,9 +831,14 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, if (result != VK_SUCCESS) goto err_free_device; + result = pvr_errlog_install(device); + if (result != VK_SUCCESS) + goto err_vk_device_finish; + device->instance = instance; device->pdevice = pdevice; device->ws = ws; + device->vk.check_status = pvr_device_check_status; /* Before anything that creates a render target dataset. */ simple_mtx_init(&device->rt_cache_mtx, mtx_plain); @@ -844,6 +859,8 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, if (result != VK_SUCCESS) goto err_vk_device_finish; + pvr_bo_log_device_init(device); + pvr_bo_suballocator_init(&device->suballoc_general, device->heaps.general_heap, device, @@ -1043,6 +1060,10 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, err_dec_device_count: p_atomic_dec(&device->instance->active_device_count); + if (unlikely(device->bo_log)) { + pvr_bo_log_summary(device, "destroy"); + pvr_bo_log_device_finish(device); + } pvr_bo_suballocator_fini(&device->suballoc_vis_test); pvr_bo_suballocator_fini(&device->suballoc_usc); pvr_bo_suballocator_fini(&device->suballoc_transfer); @@ -1052,6 +1073,7 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, pvr_bo_store_destroy(device); err_vk_device_finish: + vk_free(&device->vk.alloc, device->errlog_orig); vk_device_finish(&device->vk); err_free_device: @@ -1114,6 +1136,10 @@ void PVR_PER_ARCH(destroy_device)(struct pvr_device *device, pvr_bo_suballoc_free(device->nop_program.usc); pvr_free_list_destroy(device->global_free_list); pvr_finish_null_pages(device); + if (unlikely(device->bo_log)) { + pvr_bo_log_summary(device, "destroy"); + pvr_bo_log_device_finish(device); + } pvr_bo_suballocator_fini(&device->suballoc_vis_test); pvr_bo_suballocator_fini(&device->suballoc_usc); pvr_bo_suballocator_fini(&device->suballoc_transfer); @@ -1122,6 +1148,7 @@ void PVR_PER_ARCH(destroy_device)(struct pvr_device *device, pvr_bo_store_destroy(device); pvr_winsys_destroy(device->ws); p_atomic_dec(&device->instance->active_device_count); + vk_free(&device->vk.alloc, device->errlog_orig); vk_device_finish(&device->vk); vk_free(&device->vk.alloc, device); } diff --git a/src/imagination/vulkan/pvr_arch_g5p.c b/src/imagination/vulkan/pvr_arch_g5p.c index 9333d6738b7..3678eb55f38 100644 --- a/src/imagination/vulkan/pvr_arch_g5p.c +++ b/src/imagination/vulkan/pvr_arch_g5p.c @@ -56,7 +56,7 @@ #include "pvr_query.h" #include "pvr_rogue_pds_encode.h" -static const unsigned g5p_mrt_variant_targets[PVR_G5P_MRT_VARIANTS] = { 1, 2, 4 }; +static const unsigned g5p_mrt_variant_targets[PVR_G5P_MRT_VARIANTS] = { 1, 2, 4, 5, 6, 7, 8, 5, 6, 7, 8 }; /* The DOUTU temps of every G5P program (G5P_USC_TEMPS of the A9 ICD) */ #define G5P_TEMPS 8 @@ -230,6 +230,16 @@ g5p_free_programs(struct pvr_device *device, struct pvr_g5p_programs *p) vk_free(&device->vk.alloc, p->raw_load[i]); } } + for (unsigned i = 0; i < ARRAY_SIZE(p->mrt_load); i++) { + if (p->mrt_load[i]) { + g5p_free_programs(device, p->mrt_load[i]); + vk_free(&device->vk.alloc, p->mrt_load[i]); + } + } + if (p->ms_load) { + g5p_free_programs(device, p->ms_load); + vk_free(&device->vk.alloc, p->ms_load); + } for (unsigned i = 0; i < ARRAY_SIZE(p->raw_clear_usc); i++) { pvr_bo_suballoc_free(p->raw_clear_usc[i]); pvr_bo_suballoc_free(p->raw_clear_kick[i].pvr_bo); @@ -257,13 +267,19 @@ g5p_free_programs(struct pvr_device *device, struct pvr_g5p_programs *p) * the backend compiles from NIR, its kick, its uploads' DMA code. A program * the backend refuses is not an error here: a LOAD render is refused by * name then. */ +/* g5p_upload_load()'s variant of the 4x LOAD */ +#define G5P_LOAD_MS4 20u + static VkResult g5p_upload_load(struct pvr_device *device, struct pvr_g5p_programs *p, unsigned raw) { struct pco_g5p_program prog; VkResult result = VK_SUCCESS; - if (!(raw == 10 || raw == 11 ? pvr_g5p_compile_load_scalar8(&prog, raw == 11) : (raw == 8 || raw == 9) ? pvr_g5p_compile_load_packed_sint(&prog, raw == 8 ? 8 : 16) : raw == 7 ? pvr_g5p_compile_load_rgb10a2ui(&prog) : raw == 6 ? pvr_g5p_compile_load_rgba8ui(&prog) : raw == 5 ? pvr_g5p_compile_load_rgba16ui(&prog) : raw ? pvr_g5p_compile_load_raw(&prog, raw == 2) : pvr_g5p_compile_load(&prog))) { + /* raw 12..19: the compact LOAD of 5..8 targets, BGRA8 (12..15) then RGBA8 (16..19); G5P_LOAD_MS4: the 4x + * LOAD (Apple's loader, kicked at sample rate) */ + if (!(raw == G5P_LOAD_MS4 ? pvr_g5p_compile_load_ms(&prog) : raw >= 12 ? pvr_g5p_compile_load_mrt(&prog, 5 + (raw - 12) % 4, raw < 16) : + raw == 10 || raw == 11 ? pvr_g5p_compile_load_scalar8(&prog, raw == 11) : (raw == 8 || raw == 9) ? pvr_g5p_compile_load_packed_sint(&prog, raw == 8 ? 8 : 16) : raw == 7 ? pvr_g5p_compile_load_rgb10a2ui(&prog) : raw == 6 ? pvr_g5p_compile_load_rgba8ui(&prog) : raw == 5 ? pvr_g5p_compile_load_rgba16ui(&prog) : raw ? pvr_g5p_compile_load_raw(&prog, raw == 2) : pvr_g5p_compile_load(&prog))) { snprintf(p->load_refusal, sizeof(p->load_refusal), "%s", prog.refusal); free(prog.code); return VK_SUCCESS; @@ -276,13 +292,15 @@ g5p_upload_load(struct pvr_device *device, struct pvr_g5p_programs *p, unsigned for (unsigned i = 0; i < prog.num_uploads; i++) { const struct pco_g5p_upload *u = &prog.uploads[i]; if ((u->kind != PCO_G5P_UPLOAD_IMAGE && u->kind != PCO_G5P_UPLOAD_SAMPLER && - u->kind != PCO_G5P_UPLOAD_IMAGE_SIZE) || u->set || u->binding) { + u->kind != PCO_G5P_UPLOAD_IMAGE_SIZE) || u->set || + u->binding >= (raw >= 12 && raw != G5P_LOAD_MS4 ? 8u : 1u)) { snprintf(p->load_refusal, sizeof(p->load_refusal), "a load program upload of kind %u at %u.%u", u->kind, u->set, u->binding); free(prog.code); return VK_SUCCESS; } p->load_upload[i].kind = u->kind; + p->load_upload[i].binding = u->binding; p->load_upload[i].dwords = u->dwords; p->load_upload[i].common = u->common; } @@ -294,7 +312,8 @@ g5p_upload_load(struct pvr_device *device, struct pvr_g5p_programs *p, unsigned free(prog.code); if (result != VK_SUCCESS) return result; - result = g5p_upload_kick_temps(device, p->load_usc, p->load_temps, 1, &p->load_kick); + /* the 4x loader runs once per sample (Apple's DOUTU has src0[34:33] = 2, pvr_g5p_msaa_doutu()) */ + result = g5p_upload_kick_temps(device, p->load_usc, p->load_temps, raw == G5P_LOAD_MS4 ? 4 : 1, &p->load_kick); if (result != VK_SUCCESS) return result; uint32_t code[2 * ARRAY_SIZE(p->load_upload) + 2]; @@ -397,7 +416,8 @@ g5p_upload_programs(struct pvr_device *device, struct pvr_g5p_programs *p) for (unsigned i = 0; i < PVR_G5P_MRT_VARIANTS; i++) { const unsigned n = g5p_mrt_variant_targets[i]; uint8_t code[MAX2(G5P_MRT_CLEAR_USC_MAX, G5P_MRT_EOT_USC_MAX)]; - unsigned size = g5p_mrt_clear_usc(code, n); + /* variants 7..10: the RGBA8 byte order of the compact clear */ + unsigned size = i >= 7 ? g5p_mrt_clear_compact(code, n, false) : g5p_mrt_clear_usc(code, n); result = pvr_gpu_upload_usc(device, code, size, 64, &p->clear_usc[i]); if (result != VK_SUCCESS) @@ -533,10 +553,6 @@ g5p_pbe_encode(uint64_t q[3], uint32_t key, bool srgb, uint64_t va, uint32_t wid g5p_pbe_2d(q,key,srgb,va,width,height,row_stride,false); } -/* A refusal, by name, in the log too (vk_errorf() is silent in release builds) */ -#define G5P_REFUSE(cmd_buffer, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((cmd_buffer), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* * The background of a loaded colour attachment (loadOp LOAD): the load @@ -549,12 +565,9 @@ g5p_pbe_encode(uint64_t q[3], uint32_t key, bool srgb, uint64_t va, uint32_t wid * them. Apple's LOAD program is not used. */ static VkResult -g5p_load_background(struct pvr_cmd_buffer *cmd_buffer, const uint64_t state[4], uint32_t width, uint32_t height, - uint64_t bgnd[4], unsigned raw) +g5p_load_background(struct pvr_cmd_buffer *cmd_buffer, const uint64_t state[][4], uint32_t width, uint32_t height, + uint64_t bgnd[4], const struct pvr_g5p_programs *p) { - const struct pvr_g5p_programs *p = &cmd_buffer->device->g5p->programs; - if (raw) - p = p->raw_load[raw - 1]; const VkSamplerCreateInfo sci = { .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, .magFilter = VK_FILTER_NEAREST, @@ -572,7 +585,12 @@ g5p_load_background(struct pvr_cmd_buffer *cmd_buffer, const uint64_t state[4], /* textureSize()'s block (PCO_G5P_UPLOAD_IMAGE_SIZE): width, height, layers, levels */ const uint32_t size[4] = { width, height, 1, 1 }; struct pvr_suballoc_bo *state_bo, *sampler_bo, *size_bo; - VkResult result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, state, 4 * sizeof(state[0]), &state_bo); + /* one 32-byte texture state per target an IMAGE upload reads (5..8 targets: binding = target) */ + unsigned views = 1; + for (unsigned i = 0; i < p->load_uploads; i++) + if (p->load_upload[i].kind == PCO_G5P_UPLOAD_IMAGE) + views = MAX2(views, p->load_upload[i].binding + 1); + VkResult result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, state, views * sizeof(state[0]), &state_bo); if (result != VK_SUCCESS) return result; result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, sampler, sizeof(sampler), &sampler_bo); @@ -586,7 +604,7 @@ g5p_load_background(struct pvr_cmd_buffer *cmd_buffer, const uint64_t state[4], uint32_t data[4 * ARRAY_SIZE(p->load_upload)] = { 0 }; for (unsigned i = 0; i < n; i++) { const unsigned kind = p->load_upload[i].kind; - const uint64_t src = kind == PCO_G5P_UPLOAD_IMAGE ? state_bo->dev_addr.addr + const uint64_t src = kind == PCO_G5P_UPLOAD_IMAGE ? state_bo->dev_addr.addr + 32 * p->load_upload[i].binding : kind == PCO_G5P_UPLOAD_SAMPLER ? sampler_bo->dev_addr.addr : size_bo->dev_addr.addr; if (p->load_upload[i].dwords > (kind == PCO_G5P_UPLOAD_IMAGE ? 8u : kind == PCO_G5P_UPLOAD_SAMPLER ? 2u : 4u)) @@ -642,7 +660,8 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd case 1: v = 0; break; case 2: v = 1; break; case 4: v = 2; break; - default: return G5P_REFUSE(cmd_buffer, "%u colour targets (1, 2, 3 or 4 are built)", n); + case 5: case 6: case 7: case 8: v = nt - 2; break; /* compact UNORM8; RGBA8 adds 4 below */ + default: return G5P_REFUSE(cmd_buffer, "%u colour targets (1 to 8 are built)", n); } /* several targets: Apple's pass shape (pvr_g5p_mrt.h) */ VkResult mrt = pvr_arch_g5p_mrt_pass_check(cmd_buffer, hw_render); @@ -673,7 +692,7 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd return vdm; /* targets: PBE words and clear colours */ - uint64_t pbe[4][3]; + uint64_t pbe[G5P_MRT_MAX][3]; unsigned raw = 0; const VkFormat first_format = null_target ? VK_FORMAT_UNDEFINED : info->attachments[hw_render->eot_surfaces[0].attachment_idx]->vk.format; @@ -709,9 +728,14 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd g->raw_words = raw_total; g->raw_pixel_width = raw >= 5 ? source.pixel_output_width : raw_total; } - float clear[4][4] = { { 0 } }; - /* a loaded target: its texture state (q0, q1, q2, 0), the load program's image upload */ - uint64_t load_state[4] = { 0 }; + /* 5..8 targets: one packed UNORM8 dword each (pvr_g5p_mrt.h admits RGBA8 or BGRA8 only) */ + const bool compact = nt > 4; + if (compact && first_format == VK_FORMAT_R8G8B8A8_UNORM) + v += 4; + float clear[G5P_MRT_MAX][4] = { { 0 } }; + /* a loaded target: its texture state (q0, q1, q2, 0), the load program's image upload (every target's + * for 5..8 targets: the compact LOAD reads each) */ + uint64_t load_state[G5P_MRT_MAX][4] = { { 0 } }; if (null_target) { uint32_t key; bool srgb; @@ -761,6 +785,12 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd * the original signed image key. Blending is rejected for integers. */ uint32_t store_key = raw ? key & ~(0x80u << 5) : key; + /* Compact outputs already hold the exact four destination bytes. A + * physical R32UI store keeps them; the UNORM PBE key expects f16 + * source channels and would convert the packed bits. The logical + * sampling key and the four-byte pixel/row layout stay. */ + if (compact) + store_key = pvr_g5p_packed_word_pbe_key(); if (raw >= 5 && raw <= 11) { /* Equal-size raw physical view; logical sampling stays typed. * Native witness-r16 supplies the narrow16-bit source layout. */ @@ -784,7 +814,7 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd g->msaa_resolve = true; } } - if (!t) { + if (!t || compact) { if(twiddled) { struct pvr_image_descriptor desc; /* A one-layer array attachment is still a 2D store/restore @@ -794,10 +824,10 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd load_view.vk.view_type=VK_IMAGE_VIEW_TYPE_2D; const char *why=pvr_arch_g5p_pack_image_view(&load_view,&desc); if(why)return G5P_REFUSE(cmd_buffer,"twiddled LOAD view: %s",why); - load_state[0]=desc.words[0];load_state[1]=desc.words[1]; - load_state[2]=(uint64_t)desc.meta[0]|(uint64_t)desc.meta[1]<<32; + load_state[t][0]=desc.words[0];load_state[t][1]=desc.words[1]; + load_state[t][2]=(uint64_t)desc.meta[0]|(uint64_t)desc.meta[1]<<32; } else - pvr_g5p_tex_linear(load_state, key, srgb, addr, iview->vk.extent.width, iview->vk.extent.height,level->pitch); + pvr_g5p_tex_linear(load_state[t], key, srgb, addr, iview->vk.extent.width, iview->vk.extent.height,level->pitch); } if (!t) { g->width = iview->vk.extent.width; @@ -834,8 +864,9 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd "%ux%u): no clear of the area alone", t, area->offset.x, area->offset.y, area->extent.width, area->extent.height, g->width, g->height); if (hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD) { - if (n != 1 || t) - return G5P_REFUSE(cmd_buffer, "LOAD of colour attachment %u of %u (one target is built)", t, n); + if (!compact && (n != 1 || t)) + return G5P_REFUSE(cmd_buffer, "LOAD of colour attachment %u of %u (one target, or 5..8 compact targets, " + "are built)", t, n); load = true; } if (t < n && hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_CLEAR && @@ -845,12 +876,61 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd if (load && raw > 2 && !(raw >= 5 && raw <= 11)) return G5P_REFUSE(cmd_buffer, "integer boundary qualification supports CLEAR only"); const struct pvr_g5p_programs *load_program = raw && (raw <= 2 || (raw >= 5 && raw <= 11)) ? p->raw_load[raw - 1] : p; + if (compact && load) { + /* the compact LOAD of every target (lazy, as the integer variants) */ + const unsigned slot = v - 3; /* 0..3 BGRA8, 4..7 RGBA8 */ + struct pvr_g5p_programs *programs = &device->g5p->programs; + VkResult lr = VK_SUCCESS; + simple_mtx_lock(&device->g5p->clear_lock); + if (!programs->mrt_load[slot]) { + programs->mrt_load[slot] = vk_zalloc(&device->vk.alloc, sizeof(*programs), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); + lr = programs->mrt_load[slot] ? g5p_upload_load(device, programs->mrt_load[slot], 12 + slot) + : vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + } + load_program = programs->mrt_load[slot]; + simple_mtx_unlock(&device->g5p->clear_lock); + if (lr != VK_SUCCESS) + return lr; + } + if (load && g->samples == 4) { + /* Apple's 4x loader (an iOS capture of a 4x colour load): kicked at sample rate, each instance + * samples the multisample surface at its own sample (pvr_g5p_compile_load_ms()), from the 4x texture + * state of that surface (pvr_g5p_msaa_tex_state()). One target (pvr_arch_g5p_msaa.c); integer targets + * are refused above at 4x. */ + const struct pvr_renderpass_hwsetup_eot_surface *eot = &hw_render->eot_surfaces[0]; + if (eot->need_resolve && (partial || sub_cmd->barrier_load)) + return G5P_REFUSE(cmd_buffer, "a 4x %s with a resolve: the pass stores no samples to reload", + partial ? "partial render area" : "VDM continuation"); + const struct pvr_image_view *src = info->attachments[eot->need_resolve ? eot->src_attachment_idx + : eot->attachment_idx]; + const struct pvr_image *image = pvr_image_view_get_image(src); + const struct pvr_image_plane *plane = pvr_single_plane_const(image); + const uint64_t addr = image->dev_addr.addr + plane->offset; + uint32_t key; + bool srgb; + if (image->vk.samples != VK_SAMPLE_COUNT_4_BIT || !g5p_pbe_key(src->vk.format, &key, &srgb) || addr % 64) + return G5P_REFUSE(cmd_buffer, "a 4x LOAD of %s at %#" PRIx64 " (a 64-byte based 4-sample image)", + vk_Format_to_str(src->vk.format), addr); + pvr_g5p_msaa_tex_state(load_state[0], key, addr, src->vk.extent.width, src->vk.extent.height, + vk_format_is_srgb(src->vk.format)); + load_state[0][3] = 0; + struct pvr_g5p_programs *programs = &device->g5p->programs; + VkResult lr = VK_SUCCESS; + simple_mtx_lock(&device->g5p->clear_lock); + if (!programs->ms_load) { + programs->ms_load = vk_zalloc(&device->vk.alloc, sizeof(*programs), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); + lr = programs->ms_load ? g5p_upload_load(device, programs->ms_load, G5P_LOAD_MS4) + : vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + } + load_program = programs->ms_load; + simple_mtx_unlock(&device->g5p->clear_lock); + if (lr != VK_SUCCESS) + return lr; + } if (load && !load_program->load_usc) - return G5P_REFUSE(cmd_buffer, "LOAD of a colour attachment%s: the load program: %s", - partial ? " (a partial render area)" : "", load_program->load_refusal); - if (load && g->samples == 4) - return G5P_REFUSE(cmd_buffer, "LOAD of a 4x colour attachment%s (the load program runs per pixel)", - partial ? " (a partial render area)" : ""); + return G5P_REFUSE(cmd_buffer, "LOAD of a %scolour attachment%s: the load program: %s", + g->samples == 4 ? "4x " : "", partial ? " (a partial render area)" : "", + load_program->load_refusal); g->targets = nt; /* per render: the clear floats and the PBE block, then their DOUTD data */ @@ -882,7 +962,7 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd uint32_t pbe_words[G5P_MRT_PBE_STRIDE * G5P_MRT_MAX]; /* each target's source: output half 4t (8-bit RGBA, two dwords), 8t (raw32, four words) */ const unsigned pbe_dwords = g5p_mrt_pbe_block_halves(pbe_words, nt, (const uint64_t(*)[3])pbe, - raw ? 2 * raw_words : 4); + raw ? 2 * raw_words : compact ? 2 : 4); VkResult result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, clear, 16 * nt, &floats); if (result != VK_SUCCESS) return result; @@ -915,7 +995,8 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd /* PDS_BGRND0..3: the clear kick, the clear DMA, its data; sizes */ uint64_t bgnd[4] = { 0 }; if (load) { - result = g5p_load_background(cmd_buffer, load_state, g->width, g->height, bgnd, raw); + result = g5p_load_background(cmd_buffer, (const uint64_t(*)[4])load_state, g->width, g->height, bgnd, + load_program); if (result != VK_SUCCESS) return result; } else { diff --git a/src/imagination/vulkan/pvr_arch_g5p_blend.c b/src/imagination/vulkan/pvr_arch_g5p_blend.c index d82f5e969c3..51756df0f7a 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_blend.c +++ b/src/imagination/vulkan/pvr_arch_g5p_blend.c @@ -42,8 +42,21 @@ PVR_PER_ARCH(g5p_blend_fs_key)(const struct vk_graphics_pipeline_state *state, u BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_CB_WRITE_MASKS) || BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_CB_COLOR_WRITE_ENABLES)) return "dynamic blend enables, equations, write masks or colour-write enables"; - if (targets > 4) - return "more than four colour blend targets"; + if (targets > 8) + return "more than eight colour blend targets"; + /* 5..8: the compact UNORM8 output writes every channel of + * every target unblended; blending and masks there need its tile-read + * merge, which is not built */ + if (targets > 4) { + for (unsigned i = 0; i < targets; i++) { + const struct vk_color_blend_attachment_state *a = &cb->attachments[i]; + const unsigned mask = (cb->color_write_enables & BITFIELD_BIT(i)) ? a->write_mask : 0; + if (a->blend_enable || mask != 15) + return "five to eight colour targets with blending or a partial write mask (compact MRT is " + "full-write and unblended)"; + } + return NULL; + } if (targets > 1) { bool special = false; unsigned union_mask = 0; diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 4f3ef9d9374..d96550fc5a6 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -8,20 +8,29 @@ #include "vk_pipeline.h" #include "vk_alloc.h" #include "util/ralloc.h" +#include "util/u_atomic.h" +#include "util/u_debug.h" +#include "util/u_call_once.h" +#include "util/log.h" #include "g9p_cdm.h" #include "g9p_compile.h" #include "g5p_cdm.h" #include "g5p_usc.h" #include "g5p_end_barrier.h" #include "pvr_bo.h" +#include "pvr_compute_limits.h" #include "pvr_cmd_buffer.h" #include "pvr_csb.h" #include "pds/pvr_rogue_pds_defs.h" #include "pvr_descriptor_set.h" +#include "pvr_image.h" +#include "vk_enum_to_str.h" #include "pvr_device.h" #include "pvr_physical_device.h" #include "pvr_pipeline.h" #include "pvr_g5p.h" +#include "pvr_g5p_tex_encode.h" +#include "pvr_g5p_shader_dump.h" #include "pvr_g5p_compute.h" #include "pvr_job_context.h" #include "pvr_winsys.h" @@ -35,6 +44,8 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( struct pvr_compute_pipeline *pipeline) { struct pvr_g9p_compute_state *state = &pipeline->g9p; + state->a9_compute_log = pvr_g5p_compute_log(); + pvr_g5p_shader_id(&create_info->stage, state->a9_pipeline_id); void *mem_ctx = ralloc_context(NULL); struct g9p_shader shader; nir_shader *nir; @@ -45,13 +56,65 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( &create_info->stage, g9p_spirv_options(), g9p_nir_options(), mem_ctx, &nir); - if (result != VK_SUCCESS) + if (result != VK_SUCCESS) { + pvr_g5p_shader_dump(&create_info->stage, create_info->layout, "comp", NULL, 0, "REFUSE", + "SPIR-V to NIR failed"); goto out; + } - if (!g9p_a9_compile_compute(nir, &shader, mem_ctx)) { - result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, - "G5P compute: shader not supported by the backend yet"); - goto out; + const bool compiled = g9p_a9_compile_compute(nir, &shader, mem_ctx); + for (unsigned i = 0; i < 3; i++) state->a9_log_local[i] = nir->info.workgroup_size[i]; + state->a9_log_shared = nir->info.shared_size; + if (!compiled) { + /* PVR_DEBUG=shader_dump / PVR_DEBUG=capture_stub (pvr_g5p_shader_dump.h) */ + const bool stub = pvr_g5p_capture_stub(); + char reason[sizeof(shader.a9_refusal)]; + snprintf(reason, sizeof(reason), "%s", shader.a9_refusal[0] ? shader.a9_refusal : g9p_a9_last_refusal()); + pvr_g5p_shader_dump(&create_info->stage, create_info->layout, "comp", NULL, 0, stub ? "STUB" : "REFUSE", + reason); + if (!stub) { + mesa_logw("pvr/g5p: compute pipeline refused: %s", reason); + result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, + "G5P compute: shader not supported by the backend yet"); + goto out; + } + state->a9_capture_stub = true; + mesa_logw("G5P capture stub: compute shader refused (%s): empty kernel, dispatches omitted", reason); + VkPipelineShaderStageCreateInfo stub_info; + VkShaderModuleCreateInfo stub_module; + pvr_g5p_stub_stage(VK_SHADER_STAGE_COMPUTE_BIT, &stub_info, &stub_module); + result = vk_pipeline_shader_stage_to_nir(&device->vk, pipeline->base.pipeline_flags, &stub_info, + g9p_spirv_options(), g9p_nir_options(), mem_ctx, &nir); + if (result != VK_SUCCESS) + goto out; + const bool compiled = g9p_a9_compile_compute(nir, &shader, mem_ctx); + for (unsigned i = 0; i < 3; i++) state->a9_log_local[i] = nir->info.workgroup_size[i]; + state->a9_log_shared = nir->info.shared_size; + if (!compiled) { + result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, "G5P compute: the capture stub was refused"); + goto out; + } + } else { + pvr_g5p_shader_dump(&create_info->stage, create_info->layout, "comp", NULL, 0, "ADMIT", NULL); + } + { + const unsigned threads = shader.local_size[0] * shader.local_size[1] * shader.local_size[2]; + unsigned max; + if (!pvr_compute_wg_residency_ok(&device->pdevice->dev_info, + shader.num_regs, + shader.a9_bank1_regs, + threads, + shader.a9_has_barriers, + &max)) { + const unsigned t = 8 * DIV_ROUND_UP(MAX2(shader.num_regs, 1), 8); + const unsigned f = 8 * DIV_ROUND_UP(shader.a9_bank1_regs ? shader.a9_bank1_regs : shader.num_regs, 8); + mesa_logw("G5P refuses: a %u-lane workgroup with %u bank-0 and %u bank-1 registers per lane, above " + "the %u lanes Apple's A9 driver allows for that allocation", threads, t, f, max); + result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, + "G5P compute: %u-lane workgroup above the %u-lane limit of its allocation", threads, + max); + goto out; + } } result = pvr_gpu_upload_usc(device, shader.code, shader.code_size, 64, @@ -65,8 +128,9 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( state->buffers[i].binding = shader.buffers[i].binding; } state->num_regs = shader.num_regs; + state->a9_bank1_regs = shader.a9_bank1_regs; state->a9_bodyless = shader.a9_bodyless; - state->a9_unit_z = shader.a9_unit_z; + state->a9_wgid_z = shader.a9_wgid_z; state->a9_local_dwords = shader.a9_local_dwords; state->a9_has_barriers = shader.a9_has_barriers; state->a9_device_fence = shader.a9_device_fence; @@ -75,12 +139,28 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( state->a9_images[i].set=shader.textures[i].set; state->a9_images[i].binding=shader.textures[i].binding; } + /* sampled textures' descriptor words (g9p_a9_tex.h) */ + STATIC_ASSERT(ARRAY_SIZE(state->a9_uploads) == G9P_A9_MAX_UPLOADS); + state->a9_num_uploads = shader.a9_num_uploads; + for (unsigned i = 0; i < shader.a9_num_uploads; i++) { + const struct g9p_a9_upload *u = &shader.a9_uploads[i]; + state->a9_uploads[i].kind = u->kind; + state->a9_uploads[i].set = u->set; + state->a9_uploads[i].binding = u->binding; + state->a9_uploads[i].element = u->element; + state->a9_uploads[i].combined = u->combined; + state->a9_uploads[i].level = u->level; + state->a9_uploads[i].common = u->common; + state->a9_uploads[i].dwords = u->dwords; + state->a9_uploads[i].value = u->value; + } + state->shared_spill_stride = shader.shared_spill_stride; state->uses_workgroup_id = shader.uses_workgroup_id; memcpy(state->local_size, shader.local_size, sizeof(state->local_size)); out: ralloc_free(mem_ctx); - return result; + return pvr_refusal_to_api(result); } static void @@ -92,6 +172,103 @@ emit_record(struct pvr_csb *csb, const struct g5p_cdm_record *rec) memcpy(w, rec->w, sizeof(rec->w)); } +/* Capture complete bound compute resources, including a stub's original + * descriptors. No descriptor or pipeline pointers survive command recording. */ +static VkResult +g5p_record_dispatch(struct pvr_cmd_buffer *cmd, struct pvr_sub_cmd_compute *sub, + uint64_t indirect_va, uint64_t indirect_offset, const uint32_t groups[3]) +{ + const struct pvr_g9p_compute_state *s = &cmd->state.compute_pipeline->g9p; + struct pvr_g5p_dispatch_log *l = calloc(1, sizeof(*l)); + if (!l) return vk_error(cmd, VK_ERROR_OUT_OF_HOST_MEMORY); + size_t size; + FILE *f = open_memstream(&l->line, &size); + if (!f) { free(l); return vk_error(cmd, VK_ERROR_OUT_OF_HOST_MEMORY); } + memcpy(l->pipeline, s->a9_pipeline_id, sizeof(l->pipeline)); + l->stub = s->a9_capture_stub; + fprintf(f, "pipe=%s stub=%u local=%ux%ux%u shared=%u ", l->pipeline, l->stub, + s->a9_log_local[0], s->a9_log_local[1], s->a9_log_local[2], s->a9_log_shared); + if (indirect_va) + fprintf(f, "groups=indirect buffer_va=%#" PRIx64 " offset=%" PRIu64 " args_va=%#" PRIx64, + indirect_va - indirect_offset, indirect_offset, indirect_va); + else + fprintf(f, "groups=%ux%ux%u", groups[0], groups[1], groups[2]); + fprintf(f, " res=["); + const struct pvr_descriptor_state *desc = &cmd->state.compute_desc_state; + const char *sep = ""; + for (unsigned si = 0; si < PVR_MAX_DESCRIPTOR_SETS; si++) { + const struct pvr_descriptor_set *set = desc->sets[si]; + /* a set of dynamic buffers only has no descriptor memory (no mapping) */ + if (!set || !set->layout) continue; + for (unsigned bi = 0; bi < set->layout->binding_count; bi++) { + const struct pvr_descriptor_set_layout_binding *b = &set->layout->bindings[bi]; + if (!(b->stage_flags & VK_SHADER_STAGE_COMPUTE_BIT)) continue; + if (!set->mapping && !pvr_descriptor_set_binding_is_dynamic(b)) continue; + for (unsigned e = 0; e < b->descriptor_count; e++) { + const uint8_t *d = set->mapping ? (uint8_t *)set->mapping + b->offset + e * b->stride : NULL; + switch (b->type) { + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: { + struct pvr_buffer_descriptor buf; + if (b->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC || + b->type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) + buf = set->dynamic_buffers[b->dynamic_buffer_idx + e]; + else + memcpy(&buf, d, sizeof(buf)); + fprintf(f, "%sbuffer %u/%u[%u] VA=%#" PRIx64 " range=%u", sep, si, bi, e, buf.addr + buf.offset, buf.size); + sep = "; "; + break; + } + case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: + case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: { + const struct pvr_image_view *v; + uintptr_t ptr; + memcpy(&ptr, d + 24, sizeof(ptr)); + v = (const struct pvr_image_view *)ptr; + if (!v) { + fprintf(f, "%simage %u/%u[%u] unbound", sep, si, bi, e); + } else { + const struct pvr_image *im = pvr_image_view_get_image(v); + uint64_t q[3]; + memcpy(q, d, sizeof(q)); + const bool linear = !(q[0] & PVR_G5P_TEX_TWIDDLED); + uint64_t va = (q[1] & ((UINT64_C(1) << 39) - 1) & ~UINT64_C(31)) << 1; + if (linear && (q[1] >> 63)) va |= ((q[2] >> 58) & 3) << 4; + fprintf(f, "%s%s %u/%u[%u] VA=%#" PRIx64 " bytes=%" PRIu64 + " %ux%ux%u format=%s view=%s", sep, + b->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE ? "storage" : "sampled", + si, bi, e, va, im->total_size, + u_minify(im->vk.extent.width, v->vk.base_mip_level), + u_minify(im->vk.extent.height, v->vk.base_mip_level), v->vk.layer_count, + vk_Format_to_str(v->vk.format), vk_ImageViewType_to_str(v->vk.view_type)); + if (b->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) + fprintf(f, " dim=%#x", (q[0] & 7) == PVR_G5P_TEX_LAYOUT_2D_ARRAY ? 0xf7 : 0xf5); + } + sep = "; "; + break; + } + default: break; + } + } + } + } + fprintf(f, "]"); + const bool failed = ferror(f); + if (fclose(f) || failed) { free(l->line); free(l); return vk_error(cmd, VK_ERROR_OUT_OF_HOST_MEMORY); } + if (sub->g5p_log_last) sub->g5p_log_last->next = l; + else sub->g5p_log_first = l; + sub->g5p_log_last = l; + sub->g5p_log_count++; + return VK_SUCCESS; +} + +/* This translation unit is A9-only. Read the override once, off the hot path. */ +/* Bound per-dispatch shared-memory spill allocations to 64 MiB. */ +#define PVR_G5P_SHARED_SPILL_MAX_BYTES (64u << 20) + /* * One dispatch: a DMA task loading the buffer addresses (from the bound * descriptor sets) and the thread count into the common store, then the @@ -101,7 +278,7 @@ emit_record(struct pvr_csb *csb, const struct g5p_cdm_record *rec) VkResult PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd_compute *sub_cmd, - uint64_t indirect_va, const uint32_t base_group[static 3], + uint64_t indirect_va, uint64_t indirect_offset, const uint32_t base_group[static 3], const uint32_t group_count[static 3]) { struct pvr_device *device = cmd_buffer->device; @@ -114,34 +291,33 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, const uint32_t hw_threads = g5p_cdm_workgroup_threads(threads); const uint32_t count = threads | threads << 16; struct pvr_suballoc_bo *uniforms_bo; - uint32_t uniforms[2 * G9P_MAX_BUFFERS + 4] = { 0 }; - uint32_t bind_data[4 * (G9P_MAX_BUFFERS + 1) + 4] = { 0 }; - uint32_t bind_code[G9P_MAX_BUFFERS + 5]; + uint32_t uniforms[2 * G9P_MAX_BUFFERS + 4 + 8 * G9P_A9_MAX_UPLOADS] = { 0 }; + uint32_t bind_data[4 * (G9P_MAX_BUFFERS + 1) + 4 + 3 * G9P_A9_MAX_UPLOADS] = { 0 }; + uint32_t bind_code[G9P_MAX_BUFFERS + 5 + G9P_A9_MAX_UPLOADS]; uint32_t kernel_data[12] = { 0 }, kernel_code[8]; + bool from_uniforms[G9P_MAX_BUFFERS] = { false }; struct pvr_pds_upload bind, kernel; unsigned bind_dwords, ctrl0, nc = 0; VkResult result; + if (state->a9_compute_log) { + result = g5p_record_dispatch(cmd_buffer, sub_cmd, indirect_va, indirect_offset, group_count); + if (result != VK_SUCCESS) return result; + } + if (base_group[0] || base_group[1] || base_group[2]) - return vk_errorf(cmd_buffer, VK_ERROR_FEATURE_NOT_PRESENT, "G5P: base dispatch not implemented yet"); + return vk_errorf(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY, "G5P: base dispatch not implemented yet"); if (indirect_va && ((indirect_va & 3) || indirect_va >> 40)) - return vk_error(cmd_buffer, VK_ERROR_FEATURE_NOT_PRESENT); + return vk_error(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY); if (!indirect_va && (!group_count[0] || !group_count[1] || !group_count[2])) return VK_SUCCESS; if (!threads || threads > 512 || group_count[0] > 65536 || group_count[1] > 65536 || group_count[2] > 65536) - return vk_error(cmd_buffer, VK_ERROR_FEATURE_NOT_PRESENT); + return vk_error(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY); /* A bodyless shader has no observable invocation. Apple's standalone * empty program does not retire under our CDM launch contract, so do not * launch it: the sub-command's ordinary completion tail still establishes * command-buffer ordering and signals the submit fence. */ - /* The compiler's unit-Z specialization is valid only for these actual - * dispatch dimensions. Indirect dimensions cannot be checked here. */ - if (state->a9_unit_z && (indirect_va || group_count[2] != 1)) { - return vk_errorf(cmd_buffer, VK_ERROR_FEATURE_NOT_PRESENT, - "compute workgroup Z specialization requires a direct dispatch with one Z group"); - } - if (state->a9_bodyless) { /* An indirect bodyless dispatch has no consumer of its dimensions, * so its argument read is unobservable and is omitted like the kernel @@ -152,11 +328,68 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, return VK_SUCCESS; } + struct pvr_bo *shared_spill = NULL; + if (state->shared_spill_stride) { + if (indirect_va) { + if (p_atomic_cmpxchg(&state->shared_spill_logged, 0, 1) == 0) + mesa_logw("G5P refuses: indirect dispatch of a spilling workgroup-memory pipeline"); + vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + + uint64_t groups = (uint64_t)group_count[0] * group_count[1] * group_count[2]; + /* Divide before multiplying: even a valid grid can overflow bytes. */ + if (groups > PVR_G5P_SHARED_SPILL_MAX_BYTES / state->shared_spill_stride) { + if (p_atomic_cmpxchg(&state->shared_spill_logged, 0, 1) == 0) + mesa_logw("G5P refuses: a dispatch of %" PRIu64 " groups ร— %u bytes of spilled workgroup memory " + "(%.2f MB, above the cap)", groups, state->shared_spill_stride, + (double)groups * state->shared_spill_stride / (1024 * 1024)); + vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + /* bo_list is freed as pvr_suballoc_bo, despite its pvr_bo comment. + * Give the dedicated generic BO a normal ownership wrapper. Reset + * may cache it in suballoc_general, so use that allocator's mapping + * flags too; a later ordinary upload must find a mapped BO. */ + struct pvr_suballoc_bo *owner = vk_zalloc(&device->vk.alloc, sizeof(*owner), 8, + VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + if (!owner) { + vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + result = pvr_bo_alloc(device, device->heaps.general_heap, + groups * state->shared_spill_stride, 64, + PVR_BO_ALLOC_FLAG_CPU_MAPPED, &shared_spill); + if (result != VK_SUCCESS) { + vk_free(&device->vk.alloc, owner); + vk_command_buffer_set_error(&cmd_buffer->vk, result); + return result; + } + owner->allocator = &device->suballoc_general; + owner->bo = shared_spill; + owner->dev_addr = shared_spill->vma->dev_addr; + owner->size = groups * state->shared_spill_stride; + list_add(&owner->link, &cmd_buffer->bo_list); + } + /* * Values the DMA copies that are not in descriptor memory: the * push-constant buffer's address and the thread count. */ for (unsigned k = 0; k < n; k++) { + /* gl_NumWorkGroups (the A9 compiler's G9P_NUM_WORKGROUPS slots, g9p_compile.c): x and y in binding 0's + * slot, z in binding 1's (G9P's 496e44739dc ("pvr/g9p: gl_NumWorkGroups, workgroup ID z, storage buffer array elements in compute")). An indirect dispatch's counts are the GPU's. */ + if (state->buffers[k].set == G9P_NUM_WORKGROUPS) { + if (indirect_va) + return vk_errorf(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY, + "G5P: gl_NumWorkGroups in an indirect dispatch"); + uniforms[2 * k] = group_count[state->buffers[k].binding ? 2 : 0]; + uniforms[2 * k + 1] = state->buffers[k].binding ? 0 : group_count[1]; + } + if (state->buffers[k].set == G9P_SHARED_SPILL) { + uniforms[2 * k] = shared_spill->vma->dev_addr.addr; + uniforms[2 * k + 1] = shared_spill->vma->dev_addr.addr >> 32; + } if (state->buffers[k].set == G9P_PUSH_CONSTANTS) { const struct pvr_push_constants *pc = &cmd_buffer->state.push_consts[PVR_STAGE_ALLOCATION_COMPUTE]; @@ -164,10 +397,77 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uniforms[2 * k] = pc->dev_addr.addr; uniforms[2 * k + 1] = pc->dev_addr.addr >> 32; } + if (state->buffers[k].set < PVR_MAX_DESCRIPTOR_SETS) { + /* A dynamic buffer has no descriptor in set memory (its binding + * offset is ~0: a DMA from there reads set->dev_addr + 0xffffffff). + * Its address with the bound dynamic offset, or its range for + * arrayLength(), goes through this upload. */ + const unsigned element = (state->buffers[k].binding & ~G9P_BINDING_SIZE) >> G9P_BINDING_ELEMENT_SHIFT; + const unsigned b = state->buffers[k].binding & BITFIELD_MASK(G9P_BINDING_ELEMENT_SHIFT); + + if (!pvr_descriptor_set_dynamic_slot(desc->sets[state->buffers[k].set], b, element, + state->buffers[k].binding & G9P_BINDING_SIZE, + &uniforms[2 * k], &from_uniforms[k])) + return vk_errorf(cmd_buffer, VK_ERROR_UNKNOWN, + "G5P compute: set %u binding %u element %u holds no bound buffer", + state->buffers[k].set, b, element); + } } unsigned images=state->a9_num_images; - if(images && n>15)return vk_errorf(cmd_buffer,VK_ERROR_FEATURE_NOT_PRESENT,"G5P image LOD common31 overlaps the16th buffer pointer"); - unsigned transfers=n+1+images+(state->a9_device_fence ? 1 : 0)+(images ? 1 : 0); + if(images && n>15)return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P image LOD common31 overlaps the16th buffer pointer"); + const unsigned base_transfers=n+1+images+(state->a9_device_fence ? 1 : 0)+(images ? 1 : 0); + const unsigned tex0=2*n+1+(state->a9_device_fence ? 2 : 0)+(images ? 1 : 0); + const unsigned nup=state->a9_num_uploads; + unsigned transfers=base_transfers+nup; + /* sampled textures' words, validated as the fragment path's (pvr_arch_g5p_gfx.c), from the + * bound sets' CPU mappings into the uniforms upload, then DMAed to their common dwords */ + for(unsigned i=0;ia9_uploads[0]) *u=&state->a9_uploads[i]; + uint32_t *out=&uniforms[tex0+8*i]; + if (u->kind == G9P_A9_UP_LOD) { + out[0] = u->value; + continue; + } + if(u->kind==G9P_A9_UP_FETCH_SAMPLER) { + const uint64_t policy=pvr_g5p_fetch_sampler_word(); + memcpy(out,&policy,sizeof(policy)); + continue; + } + const struct pvr_descriptor_set *set=u->setsets[u->set] : NULL; + if(!set || !set->layout || !set->mapping || u->binding>=set->layout->binding_count) + return vk_errorf(cmd_buffer,VK_ERROR_UNKNOWN,"G5P compute: texture set %u binding %u is not bound",u->set,u->binding); + const struct pvr_descriptor_set_layout_binding *lb=&set->layout->bindings[u->binding]; + if(u->element>=MAX2(lb->descriptor_count,1)) + return vk_errorf(cmd_buffer,VK_ERROR_UNKNOWN,"G5P compute: set %u binding %u element %u past its descriptors",u->set,u->binding,u->element); + const uint32_t *d=(const uint32_t *)((const uint8_t *)set->mapping+lb->offset+(size_t)u->element*lb->stride); + switch(u->kind) { + case G9P_A9_UP_IMAGE: + if(lb->type==VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER || !pvr_g5p_tex_state_valid(d)) { + mesa_logw("G5P refuses: compute set %u binding %u: a view without A9 texture state",u->set,u->binding); + return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P compute: a view without A9 texture state"); + } + memcpy(out,d,4*8); + break; + case G9P_A9_UP_SAMPLER: + /* a combined descriptor's sampler is at +32, a standalone sampler's at +0 */ + if(lb->type!=VK_DESCRIPTOR_TYPE_SAMPLER) + d+=8; + if(!pvr_g5p_sampler_valid(d)) { + mesa_logw("G5P refuses: compute set %u binding %u: a sampler without A9 words",u->set,u->binding); + return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P compute: a sampler without A9 words"); + } + memcpy(out,d,4*2); + break; + case G9P_A9_UP_SIZE: + if(!pvr_g5p_tex_size(d,u->level,out)) { + mesa_logw("G5P refuses: compute set %u binding %u: an image size without A9 texture state",u->set,u->binding); + return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P compute: an image size without A9 texture state"); + } + break; + default: + return vk_error(cmd_buffer,VK_ERROR_UNKNOWN); + } + } if (state->a9_device_fence) { uint32_t zero=0; struct pvr_suballoc_bo *fence; @@ -178,7 +478,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, } uniforms[2 * n] = count; result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, uniforms, - sizeof(uint32_t) * (2 * n + 1 + (state->a9_device_fence ? 2 : 0) + (images ? 1 : 0)), + sizeof(uint32_t) * (tex0 + 8 * nup), &uniforms_bo); if (result != VK_SUCCESS) return result; @@ -193,15 +493,29 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uint64_t src; uint32_t dest = 2 * k, dwords = 2; - if (k < n && state->buffers[k].set != G9P_PUSH_CONSTANTS) { + if (k >= base_transfers) { + const unsigned i = k - base_transfers; + src = uniforms_bo->dev_addr.addr + 4 * (tex0 + 8 * i); + dest = state->a9_uploads[i].common; + dwords = state->a9_uploads[i].dwords; + } else if (k < n && !from_uniforms[k] && state->buffers[k].set != G9P_PUSH_CONSTANTS && + state->buffers[k].set != G9P_NUM_WORKGROUPS && + state->buffers[k].set != G9P_SHARED_SPILL) { const struct pvr_descriptor_set *set = desc->sets[state->buffers[k].set]; + /* an array's element (Zink keeps every storage buffer in one array) and the size flag (the + * descriptor's size word at byte 8, arrayLength()): G9P's 496e44739dc ("pvr/g9p: gl_NumWorkGroups, workgroup ID z, storage buffer array elements in compute"), 1d51ff5cec6 */ + const unsigned element = (state->buffers[k].binding & ~G9P_BINDING_SIZE) >> G9P_BINDING_ELEMENT_SHIFT; + const unsigned b = state->buffers[k].binding & BITFIELD_MASK(G9P_BINDING_ELEMENT_SHIFT); const struct pvr_descriptor_set_layout_binding *binding; - if (!set) + if (!set || b >= set->layout->binding_count) return vk_error(cmd_buffer, VK_ERROR_UNKNOWN); - binding = &set->layout->bindings[state->buffers[k].binding]; - /* pvr_buffer_descriptor: the address comes first */ - src = set->dev_addr.addr + binding->offset; + binding = &set->layout->bindings[b]; + if (element >= MAX2(binding->descriptor_count, 1)) + return vk_error(cmd_buffer, VK_ERROR_UNKNOWN); + /* pvr_buffer_descriptor: the address comes first, the size at byte 8 */ + src = set->dev_addr.addr + binding->offset + element * binding->stride + + (state->buffers[k].binding & G9P_BINDING_SIZE ? 8 : 0); } else if(k>n && ksets[state->a9_images[i].set]; @@ -242,9 +556,13 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, /* * Kernel task: DOUTU source (USC offset, 0x10), the local-ID control * word (temps q0 -> r0) and the workgroup-ID one (temps q0, 64 bits -> - * threadgroup words 0-1) in the coefficient task. + * threadgroup words 0-1, and z from q1's upper half -> word 3 when the + * kernel reads it) in the coefficient task. */ g9p_pds_kick_data(kernel_data, usc_offset(device, state->shader_bo)); + /* the DOUTU temps allocate bank 0 (homes up to r(num_regs - 1)); the record's register field below + * allocates bank 1, the code's own registers (g9p_cdm.h G9P_CDM_KERNEL_REGS) */ + const unsigned bank1 = state->a9_bank1_regs ? state->a9_bank1_regs : state->num_regs; kernel_data[1] = 0x10 * DIV_ROUND_UP(MAX2(state->num_regs, 1), 8); kernel_data[2] = 0x80000000; kernel_data[4] = 0x90000002; @@ -270,6 +588,22 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, kernel_code[5] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); kernel_code[6] = g9p_pds_dout(G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC); } + if (state->a9_wgid_z) { + /* Workgroup ID z, only for a kernel that reads it: AGXMetalA9's EncodeComputePDSProgram (0x207d1ea4c, flags + * bit 2) adds a DOUTW of temps q1's upper half (g9p_pds_dout(G9P_DOUTW, 1, G9P_PDS_TEMPS, ...); its source-1 + * word below: last, the common store, the destination word, the upper 32 bits) after the x/y one (all 64 + * bits), which then drops its last bit; G9P's 496e44739dc uses the same form on the A10. + * Our word is G9P_A9_TG_WGID_Z (3): word 2 is the barrier counter, whose zero pair is written first. */ + const unsigned at = state->a9_has_barriers ? 2 : 1; + kernel_data[4] = PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_DEST_COMMON_STORE | + PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_BSIZE_ALL64; + kernel_data[5] = PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_LAST_EN | + PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_DEST_COMMON_STORE | + G9P_A9_TG_WGID_Z << PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_AO_SHIFT | + PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_BSIZE_UPPER; + kernel_code[at] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, 0); + kernel_code[at + 1] = g9p_pds_dout(G9P_DOUTW, 1, G9P_PDS_TEMPS, 5, G9P_PDS_END); + } result = pvr_arch_cmd_buffer_upload_pds(cmd_buffer, kernel_data, kernel_dwords, ROGUE_CDMCTRL_KERNEL1_DATA_ADDR_ALIGNMENT, @@ -284,9 +618,10 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_csb *csb = &sub_cmd->control_stream; const struct g5p_cdm_record bind_rec = g5p_cdm_dma_task(G9P_CDM_DATA_BLOCKS(bind_dwords / 4), bind.data_offset, - bind.code_offset, (state->a9_device_fence || images) ? 4 : DIV_ROUND_UP(2 * n + 1, 16)); + bind.code_offset, (state->a9_device_fence || images || nup) ? 4 : DIV_ROUND_UP(2 * n + 1, 16)); const struct g5p_cdm_record kernel_rec = g5p_cdm_kernel( - G9P_CDM_KERNEL_REGS(state->num_regs) | G9P_CDM_ANY_CLUSTER | G9P_CDM_FENCE | + G9P_CDM_KERNEL_REGS(bank1) | G9P_CDM_ANY_CLUSTER | + G9P_CDM_FENCE | G9P_CDM_DATA_BLOCKS(kernel_dwords / 4), kernel.data_offset, kernel.code_offset, group_count[0], group_count[1], group_count[2], hw_threads, DIV_ROUND_UP(state->a9_local_dwords, 16)); @@ -516,8 +851,9 @@ PVR_PER_ARCH(g5p_compute_job_submit)(struct pvr_compute_ctx *ctx, * first block here. A chained CDM stream is refused until that form has * run on the device. */ if (cs->pvr_bo != list_first_entry(&cs->pvr_bo_list, struct pvr_bo, link)) - return vk_errorf(d, VK_ERROR_FEATURE_NOT_PRESENT, "G5P: chained CDM stream not implemented yet"); + return vk_errorf(d, VK_ERROR_OUT_OF_HOST_MEMORY, "G5P: chained CDM stream not implemented yet"); struct pvr_winsys_compute_submit_info info = { .wait = wait, + .g5p_log = sub->g5p_log_first, .g5p_log_count = sub->g5p_log_count, .frame_num = d->global_queue_present_count, .job_num = d->global_cmd_buffer_submit_count }; info.fw_stream_len = pvr_arch_g5p_compute_stream(d, pvr_csb_get_start_address(cs).addr, diff --git a/src/imagination/vulkan/pvr_arch_g5p_dgl.c b/src/imagination/vulkan/pvr_arch_g5p_dgl.c index e732e4135c9..10d9d90b154 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_dgl.c +++ b/src/imagination/vulkan/pvr_arch_g5p_dgl.c @@ -19,7 +19,7 @@ pvr_g5p_dgl_output_words(uint8_t *output, size_t capacity, const struct pvr_g5p_ { uint8_t code[10 * G5P_USC_MAX]; size_t size = 0; - if (!output || !sources || write_mask > 0xf || (words != 1 && words != 2 && words != 4 && words != 8) || (words != 4 && write_mask != 0xf)) + if (!output || !sources || write_mask > 0xf || words < 1 || words > 8 || words == 3 || (words != 4 && write_mask != 0xf)) return 0; if (write_mask != 0xf) { /* RGBA32 tile storage is four words. Preserve unmasked raw channels; diff --git a/src/imagination/vulkan/pvr_arch_g5p_ds.c b/src/imagination/vulkan/pvr_arch_g5p_ds.c index 13f01fdd4c9..909c689f8f9 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_ds.c +++ b/src/imagination/vulkan/pvr_arch_g5p_ds.c @@ -12,6 +12,7 @@ #include #include "util/log.h" +#include "pvr_g5p.h" #include "vk_format.h" #include "vk_graphics_state.h" #include "vk_log.h" @@ -22,15 +23,12 @@ #include "pvr_cmd_buffer.h" #include "pvr_csb.h" #include "pvr_g5p_ds.h" +#include "pvr_g5p_shader_dump.h" #include "pvr_g5p_tex_encode.h" #include "pvr_hw_pass.h" #include "pvr_image.h" #include "pvr_job_render.h" -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_DS_REFUSE(cmd_buffer, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((cmd_buffer), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) VkResult PVR_PER_ARCH(g5p_ds_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_renderpass_hwsetup_render *hw_render) @@ -45,38 +43,35 @@ PVR_PER_ARCH(g5p_ds_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_r struct pvr_g5p_ds_plane plane; if (!f) - return G5P_DS_REFUSE(cmd_buffer, "depth/stencil format %s (the ZLS stores D16, D32F, S8, D32F_S8)", + return G5P_REFUSE(cmd_buffer, "depth/stencil format %s (the ZLS stores D16, D32F, S8, D32F_S8)", vk_Format_to_str(iview->vk.format)); struct pvr_g5p_ds_chain chain; const bool sampled_chain = pvr_g5p_ds_sampled_chain(image, &chain); if (sampled_chain && f->stencil && (iview->vk.extent.width < 32 || iview->vk.extent.height < 32)) - return G5P_DS_REFUSE(cmd_buffer, "small combined depth/stencil chain attachment needs Apple's temporary ZLS plane"); + return G5P_REFUSE(cmd_buffer, "small combined depth/stencil chain attachment needs Apple's temporary ZLS plane"); if (iview->vk.base_mip_level || iview->vk.base_array_layer) - return G5P_DS_REFUSE(cmd_buffer, "a depth/stencil view of level %u layer %u (level 0 layer 0 are built)", + return G5P_REFUSE(cmd_buffer, "a depth/stencil view of level %u layer %u (level 0 layer 0 are built)", iview->vk.base_mip_level, iview->vk.base_array_layer); /* upstream's own test (pvr_arch_hw_pass.c): a resolve mode; the index is set only with one */ if (hw_render->depth_resolve_mode || hw_render->stencil_resolve_mode) - return G5P_DS_REFUSE(cmd_buffer, "a depth/stencil resolve (no capture)"); + return G5P_REFUSE(cmd_buffer, "a depth/stencil resolve (no capture)"); if (hw_render->eot_surface_count > 1) - return G5P_DS_REFUSE(cmd_buffer, "depth/stencil beside %u colour targets (no capture of the combination)", + return G5P_REFUSE(cmd_buffer, "depth/stencil beside %u colour targets (no capture of the combination)", hw_render->eot_surface_count); if (hw_render->eot_surface_count) { const struct pvr_image_view *c = info->attachments[hw_render->eot_surfaces[0].attachment_idx]; if (c->vk.extent.width != iview->vk.extent.width || c->vk.extent.height != iview->vk.extent.height) - return G5P_DS_REFUSE(cmd_buffer, "a depth/stencil attachment of %ux%u beside a %ux%u colour target " + return G5P_REFUSE(cmd_buffer, "a depth/stencil attachment of %ux%u beside a %ux%u colour target " "(ZLS_PIXELS describe one surface)", iview->vk.extent.width, iview->vk.extent.height, c->vk.extent.width, c->vk.extent.height); } if (!pvr_g5p_ds_layout(iview->vk.format, image->vk.extent.width, image->vk.extent.height, image->vk.samples, &plane) || image->total_size < (sampled_chain ? chain.size : plane.size)) - return G5P_DS_REFUSE(cmd_buffer, "a depth/stencil image of %" PRIu64 " bytes, the ZLS planes take %" PRIu64 + return G5P_REFUSE(cmd_buffer, "a depth/stencil image of %" PRIu64 " bytes, the ZLS planes take %" PRIu64 " (pvr_g5p_ds_layout())", (uint64_t)image->total_size, plane.size); - if (image->vk.samples == VK_SAMPLE_COUNT_4_BIT && - (hw_render->depth_init == VK_ATTACHMENT_LOAD_OP_LOAD || hw_render->stencil_init == VK_ATTACHMENT_LOAD_OP_LOAD || - hw_render->depth_store || hw_render->stencil_store)) - return G5P_DS_REFUSE(cmd_buffer, "a load or store of a 4-sample depth/stencil attachment " - "(no multisample ZLS capture: it stays on chip)"); + /* 4 samples: loaded and stored as at one sample over the 4x plane (pvr_g5p_ds_zls(), iOS + * captures of 4x depth store and load, and of 4x stencil) */ return VK_SUCCESS; } diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 13d8b312102..68d7fc3b21b 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -9,7 +9,7 @@ * static arena are not used, so a draw is only what pvr_* records here plus * the render job of pvr_arch_g5p.c. * - * The first form this drew (docs: orchard-mesa-pr-plan.md, rung A): one + * The first form this drew (the initial triangle path): one * linear colour target, no depth/stencil, one vertex buffer, non-indexed * single-instance draws, scissors and partial render areas as the ICD clips * them (region clip, ISP scissor table); blending as @@ -45,6 +45,7 @@ #include "pvr_cmd_buffer.h" #include "pvr_csb.h" #include "pvr_device.h" +#include "pvr_physical_device.h" #include "pvr_framebuffer.h" #include "pvr_g5p.h" #include "pvr_g5p_vdm.h" @@ -55,6 +56,8 @@ #include "pvr_g5p_tex_encode.h" #include "pvr_g5p_upload.h" #include "pvr_g5p_xfb.h" +#include "pvr_g5p_shader_dump.h" +#include "pvr_g9p.h" #include "pvr_g5p_ds.h" #include "pvr_g5p_msaa.h" #include "pvr_g5p_vertex.h" @@ -77,10 +80,6 @@ #define G5P_TILE 32 /* USC programs have byte sizes and need not end on a dword boundary. */ -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_REFUSE(obj, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* ---- PDS programs (pdsc, Apple's DOUT form: the A9 ICD's) ---- */ @@ -333,9 +332,12 @@ launch_program(uint32_t *dest) * the TA vertex, which drops x, y, z: dword d is TA dword d - 3, in 16-bit * units). Returns the data dwords; *code_dwords the code's. */ +/* pvr_physical_device.c reports what the fragment compiler admits */ +static_assert(4 * PCO_G5P_MAX_FS_VARYINGS == 112, "G5P varying components"); + static unsigned varying_program(const struct pco_g5p_program *fs, const struct pco_g5p_program *vs, uint32_t *data, - uint32_t *code, unsigned *code_dwords) + uint32_t *code, unsigned *code_dwords, unsigned pv) { pdsc_init(); pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-varying"); @@ -349,8 +351,10 @@ varying_program(const struct pco_g5p_program *fs, const struct pco_g5p_program * const bool flat = v->interp == PCO_G5P_INTERP_FLAT; item[items++] = (struct pdsc_douti_src0){ .dest = v->set, - /* PPP_CTRL flatshade_vtx is VERTEX_0 */ - .shademodel = flat ? PDSC_DOUTI_SHADEMODEL_FLAT_VERTEX0 : PDSC_DOUTI_SHADEMODEL_GOURAUD, + /* flat: the triangle's vertex pv (0, or 1/2 for a fan / the last provoking vertex: + * PVR_G5P_FLATSHADE_CONTROL(), G9P's c74fd92b9fc ("pvr/g9p: flat varyings") + 5a7204430d1) */ + .shademodel = flat ? (enum pdsc_douti_shademodel)(PDSC_DOUTI_SHADEMODEL_FLAT_VERTEX0 + pv) + : PDSC_DOUTI_SHADEMODEL_GOURAUD, .size = PDSC_DOUTI_SIZE_4D, .perspective = v->interp == PCO_G5P_INTERP_SMOOTH, .f32_offset = off, @@ -391,8 +395,8 @@ g5p_native_bresenham_line(const struct vk_graphics_pipeline_state *s) s->rs->line.mode == VK_LINE_RASTERIZATION_MODE_BRESENHAM; } -/* What rung A draws: refused by name otherwise (the ICD's validate_state() - * with rung A's limits) */ +/* What initial triangle path draws: refused by name otherwise (the ICD's validate_state() + * with initial triangle path's limits) */ static VkResult g5p_check_state(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *create_info, const struct vk_graphics_pipeline_state *s, struct pco_g5p_fs_key *key) @@ -506,6 +510,10 @@ g5p_check_state(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *c /* three colour targets compile as four: the fourth (the pass's null target) is written zeros */ if (colors == 3) key->color_targets = 4; + /* five to eight: one packed UNORM8 dword per target (pvr_arch_g5p_mrt_pipeline_check() admits RGBA8 or + * BGRA8, one format; pvr_arch_g5p_blend_fs_key() full-write and unblended) */ + if (colors > 4) + key->color_output = cf == VK_FORMAT_B8G8R8A8_UNORM ? PCO_G5P_COLOR_PACKED_BGRA8 : PCO_G5P_COLOR_PACKED_RGBA8; /* 32-bit integer targets: the raw32 output (four raw words, pvr_g5p_raw32_kind()); the PBE stores the * target's own channels from them */ const unsigned raw32 = colors ? pvr_g5p_raw32_kind(cf) : 0; @@ -525,6 +533,17 @@ g5p_check_state(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *c * discarding program still runs for its feedback) */ if (!colors) key->write_mask = 0; + /* Apple's masked form (discarding and non-discarding shaders under write mask RGB, + * no blending, BGRA8: kill mode 1 / 0, A none, B 7 -> FAST_PUNCH_THROUGH / OPAQUE, output mask 7; the program + * reads the tile's alpha word, then kills, and moves it to its alpha half). Enabled by default: on the + * device, masked RGB, overlap and mixed-rendering tests pass (300 s each, every frame exact, + * 0 kernel errors); without it a masked discard is skipped and a game scene + * stays black. */ + const unsigned mask_mode = 1; + key->mask_upass = mask_mode && colors == 1 && !key->blend && + key->write_mask == 0x7 && key->color_output == PCO_G5P_COLOR_F16 && + (cf == VK_FORMAT_B8G8R8A8_UNORM || cf == VK_FORMAT_R8G8B8A8_UNORM) && + (!s->ms || s->ms->rasterization_samples == VK_SAMPLE_COUNT_1_BIT) ? mask_mode : 0; return VK_SUCCESS; } @@ -544,6 +563,10 @@ g5p_pipeline_free(struct pvr_device *device, struct pvr_g5p_pipeline *g) pvr_arch_g5p_vs_uploads_destroy(device, g->vs_uploads); pvr_bo_suballoc_free(g->static_vary); pvr_bo_suballoc_free(g->static_frag); + for (unsigned i = 0; i < 2; i++) { + pvr_bo_suballoc_free(g->varying_pv[i].pvr_bo); + pvr_bo_suballoc_free(g->static_frag_pv[i]); + } g5p_pipeline_free(device, g->xfb); g5p_pipeline_free(device, g->punch); vk_free(&device->vk.alloc, g); @@ -570,7 +593,10 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo if (result != VK_SUCCESS) return result; key.punch = g->punch_variant; + g->fs_mask_upass = key.mask_upass; g->samples = s->ms ? MAX2(s->ms->rasterization_samples, VK_SAMPLE_COUNT_1_BIT) : VK_SAMPLE_COUNT_1_BIT; + /* 4x: a discard kills the four samples (Apple's 4x discard test) */ + key.ms4_kill = g->samples == 4; g->bresenham_line_probe = g5p_native_bresenham_line(s); for (uint32_t i = 0; i < create_info->stageCount; i++) { @@ -594,20 +620,65 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo key.depth_feedback = false; /* the fragment program first: the vertex program is compiled against the * varyings it reads (Apple's VertexProgramKey; Rogue's pco_rev_link_nir) */ - if (!pvr_g5p_compile_stage(fs_info, true, g->samples, NULL, &key, NULL, fs)) - return G5P_REFUSE(device, "fragment shader: %s", fs->refusal); + /* PVR_DEBUG=shader_dump / PVR_DEBUG=capture_stub (pvr_g5p_shader_dump.h) */ + VkPipelineShaderStageCreateInfo stub_info, fs_stub_info; + VkShaderModuleCreateInfo stub_module, fs_stub_module; + const uint32_t dump_samples = g->samples; + const struct pvr_g5p_dump_blob fs_blobs[] = { { "fs_key", &key, sizeof(key) }, + { "samples", &dump_samples, sizeof(dump_samples) } }; + if (!pvr_g5p_compile_stage(fs_info, true, g->samples, NULL, &key, NULL, fs)) { + const bool stub = pvr_g5p_capture_stub(); + pvr_g5p_shader_dump(fs_info, create_info->layout, "frag", fs_blobs, ARRAY_SIZE(fs_blobs), + stub ? "STUB" : "REFUSE", fs->refusal); + if (!stub) + return G5P_REFUSE(device, "fragment shader: %s", fs->refusal); + mesa_logw("G5P capture stub: fragment shader refused (%s): magenta stub", fs->refusal); + *fs = (struct pco_g5p_program){ 0 }; + fs_info = pvr_g5p_stub_stage(VK_SHADER_STAGE_FRAGMENT_BIT, &fs_stub_info, &fs_stub_module); + if (!pvr_g5p_compile_stage(fs_info, true, g->samples, NULL, &key, NULL, fs)) + return G5P_REFUSE(device, "fragment shader stub: %s", fs->refusal); + } else { + pvr_g5p_shader_dump(fs_info, create_info->layout, "frag", fs_blobs, ARRAY_SIZE(fs_blobs), "ADMIT", NULL); + } struct pco_g5p_vs_link link = { .valid = true }; /* points (Apple's g3b-points): the point size after the position; the * topology class is fixed per pipeline even when the topology is dynamic */ link.point_size = s->ia && s->ia->primitive_topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST; g->vs_point_size = link.point_size; g->polygon_mode = s->rs ? s->rs->polygon_mode : VK_POLYGON_MODE_FILL; + link.clip_distance = pvr_g5p_clip_distance_probe(); for (unsigned i = 0; i < fs->num_varyings; i++) link.interp[fs->vary[i].location] = fs->vary[i].interp; /* the transform feedback variant (pvr_g5p_xfb.h): the capturing vertex program */ + /* the vertex input state itself (the layout holds a pointer to it) and the link, for replay */ + const struct pvr_g5p_dump_blob vs_blobs[] = { { "vertex_input", s->vi, s->vi ? sizeof(*s->vi) : 0 }, + { "dynamic_strides", &vertex.dynamic_strides, 1 }, + { "vs_link", &link, sizeof(link) } }; if (g->xfb_variant ? !pvr_g5p_compile_vs_xfb(vs_info, &vertex, &link, &g->xfb_layout, vs) - : !pvr_g5p_compile_stage(vs_info, false, 1, &vertex, NULL, &link, vs)) - return G5P_REFUSE(device, "vertex shader%s: %s", g->xfb_variant ? " (transform feedback)" : "", vs->refusal); + : !pvr_g5p_compile_stage(vs_info, false, 1, &vertex, NULL, &link, vs)) { + const bool stub = pvr_g5p_capture_stub() && !g->xfb_variant; + pvr_g5p_shader_dump(vs_info, create_info->layout, g->xfb_variant ? "vert-xfb" : "vert", vs_blobs, + ARRAY_SIZE(vs_blobs), stub ? "STUB" : "REFUSE", vs->refusal); + if (!stub) + return G5P_REFUSE(device, "vertex shader%s: %s", g->xfb_variant ? " (transform feedback)" : "", vs->refusal); + mesa_logw("G5P capture stub: vertex shader refused (%s): point stub, nothing drawn", vs->refusal); + *vs = (struct pco_g5p_program){ 0 }; + vs_info = pvr_g5p_stub_stage(VK_SHADER_STAGE_VERTEX_BIT, &stub_info, &stub_module); + /* the stub writes no varyings: a fragment program reading some becomes the magenta stub too */ + if (fs->num_varyings) { + free(fs->code); + *fs = (struct pco_g5p_program){ 0 }; + fs_info = pvr_g5p_stub_stage(VK_SHADER_STAGE_FRAGMENT_BIT, &fs_stub_info, &fs_stub_module); + if (!pvr_g5p_compile_stage(fs_info, true, g->samples, NULL, &key, NULL, fs)) + return G5P_REFUSE(device, "fragment shader stub: %s", fs->refusal); + memset(link.interp, 0, sizeof(link.interp)); + } + if (!pvr_g5p_compile_stage(vs_info, false, 1, &vertex, NULL, &link, vs)) + return G5P_REFUSE(device, "vertex shader stub: %s", vs->refusal); + } else { + pvr_g5p_shader_dump(vs_info, create_info->layout, g->xfb_variant ? "vert-xfb" : "vert", vs_blobs, + ARRAY_SIZE(vs_blobs), "ADMIT", NULL); + } if (g->xfb_variant && !g->xfb_layout.buffers) return VK_SUCCESS; /* captures nothing: no variant */ /* point fill rasterizes ISP points of one size per draw (pointlinewidth): a per-vertex @@ -704,6 +775,7 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo g->vs_written_temps = vs->written_temps; g->fs_temps = fs->temps; g->vs_outputs = vs->outputs; + g->vs_clip_count = vs->clip_count; g->vs_id_mask = vs->id_mask; memcpy(g->vs_id_reg, vs->id_reg, sizeof(g->vs_id_reg)); g->fs_discard = (fs->evidence & PCO_G5P_EV_DISCARD) != 0; @@ -724,8 +796,16 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo g->fetch_count = vertex.fetch_count; memcpy(g->fetch_list, vertex.fetch_list, sizeof(g->fetch_list)); - if (!g->fetch_count && !(g->vs_id_mask & 1)) - return G5P_REFUSE(device, "a vertex shader without attributes or vertex index (no fetch-less PDS form)"); + /* A vertex program that reads no attribute and no vertex index (WGSL vertex pulling from a + * storage buffer by instance only, or constants: nfsu2) still runs once per vertex: it is + * launched with the vertex-index PDS form that a gl_VertexIndex-only program uses (proven: + * a fullscreen vertex shader), the index written to an input register the program never + * reads (r0, or the one after the instance index). Apple's vertex task always receives the + * vertex id the same way. */ + if (!g->fetch_count && !(g->vs_id_mask & 1)) { + g->vs_id_reg[0] = (g->vs_id_mask & 2) ? g->vs_id_reg[1] + 1 : 0; + g->vs_id_mask |= 1; + } /* The unified store holds every input dword: Apple's sizes fit 4 units * per 8 input dwords; above 8 temps or 8 input dwords it grows with the @@ -785,10 +865,17 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo unsigned varying_data_dwords = 0; if (fs->num_varyings || fs->w_set) { unsigned code_dwords; - varying_data_dwords = varying_program(fs, vs, data, code, &code_dwords); + varying_data_dwords = varying_program(fs, vs, data, code, &code_dwords, 0); result = pvr_gpu_upload_pds(device, data, varying_data_dwords, 16, code, code_dwords, 16, 64, &g->varying); if (result != VK_SUCCESS) return result; + for (unsigned pv = 1; g->fs_flat && pv <= 2; pv++) { + varying_program(fs, vs, data, code, &code_dwords, pv); + result = pvr_gpu_upload_pds(device, data, varying_data_dwords, 16, code, code_dwords, 16, 64, + &g->varying_pv[pv - 1]); + if (result != VK_SUCCESS) + return result; + } if (g->varying.code_offset != g->varying.data_offset + 4 * varying_data_dwords) return vk_errorf(device, VK_ERROR_INITIALIZATION_FAILED, "G5P: the coefficient code is at data + %#x, not + %#x", g->varying.code_offset - g->varying.data_offset, 4 * varying_data_dwords); @@ -866,6 +953,17 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo result = pvr_gpu_upload(device, device->heaps.general_heap, w, 24, 16, &g->static_frag); if (result != VK_SUCCESS) return result; + /* the same pointers naming the flat programs of vertices 1 and 2 */ + for (unsigned pv = 1; g->fs_flat && (fs->num_varyings || fs->w_set) && pv <= 2; pv++) { + uint32_t *v = g->static_frag_pv_words[pv - 1]; + memcpy(v, w, sizeof(g->static_frag_words)); + pvr_csb_pack (&v[5], TA_STATE_PDS_VARYINGBASE, vb) { + vb.addr = PVR_DEV_ADDR(g->varying_pv[pv - 1].data_offset); + } + result = pvr_gpu_upload(device, device->heaps.general_heap, v, 24, 16, &g->static_frag_pv[pv - 1]); + if (result != VK_SUCCESS) + return result; + } } return VK_SUCCESS; @@ -883,13 +981,7 @@ g5p_pipeline_build_punch(struct pvr_device *device, { if (g->fs_discard_tokens < 2 && !g->fs_discard_select) return VK_SUCCESS; - /* The general-blend punch variant (r39-6.1's form: PUNCH_THROUGH + - * general blend + depth write) is not built: r39-6.1 hard-hung the iPad - * in Aquarium with it, cause not yet bisected. Without the variant, - * g5p_pipeline_check_draw_variants() refuses the pipeline at creation - * when a depth/stencil state could need it, the non-fatal form r39-3 had. */ - if (g->fs_blend_mode == 4 /* general */) - return VK_SUCCESS; + /* General-blend punch variants use Apple's tile-read-before-kill order. */ struct pvr_g5p_pipeline *p = vk_zalloc(&device->vk.alloc, sizeof(*p), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!p) @@ -906,18 +998,17 @@ g5p_pipeline_build_punch(struct pvr_device *device, g5p_pipeline_free(device, p); /* A normal colour-only shader remains valid. A draw that requires the * absent punch variant refuses by name before emitting any draw state. */ - if (result == VK_ERROR_FEATURE_NOT_PRESENT) + if (result == VK_ERROR_FEATURE_NOT_PRESENT) { + g->punch_why = "the variant's compile was refused (see the refusal above)"; return VK_SUCCESS; + } } return result; } -/* Every refusal a draw could meet because of its programs is decided here, - * at pipeline creation, never while recording: a refused draw fails the - * whole command buffer (vkEndCommandBuffer), and Zink then drops the batch - * (Aquarium on r39-6: no water/background, 3 fps). The draw selects the - * punch variant from the dynamic depth/stencil state (g5p_draw()); this - * takes every state the pipeline allows: a dynamic enable may be on, a +/* Program variants are checked at creation under the selected refusal + * policy. The draw selects its punch variant from dynamic depth/stencil + * state (g5p_draw()); this takes every state the pipeline allows: a dynamic enable may be on, a * depth write needs the depth test, and a stencil test may write. */ static VkResult g5p_pipeline_check_draw_variants(struct pvr_device *device, @@ -927,8 +1018,13 @@ g5p_pipeline_check_draw_variants(struct pvr_device *device, if (state->rs && !BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_RS_RASTERIZER_DISCARD_ENABLE) && state->rs->rasterizer_discard_enable) return VK_SUCCESS; - const bool depth = !state->rp || state->rp->depth_attachment_format != VK_FORMAT_UNDEFINED; - const bool stencil = !state->rp || state->rp->stencil_attachment_format != VK_FORMAT_UNDEFINED; + /* VkRenderPass supplies attachment bits without formats; dynamic rendering + * supplies both. Include both representations when deciding which + * depth/stencil variants this pipeline can select. */ + const bool depth = !state->rp || (state->rp->attachments & MESA_VK_RP_ATTACHMENT_DEPTH_BIT) || + state->rp->depth_attachment_format != VK_FORMAT_UNDEFINED; + const bool stencil = !state->rp || (state->rp->attachments & MESA_VK_RP_ATTACHMENT_STENCIL_BIT) || + state->rp->stencil_attachment_format != VK_FORMAT_UNDEFINED; const struct vk_depth_stencil_state *ds = state->ds; #define G5P_DYN_DS(x) BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_DS_##x) const bool depth_test = depth && (G5P_DYN_DS(DEPTH_TEST_ENABLE) || (ds && ds->depth.test_enable)); @@ -937,10 +1033,22 @@ g5p_pipeline_check_draw_variants(struct pvr_device *device, #undef G5P_DYN_DS const bool testing = depth_test || stencil_test, writing = depth_write || stencil_test; const bool needs_punch = (testing && g->fs_discard_tokens > 1) || (writing && g->fs_discard_select); - if (needs_punch && !g->punch) - return G5P_REFUSE(device, "pipeline: discard under depth/stencil requires an unsupported punch-through variant"); - if (testing && (g->fs_discard || needs_punch) && g->samples != 1) - return G5P_REFUSE(device, "pipeline: discard under multisample depth/stencil is not hardware-qualified"); + /* Strict mode rejects a pipeline at creation if any permitted dynamic + * depth/stencil state requires an unavailable program variant. Tolerant + * mode admits it and skips only draws that select the missing variant. */ + /* 4x: one conditional discard kills the four samples (Apple's 4x test, pco_g5p_fs_key::ms4_kill), run + * FAST_PUNCH_THROUGH as at one sample (an iOS 4x punch-through capture); several discards are not ported */ + const char *why = needs_punch && !g->punch ? "discard under depth/stencil requires an unsupported punch-through variant" + : g->fs_discard && g->samples != 1 && (g->fs_discard_tokens != 1 || g->fs_discard_select) + ? "several discards at 4 samples (Apple's several-discard 4x form is not ported)" : NULL; + if (!why) + return VK_SUCCESS; + if (!device->pdevice->skip_refused_work) + return G5P_REFUSE(device, "pipeline: %s", why); + mesa_logw("G5P: pipeline admitted; its draws under depth/stencil will be refused: %s (discard tokens %u, " + "select %u, blend mode %u, write mask %#x, samples %u%s%s)", why, g->fs_discard_tokens, + g->fs_discard_select, g->fs_blend_mode, g->fs_write_mask, g->samples, + g->punch_why ? "; no punch variant: " : "", g->punch_why ? g->punch_why : ""); return VK_SUCCESS; } @@ -962,14 +1070,22 @@ PVR_PER_ARCH(g5p_graphics_pipeline_compile)(struct pvr_device *device, free(fs.code); if (result != VK_SUCCESS) { g5p_pipeline_free(device, g); - return result; + /* Unsupported programs fail creation unless the application explicitly + * opts into incorrect pixels. Internal pipelines always fail. */ + if (result == VK_ERROR_FEATURE_NOT_PRESENT && !pvr_g5p_internal_pipeline && + device->pdevice->skip_refused_work) { + mesa_logw("pvr/g5p: pipeline admitted without programs after a compile refusal"); + pipeline->g5p = NULL; + return VK_SUCCESS; + } + return pvr_refusal_to_api(result); } result = g5p_pipeline_build_punch(device, create_info, vk_state, g); if (result == VK_SUCCESS) result = g5p_pipeline_check_draw_variants(device, vk_state, g); if (result != VK_SUCCESS) { g5p_pipeline_free(device, g); - return result; + return pvr_refusal_to_api(result); } pipeline->g5p = g; @@ -1000,7 +1116,7 @@ PVR_PER_ARCH(g5p_graphics_pipeline_compile)(struct pvr_device *device, if (result != VK_SUCCESS) { g5p_pipeline_free(device, g); pipeline->g5p = NULL; - return result; + return pvr_refusal_to_api(result); } } } @@ -1087,7 +1203,9 @@ g5p_clear_pipeline(struct pvr_cmd_buffer *cmd_buffer, VkFormat format, uint8_t w .pStages = stages, .pVertexInputState = &vi, .pInputAssemblyState = &ia, .pViewportState = &vp, .pRasterizationState = &rs, .pMultisampleState = &ms, .pColorBlendState = &cb, .pDynamicState = &dy, .layout = gd->clear_layout }; + pvr_g5p_internal_pipeline++; result = device->vk.dispatch_table.CreateGraphicsPipelines(vk_dev, VK_NULL_HANDLE, 1, &gci, NULL, &pipe); + pvr_g5p_internal_pipeline--; pvr_g5p_clear_nir_free(vs); pvr_g5p_clear_nir_free(fs); if (result != VK_SUCCESS) { @@ -1423,6 +1541,17 @@ fs_uploads(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline *g, result = pvr_arch_g5p_upload_buffer(cmd_buffer, u->set, u->binding, u->element, &addr, &size); if (result != VK_SUCCESS) return result; + if (u->kind == PCO_G5P_UPLOAD_SSBO_ADDR && u->offset == PCO_G5P_SSBO_SIZE_OFFSET) { + /* get_ssbo_size: the bound size in bytes */ + const uint32_t sz[2] = { (uint32_t)MIN2(size, UINT32_MAX), 0 }; + if (u->dwords > 2) + return G5P_REFUSE(cmd_buffer, "fragment storage buffer size: %u dwords", u->dwords); + result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, sz, sizeof(sz), &bo); + if (result != VK_SUCCESS) + return result; + src[i] = bo->dev_addr.addr; + continue; + } if (u->dwords != 2 || (u->kind == PCO_G5P_UPLOAD_UBO_ADDR && u->offset >= size)) return G5P_REFUSE(cmd_buffer, "fragment buffer address: %u dwords at offset %u of %" PRIu64 " bytes", u->dwords, u->offset, size); @@ -1454,12 +1583,16 @@ fs_uploads(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline *g, continue; } switch (u->kind) { - case PCO_G5P_UPLOAD_IMAGE: + case PCO_G5P_UPLOAD_IMAGE: { if (!pvr_g5p_tex_state_valid(desc)) return G5P_REFUSE(cmd_buffer, "set %u binding %u: a view without A9 texture state", u->set, u->binding); memcpy(out, desc, 4 * u->dwords); + const char *why = u->dwords == 8 ? pvr_g5p_tex_state_for(out, u->offset) : NULL; + if (why) + return G5P_REFUSE(cmd_buffer, "set %u binding %u: %s", u->set, u->binding, why); break; + } case PCO_G5P_UPLOAD_SAMPLER: /* a combined descriptor's sampler is at +32 (the upload's offset), * a standalone sampler's at +0 */ @@ -1557,6 +1690,95 @@ g5p_record_ds_usage(struct pvr_sub_cmd_gfx *sub, const struct vk_dynamic_graphic } } +/* ---- unaligned vertex bindings (G9P's 52e91eea418 ("pvr/g9p: copy unaligned vertex bindings at submission, linear blits"), its pvr_arch_g9p_gfx.c realign) ---- + * The DDMAD fetch reads dwords from base + index * stride. A binding whose address or stride is not a multiple + * of 4 draws from a copy whose elements start on dwords: new stride = align(max(stride, 16), 4), room for the + * widest attribute past a small stride (GL's byte vec4 of stride 2). The bytes inside an element keep their + * offsets; an attribute at an unaligned offset is shifted out in the vertex program (pvr_g5p_vertex_nir.c). + * The copy runs when the job is submitted (pvr_arch_queue.c drains sub_cmd->gfx.g9p_realigns): a transfer + * recorded before the draw (they run on the CPU at submission) may be what fills the source; a source the GPU + * can write waits for the jobs before it. */ +static uint32_t +g5p_realigned_stride(uint32_t stride) +{ + return stride ? align(MAX2(stride, 16), 4) : 0; +} + +static uint8_t * +g5p_cpu_addr(struct pvr_device *device, uint64_t addr, uint64_t *available) +{ + uint8_t *p = NULL; + *available = 0; + simple_mtx_lock(&device->g9p_vmas.lock); + util_dynarray_foreach (&device->g9p_vmas.vmas, struct pvr_winsys_vma *, v) { + struct pvr_winsys_vma *vma = *v; + if (addr < vma->dev_addr.addr || addr >= vma->dev_addr.addr + vma->mapped_size) + continue; + if (!vma->bo->map && device->ws->ops->buffer_map(vma->bo, NULL) != VK_SUCCESS) + break; + const uint64_t offset = addr - vma->dev_addr.addr; + p = (uint8_t *)vma->bo->map + vma->bo_offset + offset; + *available = MIN2(vma->mapped_size - offset, vma->bo->size - vma->bo_offset - offset); + break; + } + simple_mtx_unlock(&device->g9p_vmas.lock); + return p; +} + +static VkResult +g5p_realign_binding(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_vertex_binding *vb, uint32_t stride, + uint32_t extra, uint64_t *addr) +{ + uint64_t available; + const uint8_t *src = g5p_cpu_addr(cmd_buffer->device, vb->buffer->dev_addr.addr + vb->offset, &available); + if (!src) + return G5P_REFUSE(cmd_buffer, "an unaligned vertex binding without a CPU mapping to copy it from"); + const uint32_t new_stride = g5p_realigned_stride(stride); + uint64_t size = vb->size == VK_WHOLE_SIZE ? vb->buffer->vk.size - vb->offset : vb->size; + size = MIN2(size, available); + const uint64_t count = stride ? size / stride + 1 : 1; + /* the sub-command's draws from the same binding share one copy (all are copied at submission) */ + if (cmd_buffer->vk.level != VK_COMMAND_BUFFER_LEVEL_SECONDARY) { + util_dynarray_foreach (&cmd_buffer->state.current_sub_cmd->gfx.g9p_realigns, struct g9p_realign, e) { + if (e->src == src && e->size == size && e->stride == stride && e->extra >= extra && e->dst_addr) { + *addr = e->dst_addr; + return VK_SUCCESS; + } + } + } + /* the elements, then zeros for the fetches' reads past the last one (extra: its fetch extent) */ + const uint64_t copy_size = (uint64_t)MAX2(new_stride, 16) * count + extra; + if (copy_size > 64u << 20) + return G5P_REFUSE(cmd_buffer, "an unaligned vertex binding of %" PRIu64 " bytes to copy", copy_size); + uint8_t *zero = calloc(1, copy_size); + if (!zero) + return vk_error(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY); + struct pvr_suballoc_bo *bo; + VkResult result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, zero, copy_size, &bo); + free(zero); + if (result != VK_SUCCESS) + return result; + *addr = bo->dev_addr.addr; + const struct g9p_realign r = { + .src = src, .dst = pvr_bo_suballoc_get_map_addr(bo), .size = size, .count = count, .stride = stride, + .new_stride = new_stride, .dst_addr = bo->dev_addr.addr, .extra = extra, + .gpu_written = vb->buffer->g9p_gpu_written || + (vb->buffer->vk.usage & (VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | + VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT | + VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT)), + }; + /* a secondary's render joins its primary's job: copied now */ + if (cmd_buffer->vk.level == VK_COMMAND_BUFFER_LEVEL_SECONDARY) { + for (uint64_t i = 0; i < r.count; i++) + if (i * r.stride < r.size) + memcpy(r.dst + i * r.new_stride, r.src + i * r.stride, + MIN2(MAX2(r.new_stride, 16), r.size - i * r.stride)); + } else { + util_dynarray_append_typed(&cmd_buffer->state.current_sub_cmd->gfx.g9p_realigns, struct g9p_realign, r); + } + return VK_SUCCESS; +} + VkResult PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_draw *draw) { @@ -1571,7 +1793,7 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g bool lines; if (!g) - return G5P_REFUSE(cmd_buffer, "a draw without a G5P graphics pipeline"); + return G5P_REFUSE(cmd_buffer, "a draw of a pipeline without G5P programs (refused at creation, named there)"); /* transform feedback: while capturing, the pipeline's capturing variant * (pvr_g5p_xfb.h); a pipeline that captures nothing draws as it is */ const bool xfb = g->xfb && pvr_arch_g5p_xfb_capturing(cmd_buffer); @@ -1645,9 +1867,9 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g return G5P_REFUSE(cmd_buffer, "discard under depth/stencil requires an unsupported punch-through variant"); g = g->punch; } - if (testing && g->fs_discard && g->samples != 1) - return G5P_REFUSE(cmd_buffer, "discard under multisample depth/stencil is not hardware-qualified"); } + if (g->fs_discard && g->samples != 1 && (g->fs_discard_tokens != 1 || g->fs_discard_select)) + return G5P_REFUSE(cmd_buffer, "several discards at 4 samples (Apple's several-discard 4x form is not ported)"); result = pvr_arch_g5p_vertex_draw_check(cmd_buffer, g, draw); if (result != VK_SUCCESS) return result; @@ -1715,9 +1937,8 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g return result; } - /* flat varyings take vertex 0; a fan's first vertex is the hardware's vertex 1 */ - if (dyn->ia.primitive_topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN && g->fs_flat) - return G5P_REFUSE(cmd_buffer, "flat varyings with a triangle fan"); + /* flat varyings: a fan's triangle takes them from the vertex VDM_STATE0's flatshade_control names + * (PVR_G5P_FLATSHADE_CONTROL(): 1 for fans; the refusal "flat varyings with a triangle fan" before) */ /* a depth/stencil attachment: its tests and bias (pvr_arch_g5p_ds.c) */ /* rasterizer discard: no depth/stencil aspects (Rogue's rule), the colour-only ISP form */ const VkImageAspectFlags ds_aspects = discard ? 0 : pvr_arch_g5p_ds_aspects(cmd_buffer); @@ -1767,15 +1988,47 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g const unsigned last_id = g->vs_id_mask ? util_last_bit(g->vs_id_mask) - 1 : 0; u_foreach_bit (i, g->vs_id_mask) fdata[fetch_ids(g->fetch_count) + k++] = doutw_control(g->vs_id_reg[i], i == 1, i == last_id); + uint64_t realigned[PVR_MAX_VERTEX_INPUT_BINDINGS] = { 0 }; + /* A binding draws from a dword-aligned, zero-padded copy (g5p_realign_binding()) when its address or stride + * is off dwords, or when a fetch's whole-dword overread past the bound range could leave the buffer's GPU + * mapping (a sub-dword attribute at the very end of its BO) */ + bool copy[PVR_MAX_VERTEX_INPUT_BINDINGS] = { false }; + uint32_t extent[PVR_MAX_VERTEX_INPUT_BINDINGS] = { 0 }; for (unsigned f = 0; f < g->fetch_count; f++) { const struct pvr_g5p_fetch *fe = &g->fetch_list[f]; const struct pvr_vertex_binding *vb = &cs->vertex_bindings[fe->binding]; if (!vb->buffer) return G5P_REFUSE(cmd_buffer, "no vertex buffer bound at binding %u", fe->binding); const uint32_t stride = dyn->vi_binding_strides[fe->binding]; + const uint64_t binding_base = vb->buffer->dev_addr.addr + vb->offset; + extent[fe->binding] = MAX2(extent[fe->binding], fe->offset + 4 * fe->dwords); + if ((binding_base | stride) & 3) { + copy[fe->binding] = true; + } else if (fe->overread) { + const uint64_t range = vb->size == VK_WHOLE_SIZE ? vb->buffer->vk.size - vb->offset : vb->size; + const struct pvr_winsys_vma *vma = vb->buffer->vma; + const uint64_t gpu_end = vma && vma->bo ? vma->dev_addr.addr + vma->mapped_size : 0; + if (gpu_end && binding_base + range + fe->overread > gpu_end) + copy[fe->binding] = true; + } + } + for (unsigned f = 0; f < g->fetch_count; f++) { + const struct pvr_g5p_fetch *fe = &g->fetch_list[f]; + const struct pvr_vertex_binding *vb = &cs->vertex_bindings[fe->binding]; + uint32_t stride = dyn->vi_binding_strides[fe->binding]; + uint64_t binding_base = vb->buffer->dev_addr.addr + vb->offset; + if (copy[fe->binding]) { + if (!realigned[fe->binding]) { + result = g5p_realign_binding(cmd_buffer, vb, stride, extent[fe->binding], &realigned[fe->binding]); + if (result != VK_SUCCESS) + return result; + } + binding_base = realigned[fe->binding]; + stride = g5p_realigned_stride(stride); + } /* divisor 0: every instance reads the first instance's element */ const bool divisor0 = fe->rate == PVR_G5P_FETCH_INSTANCE_DIVISOR0; - const uint64_t base = vb->buffer->dev_addr.addr + vb->offset + fe->offset + + const uint64_t base = binding_base + fe->offset + (divisor0 ? (uint64_t)draw->first_instance * stride : 0); const uint64_t ctl = ddmad_src3(fe->dwords, fe->ao, f == g->fetch_count - 1); fdata[8 * f] = (uint32_t)base; @@ -1809,6 +2062,17 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g os.rhw_pres = true; /* points: the size follows the position (g3b-points) */ os.psprite_size_pres = g->vs_point_size; + /* user planes 0..n-1 from the declared clip distances, the vertex's + * last dwords (AGXMetalA9 VertexProgramVariant 0x207d28594: + * ~(-1 << count) into TA_OUTPUT_SEL; no cull plane) */ + os.plane0 = g->vs_clip_count > 0; + os.plane1 = g->vs_clip_count > 1; + os.plane2 = g->vs_clip_count > 2; + os.plane3 = g->vs_clip_count > 3; + os.plane4 = g->vs_clip_count > 4; + os.plane5 = g->vs_clip_count > 5; + os.plane6 = g->vs_clip_count > 6; + os.plane7 = g->vs_clip_count > 7; } PPP_BLOCK(w, 2); /* 6. ISP: the colour-only form (no depth/stencil attachment); Apple's @@ -1817,6 +2081,12 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g { struct pvr_g5p_blend_isp isp; pvr_arch_g5p_blend_isp(g->fs_blend_mode, g->fs_discard, g->fs_write_mask, discard, &isp); + /* Apple's masked form (pco_g5p_fs_key::mask_upass): no destination term in the colour math, so OPAQUE, or + * FAST_PUNCH_THROUGH with a discard, and output mask 7 */ + if (g->fs_mask_upass && !discard) + isp.pass_type = !g->fs_discard ? ROGUE_TA_PASSTYPE_OPAQUE + : ROGUE_TA_PASSTYPE_FAST_PUNCH_THROUGH; + const unsigned upass = g->fs_mask_upass && !discard ? 7 : pvr_g5p_msaa_upass(g->samples); const enum ROGUE_TA_PASSTYPE pass = g->fs_depth_feedback ? ROGUE_TA_PASSTYPE_DEPTH_FEEDBACK : isp.pass_type; /* points rasterize as sprites (g3b-points: SPRITE_01UV, front and back) */ /* triangles in line/point polygon mode: AGXMetalA9 encodeISPPrimitiveType @@ -1845,7 +2115,7 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g pvr_csb_pack (&w[1], TA_STATE_ISPCTL, ctl) { /* 0 for a program that writes no colour (pvr_g5p_blend.h), else the sample rate's user pass * (pvr_g5p_msaa.h: 15, or 7 at 4x) */ - ctl.upass = isp.upass ? pvr_g5p_msaa_upass(g->samples) : 0; + ctl.upass = isp.upass ? upass : 0; ctl.bpres = true; ctl.scenable = scissor; ctl.tagwritedisable = isp.tag_write_disable; @@ -1877,7 +2147,7 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g * Mask-zero kill programs still need feedback, with upass0. */ struct ROGUE_TA_STATE_ISPCTL ctl; ROGUE_TA_STATE_ISPCTL_unpack(&w[1], &ctl); - ctl.upass = isp.upass ? pvr_g5p_msaa_upass(g->samples) : 0; + ctl.upass = isp.upass ? upass : 0; ctl.tagwritedisable = isp.tag_write_disable; /* Rogue TA_STATE_ISPCTL: VISTEST enables accumulated sample counting; * VISREG selects the counter in ISP_OCLQRY_BASE. Match the generic @@ -2013,6 +2283,7 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g pvr_csb_pack (&w[0], VDMCTRL_VDM_STATE0, v0) { v0.vs_other_present = true; v0.cam_size = cam; + v0.flatshade_control = PVR_G5P_FLATSHADE_CONTROL(dyn); } pvr_csb_pack (&w[1], VDMCTRL_VDM_STATE3, v3) { v3.vs_pds_code_base_addr = PVR_DEV_ADDR(fetch_code->code_offset); @@ -2050,6 +2321,7 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g pvr_csb_pack (&w[0], VDMCTRL_VDM_STATE0, v0) { v0.vs_data_addr_present = true; v0.cam_size = cam; + v0.flatshade_control = PVR_G5P_FLATSHADE_CONTROL(dyn); } pvr_csb_pack (&w[1], VDMCTRL_VDM_STATE2, v2) { v2.vs_pds_data_base_addr = PVR_DEV_ADDR(fetch_data.data_offset); @@ -2058,7 +2330,14 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g } /* 4. the fragment pointers and sizes */ - emit_ppp(csb, "ppp fragment", g->static_frag->dev_addr.addr, g->static_frag_words, 6); + { + /* flat varyings from the draw's provoking vertex (PVR_G5P_FLATSHADE_CONTROL()) */ + const unsigned pv = g->static_frag_pv[0] ? PVR_G5P_FLATSHADE_CONTROL(dyn) : 0; + if (pv) + emit_ppp(csb, "ppp fragment", g->static_frag_pv[pv - 1]->dev_addr.addr, g->static_frag_pv_words[pv - 1], 6); + else + emit_ppp(csb, "ppp fragment", g->static_frag->dev_addr.addr, g->static_frag_words, 6); + } if (g->fs_num_uploads) { /* then, as in the ICD's draws, the per-draw loader data pointer */ emit_ppp(csb, "ppp uniform data", uniform_va, uniform_block, 2); diff --git a/src/imagination/vulkan/pvr_arch_g5p_layout.c b/src/imagination/vulkan/pvr_arch_g5p_layout.c index 6725acaed96..d333074bc16 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_layout.c +++ b/src/imagination/vulkan/pvr_arch_g5p_layout.c @@ -16,6 +16,8 @@ */ #include +#include +#include #include #include #include @@ -24,10 +26,12 @@ #include "vk_format.h" #include "vk_sync.h" +#include "vk_enum_to_str.h" #include "util/format/u_format.h" #include "util/log.h" #include "util/texcompress_etc.h" +#include "util/texcompress_astc.h" #include "util/u_math.h" #include "util/os_time.h" #include @@ -38,6 +42,9 @@ #include "pvr_g5p_ds.h" #include "pvr_g9p.h" #include "pvr_image.h" + +/* the most texels one compressed block holds: ASTC 12x12 (ETC2/EAC 16) */ +#define G5P_BLOCK_TEXELS 144 #include "pvr_job_transfer.h" @@ -53,6 +60,18 @@ struct g5p_surface { uint64_t image_offset; }; +/* Why the CPU transfer refused (the last refusal on this thread, for the error line) */ +static thread_local char g5p_why[192]; +static bool PRINTFLIKE(1, 2) +G5P_WHY(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vsnprintf(g5p_why, sizeof(g5p_why), fmt, ap); + va_end(ap); + return false; +} + static bool g5p_surface_init(struct pvr_device *device, const struct pvr_transfer_cmd_surface *s, struct g5p_surface *out) { @@ -89,8 +108,8 @@ g5p_surface_init(struct pvr_device *device, const struct pvr_transfer_cmd_surfac out->aspects = s->g5p_aspect; out->image_offset = s->g5p_image ? s->dev_addr.addr - s->g5p_image->dev_addr.addr - pvr_single_plane_const(s->g5p_image)->offset : 0; if (!out->base) - mesa_loge("G5P transfer: no CPU mapping for %#" PRIx64, s->dev_addr.addr); - return out->base != NULL; + return G5P_WHY("no CPU mapping of a tracked memory binding covers %#" PRIx64, s->dev_addr.addr); + return true; } /* Spread the low 16 coordinate bits into even bit positions. G5P uses @@ -126,13 +145,18 @@ g5p_offset(const struct g5p_surface *s, unsigned x, unsigned y) static uint8_t * g5p_element(struct g5p_surface *s, unsigned x, unsigned y) { - if (x >= s->width || y >= s->height) + if (x >= s->width || y >= s->height) { + G5P_WHY("element (%u,%u) outside the %ux%u surface", x, y, s->width, s->height); return NULL; + } const size_t offset = g5p_offset(s, x, y); - if (offset > s->available || s->bpp > s->available - offset) + if (offset > s->available || s->bpp > s->available - offset) { + G5P_WHY("element (%u,%u) at byte %zu + %u past the %" PRIu64 " bytes bound from its address", + x, y, offset, s->bpp, (uint64_t)s->available); return NULL; + } return s->base + offset + s->component_offset; } @@ -158,7 +182,8 @@ g5p_ds_plane(const struct g5p_surface *s, VkImageAspectFlagBits aspect, const struct pvr_image *image = s->image; const struct pvr_g5p_ds_format *own = pvr_g5p_ds_format(image->vk.format); if (!own || (stencil ? !own->stencil : !own->depth) || image->vk.samples != 1) - return false; + return G5P_WHY("depth/stencil plane of %s, %u samples (one-sample depth/stencil images only)", + vk_Format_to_str(image->vk.format), image->vk.samples); p->format = stencil ? PIPE_FORMAT_S8_UINT : own->depth_bytes == 2 ? PIPE_FORMAT_Z16_UNORM : PIPE_FORMAT_Z32_FLOAT; p->cpp = stencil ? 1 : own->depth_bytes; @@ -267,9 +292,22 @@ g5p_etc2_decoded(enum pipe_format p) } /* A 4x4 block as 16 RGBA floats, row by row */ +/* The most texels of one block: ASTC 12x12 */ static void -g5p_decode_block(enum pipe_format format, const uint8_t *p, unsigned bpp, float block[16][4]) +g5p_decode_block(enum pipe_format format, const uint8_t *p, unsigned bpp, float block[G5P_BLOCK_TEXELS][4]) { + if (util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_ASTC) { + /* util's ASTC LDR decoder: RGBA8 (sRGB-encoded for the sRGB formats), + * one block of bw x bh texels, then the 8-bit format's unpack as for + * ETC2 below */ + const unsigned bw = util_format_get_blockwidth(format), bh = util_format_get_blockheight(format); + uint8_t texels[G5P_BLOCK_TEXELS * 4]; + + _mesa_unpack_astc_2d_ldr(texels, 4 * bw, p, bpp, bw, bh, format); + util_format_unpack_rgba(util_format_is_srgb(format) ? PIPE_FORMAT_R8G8B8A8_SRGB : PIPE_FORMAT_R8G8B8A8_UNORM, + block, texels, bw * bh); + return; + } const enum pipe_format dec = g5p_etc2_decoded(format); if (dec != PIPE_FORMAT_NONE) { @@ -290,7 +328,7 @@ g5p_decode_block(enum pipe_format format, const uint8_t *p, unsigned bpp, float * the last block decoded is kept */ static bool g5p_block_texel(const struct g5p_surface *s, unsigned x, unsigned y, float out[4], const uint8_t **last, - float block[16][4]) + float block[G5P_BLOCK_TEXELS][4]) { const unsigned bw = util_format_get_blockwidth(s->format), bh = util_format_get_blockheight(s->format); struct g5p_surface b = *s; @@ -304,7 +342,7 @@ g5p_block_texel(const struct g5p_surface *s, unsigned x, unsigned y, float out[4 const uint8_t *p = g5p_element(&b, x / bw, y / bh); - assert(bw == 4 && bh == 4); + assert(bw * bh <= G5P_BLOCK_TEXELS); if (!p) return false; if (p != *last) { @@ -335,7 +373,7 @@ g5p_normalize_axis(int32_t *offset, uint32_t *extent, bool *flip) * util_format_unpack_rgba() looks it up (a call_once) on every call. */ static bool g5p_texel(struct g5p_surface *s, const struct util_format_unpack_description *unpack, unsigned x, unsigned y, - float out[4], const uint8_t **last, float block[16][4]) + float out[4], const uint8_t **last, float block[G5P_BLOCK_TEXELS][4]) { if (s->blocks) return g5p_block_texel(s, x, y, out, last, block); @@ -356,7 +394,7 @@ g5p_texel(struct g5p_surface *s, const struct util_format_unpack_description *un static bool g5p_linear_sample(struct g5p_surface *src, const struct util_format_unpack_description *unpack, const struct pvr_rect_mapping *m, unsigned x, unsigned y, float out[4], - const uint8_t **last, float block[16][4]) + const uint8_t **last, float block[G5P_BLOCK_TEXELS][4]) { const double sw = m->src_rect.extent.width, sh = m->src_rect.extent.height; const double dw = m->dst_rect.extent.width, dh = m->dst_rect.extent.height; @@ -570,7 +608,7 @@ g5p_copy(const struct pvr_rect_mapping *mapping, struct g5p_surface *src, struct const unsigned dw = m->dst_rect.extent.width, dh = m->dst_rect.extent.height; const unsigned sw = m->src_rect.extent.width, sh = m->src_rect.extent.height; const uint8_t *last = NULL; - float block[16][4]; + float block[G5P_BLOCK_TEXELS][4]; if (!dw || !dh || !sw || !sh) return true; @@ -686,8 +724,8 @@ g5p_copy(const struct pvr_rect_mapping *mapping, struct g5p_surface *src, struct util_format_unpack_s_8uint(src->format, &stencil, from, 1); util_format_pack_s_8uint(dst->format, to, &stencil, 1); } else { - mesa_loge("G5P transfer: unsupported depth/stencil conversion %u -> %u", src->format, dst->format); - return false; + return G5P_WHY("depth/stencil conversion %s -> %s", util_format_short_name(src->format), + util_format_short_name(dst->format)); } } else if (integer) { uint32_t v[4]; @@ -714,7 +752,7 @@ g5p_copy_aspects(const struct pvr_rect_mapping *mapping, struct g5p_surface *src VkImageAspectFlags aspects = src->image ? src->aspects : dst->aspects; if (src->image && dst->image) aspects &= dst->aspects; const struct pvr_g5p_ds_format *f = pvr_g5p_ds_format(image->vk.format); - if (!f) return false; + if (!f) return G5P_WHY("no G5P depth/stencil layout for %s", vk_Format_to_str(image->vk.format)); for (unsigned i = 0; i < 2; i++) { VkImageAspectFlagBits aspect = i ? VK_IMAGE_ASPECT_STENCIL_BIT : VK_IMAGE_ASPECT_DEPTH_BIT; if (!(aspects & aspect)) continue; @@ -736,7 +774,7 @@ g5p_run(struct pvr_device *device, const struct pvr_transfer_cmd *cmd) if (cmd->flags & PVR_TRANSFER_CMD_FLAGS_FILL) { if (dst.image && vk_format_is_depth_or_stencil(dst.image->vk.format)) { const struct pvr_g5p_ds_format *f = pvr_g5p_ds_format(dst.image->vk.format); - if (!f) return false; + if (!f) return G5P_WHY("no G5P depth/stencil layout for %s", vk_Format_to_str(dst.image->vk.format)); for (unsigned i = 0; i < 2; i++) { VkImageAspectFlagBits aspect = i ? VK_IMAGE_ASPECT_STENCIL_BIT : VK_IMAGE_ASPECT_DEPTH_BIT; struct g5p_surface p; @@ -767,6 +805,51 @@ g5p_run(struct pvr_device *device, const struct pvr_transfer_cmd *cmd) return true; } +/* One surface of a refused transfer, for the error line */ +static int +g5p_describe_surface(char *out, size_t n, const char *what, const struct pvr_transfer_cmd_surface *s) +{ + const struct pvr_image *im = s->g5p_image; + int k = snprintf(out, n, " %s{%s %ux%ux%u stride %u %s samples %u z %.1f va %#" PRIx64, what, + vk_Format_to_str(s->vk_format), s->width, s->height, s->depth, s->stride, + s->mem_layout == PVR_MEMLAYOUT_TWIDDLED ? "twiddled" : + s->mem_layout == PVR_MEMLAYOUT_LINEAR ? "linear" : "3d-twiddled", + s->sample_count, s->z_position, s->dev_addr.addr); + if (im && k >= 0 && (size_t)k < n) + k += snprintf(out + k, n - k, " image %s %ux%ux%u levels %u layers %u samples %u size %" PRIu64 " at +%" PRIu64, + vk_Format_to_str(im->vk.format), im->vk.extent.width, im->vk.extent.height, im->vk.extent.depth, + im->vk.mip_levels, im->vk.array_layers, im->vk.samples, (uint64_t)im->total_size, + s->dev_addr.addr - im->dev_addr.addr); + if (k >= 0 && (size_t)k < n) + k += snprintf(out + k, n - k, "}"); + return k; +} + +/* A CPU transfer G5P refuses (g5p_run() false): which surface and why */ +static void +g5p_transfer_refused(const struct pvr_transfer_cmd *cmd) +{ + char line[1024]; + int k = snprintf(line, sizeof(line), "G5P transfer refused: %s; %s scissor %ux%u+%d+%d", + g5p_why[0] ? g5p_why : "?", (cmd->flags & PVR_TRANSFER_CMD_FLAGS_FILL) ? "fill" : "copy", + cmd->scissor.extent.width, cmd->scissor.extent.height, cmd->scissor.offset.x, + cmd->scissor.offset.y); + if (k >= 0 && (size_t)k < sizeof(line)) + k += g5p_describe_surface(line + k, sizeof(line) - k, "dst", &cmd->dst); + for (unsigned i = 0; i < cmd->source_count && k >= 0 && (size_t)k < sizeof(line); i++) { + const struct pvr_transfer_cmd_source *src = &cmd->sources[i]; + k += g5p_describe_surface(line + k, sizeof(line) - k, "src", &src->surface); + if (src->mapping_count && k >= 0 && (size_t)k < sizeof(line)) { + const struct pvr_rect_mapping *m = &src->mappings[0]; + k += snprintf(line + k, sizeof(line) - k, " map[%u] %ux%u+%d+%d -> %ux%u+%d+%d", src->mapping_count, + m->src_rect.extent.width, m->src_rect.extent.height, m->src_rect.offset.x, + m->src_rect.offset.y, m->dst_rect.extent.width, m->dst_rect.extent.height, + m->dst_rect.offset.x, m->dst_rect.offset.y); + } + } + mesa_logw("%s", line); +} + VkResult PVR_PER_ARCH(g5p_transfer_submit)(struct pvr_device *device, struct pvr_sub_cmd_transfer *sub_cmd, const struct vk_sync_wait *waits, uint32_t wait_count, @@ -780,9 +863,13 @@ PVR_PER_ARCH(g5p_transfer_submit)(struct pvr_device *device, struct pvr_sub_cmd_ return result; } list_for_each_entry (struct pvr_transfer_cmd, cmd, sub_cmd->transfer_cmds, link) { - if (!g5p_run(device, cmd)) - return vk_errorf(device, VK_ERROR_DEVICE_LOST, - "G5P transfer surface exceeds its memory's mapping"); + g5p_why[0] = 0; + if (!g5p_run(device, cmd)) { + /* A transfer the CPU path cannot do is named (surface, size, format, layout and why) and skipped: + * its destination keeps what it held and the queue goes on (the game workload previously returned + * DEVICE_LOST). Decide unsupported copies at recording time in a future change. */ + g5p_transfer_refused(cmd); + } } return vk_sync_signal(&device->vk, signal_sync, 0); } diff --git a/src/imagination/vulkan/pvr_arch_g5p_meta.c b/src/imagination/vulkan/pvr_arch_g5p_meta.c index 441b12e90b6..ea09c845e3a 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_meta.c +++ b/src/imagination/vulkan/pvr_arch_g5p_meta.c @@ -24,6 +24,7 @@ #include "pvr_buffer.h" #include "pvr_cmd_buffer.h" +#include "pvr_g5p_shader_dump.h" #include "pvr_descriptor_set.h" #include "pvr_device.h" #include "pvr_g5p.h" @@ -183,7 +184,9 @@ PVR_PER_ARCH(g5p_meta_blit)(struct pvr_cmd_buffer *cmd_buffer, const VkBlitImage struct list_head *mark = cmd_buffer->sub_cmds.prev; g5p_meta_begin(cmd_buffer, save); + pvr_g5p_internal_pipeline++; vk_meta_blit_image2(&cmd_buffer->vk, &cmd_buffer->device->g5p->meta, info); + pvr_g5p_internal_pipeline--; g5p_meta_end(cmd_buffer, save); free(save); diff --git a/src/imagination/vulkan/pvr_arch_g5p_mrt.c b/src/imagination/vulkan/pvr_arch_g5p_mrt.c index 0e726498a59..bf6d34ca220 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_mrt.c +++ b/src/imagination/vulkan/pvr_arch_g5p_mrt.c @@ -25,10 +25,6 @@ #include "pvr_hw_pass.h" #include "pvr_image.h" -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_MRT_REFUSE(obj, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* Two 32-bit integer targets of one format (dEQP's ShaderExecutor: a mat2xN * output's columns, modf's two results): the raw32 output's four words each, @@ -61,19 +57,19 @@ PVR_PER_ARCH(g5p_mrt_pipeline_check)(struct pvr_device *device, unsigned n, cons * whose fourth target stores to the null target (pvr_arch_g5p.c, pvr_g5p_nir.c) */ enum g5p_mrt_refusal r = g5p_mrt_out_plan_make(n == 3 ? 4 : n, &plan); if (r != G5P_MRT_OK) - return G5P_MRT_REFUSE(device, "%u colour attachments: %s", n, g5p_mrt_refusal_str(r)); + return G5P_REFUSE(device, "%u colour attachments: %s", n, g5p_mrt_refusal_str(r)); for (unsigned i = 0; i < n; i++) { const int pair = g5p_mrt_raw32_pair(n, formats[0], formats[i]); if (pair < 0) - return G5P_MRT_REFUSE(device, "colour attachment %u of %u is %s: %s", i, n, vk_Format_to_str(formats[i]), + return G5P_REFUSE(device, "colour attachment %u of %u is %s: %s", i, n, vk_Format_to_str(formats[i]), G5P_MRT_RAW32_PROBE); if (pair) continue; if (n > 2 && pvr_g5p_raw32_kind(formats[i])) - return G5P_MRT_REFUSE(device, "%u colour attachments of %s: %s", n, vk_Format_to_str(formats[i]), + return G5P_REFUSE(device, "%u colour attachments of %s: %s", n, vk_Format_to_str(formats[i]), G5P_MRT_RAW32_WIDE); if (!pvr_g5p_mrt_format_ok(formats[i]) || formats[i] != formats[0]) - return G5P_MRT_REFUSE(device, "colour attachment %u of %u is %s, attachment 0 %s: %s", i, n, + return G5P_REFUSE(device, "colour attachment %u of %u is %s, attachment 0 %s: %s", i, n, vk_Format_to_str(formats[i]), vk_Format_to_str(formats[0]), g5p_mrt_refusal_str(G5P_MRT_MIXED_FORMATS)); } @@ -104,43 +100,55 @@ PVR_PER_ARCH(g5p_mrt_pass_check)(struct pvr_cmd_buffer *cmd_buffer, return VK_SUCCESS; /* three: Apple's 4-target pass with a padding fourth target (pvr_arch_g5p.c) */ if (n > G5P_MRT_MAX_TARGETS) - return G5P_MRT_REFUSE(cmd_buffer, "%u colour targets: %s", n, g5p_mrt_refusal_str(G5P_MRT_TOO_MANY_TARGETS)); + return G5P_REFUSE(cmd_buffer, "%u colour targets: %s", n, g5p_mrt_refusal_str(G5P_MRT_TOO_MANY_TARGETS)); for (unsigned t = 0; t < n; t++) { const uint32_t idx = hw_render->eot_surfaces[t].attachment_idx; if (idx >= info->attachment_count || !info->attachments[idx]) - return G5P_MRT_REFUSE(cmd_buffer, "colour target %u of %u has no image view", t, n); + return G5P_REFUSE(cmd_buffer, "colour target %u of %u has no image view", t, n); iview[t] = info->attachments[idx]; const struct pvr_image *image = pvr_image_view_get_image(iview[t]); const int pair = g5p_mrt_raw32_pair(n, iview[0]->vk.format, iview[t]->vk.format); if (pair < 0) - return G5P_MRT_REFUSE(cmd_buffer, "colour target %u of %u is %s: %s", t, n, + return G5P_REFUSE(cmd_buffer, "colour target %u of %u is %s: %s", t, n, vk_Format_to_str(iview[t]->vk.format), G5P_MRT_RAW32_PROBE); if (!pair && n > 2 && pvr_g5p_raw32_kind(iview[t]->vk.format)) - return G5P_MRT_REFUSE(cmd_buffer, "%u colour targets of %s: %s", n, vk_Format_to_str(iview[t]->vk.format), + return G5P_REFUSE(cmd_buffer, "%u colour targets of %s: %s", n, vk_Format_to_str(iview[t]->vk.format), G5P_MRT_RAW32_WIDE); if (!pair && (!pvr_g5p_mrt_format_ok(iview[t]->vk.format) || iview[t]->vk.format != iview[0]->vk.format)) - return G5P_MRT_REFUSE(cmd_buffer, "colour target %u of %u is %s, target 0 %s: %s", t, n, + return G5P_REFUSE(cmd_buffer, "colour target %u of %u is %s, target 0 %s: %s", t, n, vk_Format_to_str(iview[t]->vk.format), vk_Format_to_str(iview[0]->vk.format), g5p_mrt_refusal_str(G5P_MRT_MIXED_FORMATS)); if (iview[t]->vk.base_mip_level || iview[t]->vk.layer_count != 1 || image->vk.samples != VK_SAMPLE_COUNT_1_BIT) - return G5P_MRT_REFUSE(cmd_buffer, "colour target %u of %u: mip %u, %u layers, %u samples (one level, one layer, " + return G5P_REFUSE(cmd_buffer, "colour target %u of %u: mip %u, %u layers, %u samples (one level, one layer, " "one sample)", t, n, iview[t]->vk.base_mip_level, iview[t]->vk.layer_count, image->vk.samples); if (iview[t]->vk.extent.width != iview[0]->vk.extent.width || iview[t]->vk.extent.height != iview[0]->vk.extent.height) - return G5P_MRT_REFUSE(cmd_buffer, "colour target %u is %ux%u, target 0 %ux%u (one size per pass)", t, + return G5P_REFUSE(cmd_buffer, "colour target %u is %ux%u, target 0 %ux%u (one size per pass)", t, iview[t]->vk.extent.width, iview[t]->vk.extent.height, iview[0]->vk.extent.width, iview[0]->vk.extent.height); va[t] = g5p_mrt_target_va(iview[t], &bytes[t]); } - /* The background programs clear or leave every target; a LOAD would need - * an N-target LOAD background (none exists) */ + /* 5..8 compact targets: the compact LOAD background loads every target, so a pass mixing CLEAR and LOAD + * would load the cleared ones */ + if (n > 4) { + bool has_clear = false, has_load = false; + for (unsigned i = 0; i < hw_render->color_init_count; i++) { + has_clear |= hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_CLEAR; + has_load |= hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD; + } + if (has_clear && has_load) + return G5P_REFUSE(cmd_buffer, "%u colour targets mixing CLEAR and LOAD (the compact background loads or " + "clears every target)", n); + } + /* The background programs clear or leave every target; a LOAD of 2..4 targets would need an N-target LOAD + * background (none exists; 5..8 have the compact one) */ for (unsigned i = 0; i < hw_render->color_init_count; i++) - if (hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD) - return G5P_MRT_REFUSE(cmd_buffer, "LOAD of a colour attachment with %u colour targets (no N-target LOAD " + if (n <= 4 && hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD) + return G5P_REFUSE(cmd_buffer, "LOAD of a colour attachment with %u colour targets (no N-target LOAD " "background)", n); /* The clear and end-of-tile programs cover whole tiles of every target: a @@ -148,7 +156,7 @@ PVR_PER_ARCH(g5p_mrt_pass_check)(struct pvr_cmd_buffer *cmd_buffer, const VkRect2D *area = &info->render_area; if (area->offset.x || area->offset.y || area->extent.width != iview[0]->vk.extent.width || area->extent.height != iview[0]->vk.extent.height) - return G5P_MRT_REFUSE(cmd_buffer, "%u colour targets in a partial render area (%d,%d %ux%u of %ux%u)", n, + return G5P_REFUSE(cmd_buffer, "%u colour targets in a partial render area (%d,%d %ux%u of %ux%u)", n, area->offset.x, area->offset.y, area->extent.width, area->extent.height, iview[0]->vk.extent.width, iview[0]->vk.extent.height); @@ -156,7 +164,7 @@ PVR_PER_ARCH(g5p_mrt_pass_check)(struct pvr_cmd_buffer *cmd_buffer, for (unsigned a = 0; a < n; a++) for (unsigned c = a + 1; c < n; c++) if (va[a] < va[c] + bytes[c] && va[c] < va[a] + bytes[a]) - return G5P_MRT_REFUSE(cmd_buffer, "colour targets %u and %u overlap in memory", a, c); + return G5P_REFUSE(cmd_buffer, "colour targets %u and %u overlap in memory", a, c); return VK_SUCCESS; } @@ -168,13 +176,13 @@ PVR_PER_ARCH(g5p_mrt_draw_check)(struct pvr_cmd_buffer *cmd_buffer, unsigned pip const unsigned pn = MAX2(pipeline_targets, 1); if (!info->pass && !info->dr_info) - return G5P_MRT_REFUSE(cmd_buffer, "a draw outside a render pass"); + return G5P_REFUSE(cmd_buffer, "a draw outside a render pass"); /* the render of the current subpass (dynamic rendering has one) */ const struct pvr_renderpass_hwsetup_render *hw_render = pvr_arch_pass_info_get_hw_render(info, info->current_hw_subpass); /* a 3-target pass runs, and its pipelines compile, as four targets */ const unsigned rn = hw_render->eot_surface_count == 3 ? 4 : hw_render->eot_surface_count; if ((rn > 1 || pn > 1) && pn != rn) - return G5P_MRT_REFUSE(cmd_buffer, "the pipeline writes %u colour targets, the pass has %u", pn, rn); + return G5P_REFUSE(cmd_buffer, "the pipeline writes %u colour targets, the pass has %u", pn, rn); return VK_SUCCESS; } diff --git a/src/imagination/vulkan/pvr_arch_g5p_msaa.c b/src/imagination/vulkan/pvr_arch_g5p_msaa.c index 15c46265a62..a978ca033c4 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_msaa.c +++ b/src/imagination/vulkan/pvr_arch_g5p_msaa.c @@ -8,6 +8,7 @@ #include #include "util/log.h" +#include "pvr_g5p.h" #include "vk_format.h" #include "vk_log.h" @@ -16,14 +17,11 @@ #include "pvr_hw_pass.h" #include "pvr_image.h" -#define G5P_MSAA_REFUSE(cmd_buffer, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((cmd_buffer), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* - * The captured 4x pass: one colour target of 4-byte samples, cleared or - * don't-care, either stored with its samples (msaa4-store) or resolved by - * the PBE into a one-sample target at the end of each tile (msaa4-resolve), + * The captured 4x pass: one colour target of 4-byte samples, cleared, + * loaded or don't-care, either stored with its samples or resolved by + * the PBE into a one-sample target at the end of each tile, * a depth/stencil attachment on chip only. Upstream's hardware setup lists * the resolve as an end-of-tile surface with need_resolve and resolve_type * PVR_RESOLVE_TYPE_PBE. @@ -36,14 +34,12 @@ PVR_PER_ARCH(g5p_msaa_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr if (hw_render->sample_count <= 1) return VK_SUCCESS; if (hw_render->sample_count != 4) - return G5P_MSAA_REFUSE(cmd_buffer, "%u samples (1 and 4 are built)", hw_render->sample_count); + return G5P_REFUSE(cmd_buffer, "%u samples (1 and 4 are built)", hw_render->sample_count); if (hw_render->eot_surface_count != 1) - return G5P_MSAA_REFUSE(cmd_buffer, "4x with %u end-of-tile surfaces (one target, stored or resolved: a " + return G5P_REFUSE(cmd_buffer, "4x with %u end-of-tile surfaces (one target, stored or resolved: a " "STORE with a resolve needs the CPU resolve after the job)", hw_render->eot_surface_count); - for (unsigned i = 0; i < hw_render->color_init_count; i++) - if (hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_LOAD) - return G5P_MSAA_REFUSE(cmd_buffer, "LOAD of a 4-sample attachment (no multisample background loader)"); + /* a LOAD of the target: Apple's 4x loader (pvr_arch_g5p.c) */ const struct pvr_renderpass_hwsetup_eot_surface *eot = &hw_render->eot_surfaces[0]; const struct pvr_image_view *iview = info->attachments[eot->need_resolve ? eot->src_attachment_idx @@ -51,20 +47,20 @@ PVR_PER_ARCH(g5p_msaa_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr const struct pvr_image *image = pvr_image_view_get_image(iview); if (image->vk.samples != VK_SAMPLE_COUNT_4_BIT || vk_format_get_blocksize(iview->vk.format) != 4 || vk_format_is_depth_or_stencil(iview->vk.format)) - return G5P_MSAA_REFUSE(cmd_buffer, "a 4x colour target %s of %u samples (4-byte samples are built)", + return G5P_REFUSE(cmd_buffer, "a 4x colour target %s of %u samples (4-byte samples are built)", vk_Format_to_str(iview->vk.format), image->vk.samples); if (iview->vk.base_mip_level || iview->vk.base_array_layer) - return G5P_MSAA_REFUSE(cmd_buffer, "a 4-sample view of level %u layer %u", iview->vk.base_mip_level, + return G5P_REFUSE(cmd_buffer, "a 4-sample view of level %u layer %u", iview->vk.base_mip_level, iview->vk.base_array_layer); if (eot->need_resolve) { const struct pvr_image_view *r = info->attachments[eot->attachment_idx]; const struct pvr_image *ri = pvr_image_view_get_image(r); if (eot->resolve_type != PVR_RESOLVE_TYPE_PBE) - return G5P_MSAA_REFUSE(cmd_buffer, "a 4x resolve outside the PBE (transfer resolves are lane g's)"); + return G5P_REFUSE(cmd_buffer, "a 4x resolve outside the PBE (transfer resolves are lane g's)"); if (ri->vk.samples != VK_SAMPLE_COUNT_1_BIT || ri->memlayout != PVR_MEMLAYOUT_LINEAR || r->vk.format != iview->vk.format || r->vk.extent.width != iview->vk.extent.width || r->vk.extent.height != iview->vk.extent.height || r->vk.base_mip_level) - return G5P_MSAA_REFUSE(cmd_buffer, "a resolve view that is not a %ux%u one-sample linear %s view", + return G5P_REFUSE(cmd_buffer, "a resolve view that is not a %ux%u one-sample linear %s view", iview->vk.extent.width, iview->vk.extent.height, vk_Format_to_str(iview->vk.format)); } return VK_SUCCESS; diff --git a/src/imagination/vulkan/pvr_arch_g5p_tex.c b/src/imagination/vulkan/pvr_arch_g5p_tex.c index 84d25aba15c..edb1debaab9 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_tex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_tex.c @@ -14,6 +14,7 @@ #include "vk_enum_to_str.h" #include "vk_format.h" +#include "pvr_g5p_shader_dump.h" #include "vk_internal_exts.h" #include "vk_log.h" #include "vk_util.h" @@ -38,7 +39,7 @@ g5p_tex_format(VkFormat format, struct pvr_g5p_tex_format *out) /* The three qwords of a view, or the reason it has none */ static const char * -g5p_view_state(const struct pvr_image_view *iview, uint64_t q[3]) +g5p_view_state(const struct pvr_image_view *iview, bool storage, uint64_t q[3]) { const struct pvr_image *image = pvr_image_view_get_image(iview); const struct pvr_image_plane *plane = pvr_single_plane_const(image); @@ -57,7 +58,10 @@ g5p_view_state(const struct pvr_image_view *iview, uint64_t q[3]) const bool meta_2d = (iview->vk.create_flags & VK_IMAGE_VIEW_CREATE_DRIVER_INTERNAL_BIT_MESA) && iview->vk.view_type == VK_IMAGE_VIEW_TYPE_2D_ARRAY && iview->vk.layer_count == 1 && image->vk.image_type == VK_IMAGE_TYPE_2D; - const VkImageViewType view_type = meta_2d ? VK_IMAGE_VIEW_TYPE_2D : iview->vk.view_type; + const bool storage_cube = storage && (iview->vk.view_type == VK_IMAGE_VIEW_TYPE_CUBE || + iview->vk.view_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY); + const VkImageViewType view_type = storage_cube ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : + meta_2d ? VK_IMAGE_VIEW_TYPE_2D : iview->vk.view_type; if (image->memlayout == PVR_MEMLAYOUT_LINEAR) { const struct pvr_mip_level *level = &plane->mip_levels[0]; @@ -81,7 +85,13 @@ g5p_view_state(const struct pvr_image_view *iview, uint64_t q[3]) pvr_g5p_tex_twiddled(q, &v); return NULL; } - if (view_type != VK_IMAGE_VIEW_TYPE_2D || iview->vk.base_mip_level) + /* A single-layer array view (PPSSPP binds every texture so) gets the + * image's 2D state: the compiler draws its layer-0 samples as 2D + * (PCO_G5P_UPLOAD_IMAGE_AS_2D); an array sample at another layer is + * refused at its draw, which is skipped (pvr_g5p_tex_state_for()). */ + const bool one_layer_array = view_type == VK_IMAGE_VIEW_TYPE_2D_ARRAY && + image->vk.image_type == VK_IMAGE_TYPE_2D && iview->vk.layer_count == 1; + if ((view_type != VK_IMAGE_VIEW_TYPE_2D && !one_layer_array) || iview->vk.base_mip_level) return "array/3D/cube or mip view of a linear image (Apple samples those twiddled only)"; if (va % 16 || level->pitch % 16) return "linear texture off a 16-byte base or 16-byte rows"; @@ -90,6 +100,27 @@ g5p_view_state(const struct pvr_image_view *iview, uint64_t q[3]) return NULL; } + /* A 4-sample colour image: each pixel's four samples are 16 consecutive bytes over the + * twiddled power-of-two surface (pvr_g5p_msaa_layout(), the PBE's multisample store), which is + * the 2W x 2H twiddled 2D texture whose texel (2x + (s >> 1), 2y + (s & 1)) is sample s of + * pixel (x, y) (Morton order, y in bit 0). The compiler reads sample s there (txf_ms lowered + * to txf, layer0/txs halved: pco_g5p_general.c ms_as_2d()). */ + if (image->vk.samples == VK_SAMPLE_COUNT_4_BIT) { + if (image->vk.image_type != VK_IMAGE_TYPE_2D || iview->vk.base_mip_level || iview->vk.layer_count != 1 || + (view_type != VK_IMAGE_VIEW_TYPE_2D && view_type != VK_IMAGE_VIEW_TYPE_2D_ARRAY) || + vk_format_get_blocksize(iview->vk.format) != 4) + return "a 4-sample view other than one layer of a 2D 4-byte colour image"; + const struct pvr_g5p_tex_twiddled_view v = { + .key = key, .layout = PVR_G5P_TEX_LAYOUT_2D, + .width = 2 * image->vk.extent.width, .height = 2 * image->vk.extent.height, .depth = 1, + .level_count = 1, .srgb = f->srgb, + .va = image->dev_addr.addr + plane->offset + iview->vk.base_array_layer * plane->layer_size, + }; + if (v.va % 64) + return "a 4-sample view off a 64-byte base"; + pvr_g5p_tex_twiddled(q, &v); + return NULL; + } if (image->vk.image_type == VK_IMAGE_TYPE_3D && iview->vk.view_type != VK_IMAGE_VIEW_TYPE_3D) return "2D view of a 3D image (a slice of the A9's 3D bricks is no 2D texture)"; @@ -135,13 +166,13 @@ g5p_view_state(const struct pvr_image_view *iview, uint64_t q[3]) /* * The sampled state of a view: q0, q1 in words[], q2 in meta[0..1], - * meta[2..3] zero, the 32 bytes the backend's image upload reads. + * meta[2..3] diagnostic view pointer when logging, otherwise zero. */ -const char * -PVR_PER_ARCH(g5p_pack_image_view)(const struct pvr_image_view *iview, struct pvr_image_descriptor *desc) +static const char * +g5p_pack_view(const struct pvr_image_view *iview, bool storage, struct pvr_image_descriptor *desc) { uint64_t q[3]; - const char *why = g5p_view_state(iview, q); + const char *why = g5p_view_state(iview, storage, q); static_assert(sizeof(desc->words) + sizeof(desc->meta) == 32, "G5P image state is 32 bytes"); memset(desc, 0, sizeof(*desc)); @@ -161,9 +192,32 @@ PVR_PER_ARCH(g5p_pack_image_view)(const struct pvr_image_view *iview, struct pvr desc->words[1] = q[1]; desc->meta[0] = (uint32_t)q[2]; desc->meta[1] = (uint32_t)(q[2] >> 32); + /* q3 is CPU metadata, not one of the three A9 texture-state qwords. + * Descriptor copies/templates preserve it. Dispatch logging snapshots the + * view immediately, never retaining this pointer in a recorded command. */ + if (pvr_g5p_compute_log()) { + const uintptr_t ptr = (uintptr_t)iview; + static_assert(sizeof(ptr) == 8, "A9 userspace pointers"); + memcpy(&desc->meta[2], &ptr, sizeof(ptr)); + } return NULL; } +const char * +PVR_PER_ARCH(g5p_pack_image_view)(const struct pvr_image_view *iview, struct pvr_image_descriptor *desc) +{ + return g5p_pack_view(iview, false, desc); +} + +/* Vulkan layer_count already counts faces (six for each cube). Storage + * descriptors use that same face order as a 2D array; sampled state stays cube. */ +const char * +PVR_PER_ARCH(g5p_pack_storage_image_view)(const struct pvr_image_view *iview, + struct pvr_image_descriptor *desc) +{ + return g5p_pack_view(iview, true, desc); +} + /* * Both forms of a sampler's word: the general one (Vulkan NEAREST * mipmapping as Apple's mip-nearest) in words[0], for standalone samplers diff --git a/src/imagination/vulkan/pvr_arch_g5p_upload.c b/src/imagination/vulkan/pvr_arch_g5p_upload.c index 1a0d8c19f18..3cf3c9dbd70 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_upload.c +++ b/src/imagination/vulkan/pvr_arch_g5p_upload.c @@ -13,6 +13,7 @@ #include #include "util/log.h" +#include "pvr_g5p.h" #include "vk_alloc.h" #include "vk_enum_to_str.h" #include "vk_log.h" @@ -34,10 +35,6 @@ #include "pvr_g5p_upload.h" #include "pvr_g5p_vtex.h" -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_UP_REFUSE(obj, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* The loader: DOUTD c(2i):c(2i+1) -> common store with the control word in * c(2n+2i), the last one ends, then HALT (the fragment loader's shape) */ @@ -100,7 +97,7 @@ PVR_PER_ARCH(g5p_vs_uploads_create)(struct pvr_device *device, const struct pco_ return pvr_arch_g5p_vtex_create(device, vs, out); } if (vs->num_uploads > PVR_G5P_MAX_STAGE_UPLOADS) - return G5P_UP_REFUSE(device, "%u vertex-stage uploads (at most %u)", vs->num_uploads, + return G5P_REFUSE(device, "%u vertex-stage uploads (at most %u)", vs->num_uploads, PVR_G5P_MAX_STAGE_UPLOADS); for (unsigned i = 0; i < vs->num_uploads; i++) { @@ -115,11 +112,11 @@ PVR_PER_ARCH(g5p_vs_uploads_create)(struct pvr_device *device, const struct pco_ default: /* textures and samplers in a vertex program: lane b's descriptor * words exist, the vertex stage does not take them yet */ - return G5P_UP_REFUSE(device, "vertex-stage upload %u: %s", i, g5p_up_kind_str(u->kind)); + return G5P_REFUSE(device, "vertex-stage upload %u: %s", i, g5p_up_kind_str(u->kind)); } if (!u->dwords || u->dwords > 0xfff || u->set >= PVR_MAX_DESCRIPTOR_SETS || u->binding > UINT8_MAX || u->element > UINT16_MAX || u->offset > UINT16_MAX || u->common > UINT16_MAX) - return G5P_UP_REFUSE(device, + return G5P_REFUSE(device, "vertex-stage upload %u: set %u binding %u[%u] offset %u, %u dwords at common %u", i, u->set, u->binding, u->element, u->offset, u->dwords, u->common); } @@ -168,10 +165,10 @@ g5p_up_buffer(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_stage_uplo /* a set without a layout: a pushed set not instantiated (never dereferenced) */ if (!set || !set->layout || u->binding >= set->layout->binding_count) - return G5P_UP_REFUSE(cmd_buffer, "upload: set %u binding %u is not bound", u->set, u->binding); + return G5P_REFUSE(cmd_buffer, "upload: set %u binding %u is not bound", u->set, u->binding); const struct pvr_descriptor_set_layout_binding *lb = &set->layout->bindings[u->binding]; if (u->element >= lb->descriptor_count) - return G5P_UP_REFUSE(cmd_buffer, "upload: set %u binding %u element %u past %u descriptors", u->set, + return G5P_REFUSE(cmd_buffer, "upload: set %u binding %u element %u past %u descriptors", u->set, u->binding, u->element, lb->descriptor_count); struct pvr_buffer_descriptor d; switch (lb->type) { @@ -185,11 +182,11 @@ g5p_up_buffer(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_stage_uplo d = set->dynamic_buffers[lb->dynamic_buffer_idx + u->element]; break; default: - return G5P_UP_REFUSE(cmd_buffer, "upload: set %u binding %u is a %s, not a buffer", u->set, + return G5P_REFUSE(cmd_buffer, "upload: set %u binding %u is a %s, not a buffer", u->set, u->binding, vk_DescriptorType_to_str(lb->type)); } if (!d.addr) - return G5P_UP_REFUSE(cmd_buffer, "upload: set %u binding %u holds no buffer", u->set, u->binding); + return G5P_REFUSE(cmd_buffer, "upload: set %u binding %u holds no buffer", u->set, u->binding); /* a dynamic buffer's range starts at its dynamic offset (VkDescriptorBufferInfo.range from there) */ *addr = d.addr + d.offset; *size = d.size; @@ -203,7 +200,7 @@ PVR_PER_ARCH(g5p_upload_buffer)(struct pvr_cmd_buffer *cmd_buffer, unsigned set, { const struct pvr_g5p_stage_upload u = { .set = set, .binding = binding, .element = element }; if (set >= PVR_MAX_DESCRIPTOR_SETS) - return G5P_UP_REFUSE(cmd_buffer, "upload: set %u", set); + return G5P_REFUSE(cmd_buffer, "upload: set %u", set); return g5p_up_buffer(cmd_buffer, &u, addr, size); } @@ -229,7 +226,7 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc case PCO_G5P_UPLOAD_PUSH: /* the bytes the program reads, as they are now (a copy per draw) */ if (u->offset + bytes > PVR_MAX_PUSH_CONSTANTS_SIZE) - return G5P_UP_REFUSE(cmd_buffer, "vertex push constants [%u, %u) past %u bytes", u->offset, + return G5P_REFUSE(cmd_buffer, "vertex push constants [%u, %u) past %u bytes", u->offset, u->offset + bytes, PVR_MAX_PUSH_CONSTANTS_SIZE); result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, push->data + u->offset, bytes, &bo); if (result != VK_SUCCESS) @@ -242,7 +239,7 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc if (result != VK_SUCCESS) return result; if (u->offset + bytes > size) - return G5P_UP_REFUSE(cmd_buffer, "vertex uniform buffer at set %u binding %u: [%u, %u) past its %" PRIu64 + return G5P_REFUSE(cmd_buffer, "vertex uniform buffer at set %u binding %u: [%u, %u) past its %" PRIu64 " bytes", u->set, u->binding, u->offset, u->offset + bytes, size); src[i] = addr + u->offset; break; @@ -252,8 +249,19 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc result = g5p_up_buffer(cmd_buffer, u, &addr, &size); if (result != VK_SUCCESS) return result; + if (u->kind == PCO_G5P_UPLOAD_SSBO_ADDR && u->offset == PCO_G5P_SSBO_SIZE_OFFSET) { + /* get_ssbo_size: the bound size in bytes */ + const uint32_t sz[2] = { (uint32_t)MIN2(size, UINT32_MAX), 0 }; + if (u->dwords > 2) + return G5P_REFUSE(cmd_buffer, "vertex storage buffer size: %u dwords", u->dwords); + result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, sz, sizeof(sz), &bo); + if (result != VK_SUCCESS) + return result; + src[i] = bo->dev_addr.addr; + break; + } if (u->dwords != 2 || (u->kind == PCO_G5P_UPLOAD_UBO_ADDR && u->offset >= size)) - return G5P_UP_REFUSE(cmd_buffer, "vertex %s: %u dwords at offset %u of %" PRIu64 " bytes", + return G5P_REFUSE(cmd_buffer, "vertex %s: %u dwords at offset %u of %" PRIu64 " bytes", g5p_up_kind_str(u->kind), u->dwords, u->offset, size); const uint64_t va = addr + (u->kind == PCO_G5P_UPLOAD_UBO_ADDR ? u->offset : 0); result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, &va, sizeof(va), &bo); @@ -268,7 +276,7 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc uint32_t zero[PCO_G5P_XFB_DWORDS]; memset(zero, 0, sizeof(zero)); if (u->dwords != PCO_G5P_XFB_DWORDS || out->xfb_block) - return G5P_UP_REFUSE(cmd_buffer, "a transform feedback block of %u dwords", u->dwords); + return G5P_REFUSE(cmd_buffer, "a transform feedback block of %u dwords", u->dwords); result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, zero, sizeof(zero), &bo); if (result != VK_SUCCESS) return result; @@ -278,7 +286,7 @@ PVR_PER_ARCH(g5p_vs_uploads_draw)(struct pvr_cmd_buffer *cmd_buffer, const struc break; } default: - return G5P_UP_REFUSE(cmd_buffer, "vertex upload %u: %s", i, g5p_up_kind_str(u->kind)); + return G5P_REFUSE(cmd_buffer, "vertex upload %u: %s", i, g5p_up_kind_str(u->kind)); } } diff --git a/src/imagination/vulkan/pvr_arch_g5p_vdm.c b/src/imagination/vulkan/pvr_arch_g5p_vdm.c index a2e1b0f6a22..3127e155308 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vdm.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vdm.c @@ -34,6 +34,7 @@ #include #include "util/log.h" +#include "pvr_g5p.h" #include "vk_log.h" #include "pvr_bo.h" @@ -43,9 +44,6 @@ #include "winsys/pvr_winsys.h" #include "pvr_g5p_vdm.h" -#define G5P_VDM_REFUSE(cmd_buffer, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((cmd_buffer), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* The bytes of a block before its chaining link to `next`, or -1 when the * block holds no such link */ @@ -91,13 +89,13 @@ PVR_PER_ARCH(g5p_vdm_stream)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr const struct pvr_bo *n = list_entry(b->link.next, struct pvr_bo, link); const int64_t len = block_commands(b, n->vma->dev_addr); if (len < 0) - return G5P_VDM_REFUSE(cmd_buffer, "control stream block %u of %u has no link to block %u", i, blocks, + return G5P_REFUSE(cmd_buffer, "control stream block %u of %u has no link to block %u", i, blocks, i + 1); total += (uint64_t)len; } const uint32_t limit = PVR_G5P_MAX_VDM_BYTES; if (total > limit) - return G5P_VDM_REFUSE(cmd_buffer, "a control stream of %" PRIu64 " bytes in %u blocks (at most %u)", + return G5P_REFUSE(cmd_buffer, "a control stream of %" PRIu64 " bytes in %u blocks (at most %u)", total, blocks, limit); const size_t size = total + ROGUE_VDMCTRL_GUARD_SIZE_DEFAULT; diff --git a/src/imagination/vulkan/pvr_arch_g5p_vertex.c b/src/imagination/vulkan/pvr_arch_g5p_vertex.c index e3f0651a393..3806bd02911 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vertex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vertex.c @@ -26,10 +26,6 @@ #include "pvr_pipeline.h" #include "pvr_g5p_vertex.h" -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_REFUSE(obj, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) VkResult PVR_PER_ARCH(g5p_vertex_draw_check)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline *g, @@ -45,15 +41,13 @@ PVR_PER_ARCH(g5p_vertex_draw_check)(struct pvr_cmd_buffer *cmd_buffer, const str if (t != VK_INDEX_TYPE_UINT16 && t != VK_INDEX_TYPE_UINT32 && t != VK_INDEX_TYPE_UINT8_KHR) return G5P_REFUSE(cmd_buffer, "index type %d", t); } - /* the DDMAD reads dwords: every fetch's base and stride are dword-aligned */ + /* the DDMAD reads dwords: a binding whose address or stride is off dwords draws from a dword-aligned copy + * (pvr_arch_g5p_gfx.c g5p_realign_binding()) */ for (unsigned f = 0; f < g->fetch_count; f++) { const struct pvr_g5p_fetch *fe = &g->fetch_list[f]; const struct pvr_vertex_binding *vb = &cs->vertex_bindings[fe->binding]; if (!vb->buffer) return G5P_REFUSE(cmd_buffer, "no vertex buffer bound at binding %u", fe->binding); - if ((vb->offset + fe->offset) % 4 || dyn->vi_binding_strides[fe->binding] % 4) - return G5P_REFUSE(cmd_buffer, "vertex binding %u at offset %" PRIu64 ", stride %u (the fetch is dword-granular)", - fe->binding, (uint64_t)(vb->offset + fe->offset), dyn->vi_binding_strides[fe->binding]); } return VK_SUCCESS; } @@ -85,6 +79,7 @@ PVR_PER_ARCH(g5p_vertex_index_list)(struct pvr_cmd_buffer *cmd_buffer, const str v0.cut_index_present = true; v0.cut_index_enable = true; v0.cam_size = vector <= 32 ? 95 : vector <= 48 ? 63 : vector <= 64 ? 31 : 15; + v0.flatshade_control = PVR_G5P_FLATSHADE_CONTROL(&cmd_buffer->vk.dynamic_graphics_state); } pvr_csb_pack (&w[(*n)++], VDMCTRL_VDM_STATE1, v1) { v1.cut_index = isize == 1 ? 0xff : isize == 2 ? 0xffff : 0xffffffff; @@ -192,6 +187,7 @@ PVR_PER_ARCH(g5p_vertex_indirect)(struct pvr_cmd_buffer *cmd_buffer, const struc state.cut_index_present = true; state.cut_index_enable = true; state.cam_size = vector <= 32 ? 95 : vector <= 48 ? 63 : vector <= 64 ? 31 : 15; + state.flatshade_control = PVR_G5P_FLATSHADE_CONTROL(&cmd_buffer->vk.dynamic_graphics_state); } pvr_csb_emit (csb, VDMCTRL_VDM_STATE1, state) { state.cut_index = isize == 1 ? 0xff : isize == 2 ? 0xffff : 0xffffffff; diff --git a/src/imagination/vulkan/pvr_arch_g5p_vtex.c b/src/imagination/vulkan/pvr_arch_g5p_vtex.c index dffbe29c74f..2cf6c436306 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vtex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vtex.c @@ -122,6 +122,9 @@ vtex_descriptor(struct pvr_cmd_buffer *cmd, const struct pco_g5p_upload *u, uint if (!image || u->dwords != 8 || !pvr_g5p_tex_state_valid(desc)) return VTEX_REFUSE(cmd, "set %u binding %u has no A9 image state", u->set, u->binding); memcpy(out, desc, 32); + const char *why = pvr_g5p_tex_state_for(out, u->offset); + if (why) + return VTEX_REFUSE(cmd, "set %u binding %u: %s", u->set, u->binding, why); } else if (u->kind == PCO_G5P_UPLOAD_SAMPLER) { if (lb->type != VK_DESCRIPTOR_TYPE_SAMPLER && lb->type != VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) return VTEX_REFUSE(cmd, "set %u binding %u is not a sampler", u->set, u->binding); @@ -174,6 +177,12 @@ PVR_PER_ARCH(g5p_vtex_draw)(struct pvr_cmd_buffer *cmd, const struct pvr_g5p_sta if (u->offset > size || bytes > size - u->offset) return VTEX_REFUSE(cmd, "UBO interval exceeds the bound range"); direct[i] = addr + u->offset; + } else if (u->kind == PCO_G5P_UPLOAD_SSBO_ADDR && u->offset == PCO_G5P_SSBO_SIZE_OFFSET) { + /* get_ssbo_size: the bound size in bytes */ + const uint32_t sz = (uint32_t)MIN2(size, UINT32_MAX); + if (u->dwords != 1) + return VTEX_REFUSE(cmd, "invalid storage buffer size upload"); + memcpy(dst, &sz, sizeof(sz)); } else { if (u->dwords != 2 || (u->kind == PCO_G5P_UPLOAD_UBO_ADDR && u->offset >= size)) return VTEX_REFUSE(cmd, "invalid buffer pointer upload"); diff --git a/src/imagination/vulkan/pvr_arch_g5p_xfb.c b/src/imagination/vulkan/pvr_arch_g5p_xfb.c index 566a5857c19..955cb77bb33 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_xfb.c +++ b/src/imagination/vulkan/pvr_arch_g5p_xfb.c @@ -15,6 +15,7 @@ #include #include "util/log.h" +#include "pvr_g5p.h" #include "util/os_time.h" #include "util/u_dynarray.h" #include "vk_command_buffer.h" @@ -31,10 +32,6 @@ #include "pvr_query.h" #include "pvr_queue.h" -/* A refusal, by name, on stderr too (vk_errorf() is silent in release builds) */ -#define G5P_XFB_REFUSE(obj, ...) \ - (mesa_logw("G5P refuses: " __VA_ARGS__), \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__)) /* A buffer range named by a command (a binding or a counter) */ struct g5p_xfb_range { @@ -93,7 +90,7 @@ g5p_xfb_rec_new(struct pvr_cmd_buffer *cmd_buffer, struct pvr_g5p_xfb_cmd *st, e const struct pvr_sub_cmd *sub = cmd_buffer->state.current_sub_cmd; if ((!sub || sub->type != PVR_SUB_CMD_TYPE_GRAPHICS) && kind <= G5P_XFB_DRAW) { vk_command_buffer_set_error(&cmd_buffer->vk, - G5P_XFB_REFUSE(cmd_buffer, "transform feedback outside a graphics sub-command")); + pvr_refusal_to_api(G5P_REFUSE(cmd_buffer, "transform feedback outside a graphics sub-command"))); return NULL; } struct g5p_xfb_rec *r = util_dynarray_grow(&st->recs, struct g5p_xfb_rec, 1); @@ -143,8 +140,8 @@ PVR_PER_ARCH(g5p_xfb_begin)(struct pvr_cmd_buffer *cmd_buffer, uint32_t first, u return; if (cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT) { vk_command_buffer_set_error(&cmd_buffer->vk, - G5P_XFB_REFUSE(cmd_buffer, "transform feedback in a SIMULTANEOUS_USE command " - "buffer (its per-draw blocks are filled per submission)")); + pvr_refusal_to_api(G5P_REFUSE(cmd_buffer, "transform feedback in a SIMULTANEOUS_USE command " + "buffer (its per-draw blocks are filled per submission)"))); return; } struct g5p_xfb_rec *r = g5p_xfb_rec_new(cmd_buffer, st, G5P_XFB_BEGIN); @@ -192,7 +189,7 @@ PVR_PER_ARCH(g5p_xfb_draw_check)(struct pvr_cmd_buffer *cmd_buffer, VkPrimitiveT /* the stores follow the vertex invocations: they are the primitive * stream only for lists (GLES 3.0 captures lists only) */ if (!g5p_xfb_vpp(topology)) - return G5P_XFB_REFUSE(cmd_buffer, "transform feedback with topology %d (point, line and triangle lists)", + return G5P_REFUSE(cmd_buffer, "transform feedback with topology %d (point, line and triangle lists)", topology); return VK_SUCCESS; } @@ -205,7 +202,7 @@ PVR_PER_ARCH(g5p_xfb_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct g5p_x if (!st) return VK_ERROR_OUT_OF_HOST_MEMORY; if (!block) - return G5P_XFB_REFUSE(cmd_buffer, "a capturing vertex program without its transform feedback block"); + return G5P_REFUSE(cmd_buffer, "a capturing vertex program without its transform feedback block"); struct g5p_xfb_rec *r = g5p_xfb_rec_new(cmd_buffer, st, G5P_XFB_DRAW); if (!r) return VK_ERROR_OUT_OF_HOST_MEMORY; @@ -232,8 +229,8 @@ g5p_xfb_query_event(struct pvr_cmd_buffer *cmd_buffer, enum g5p_xfb_kind kind, c return; if (pool->result_elem_stride < 2 * sizeof(uint64_t)) { vk_command_buffer_set_error(&cmd_buffer->vk, - G5P_XFB_REFUSE(cmd_buffer, "a stream query pool of %u bytes per query", - pool->result_elem_stride)); + pvr_refusal_to_api(G5P_REFUSE(cmd_buffer, "a stream query pool of %u bytes per query", + pool->result_elem_stride))); return; } struct g5p_xfb_rec *r = g5p_xfb_rec_new(cmd_buffer, st, kind); @@ -250,7 +247,7 @@ PVR_PER_ARCH(g5p_xfb_query_begin)(struct pvr_cmd_buffer *cmd_buffer, const struc { if (index) { vk_command_buffer_set_error(&cmd_buffer->vk, - G5P_XFB_REFUSE(cmd_buffer, "a stream query of stream %u (stream 0 only)", index)); + pvr_refusal_to_api(G5P_REFUSE(cmd_buffer, "a stream query of stream %u (stream 0 only)", index))); return; } g5p_xfb_query_event(cmd_buffer, G5P_XFB_QUERY_BEGIN, pool, query, 1); @@ -355,10 +352,10 @@ g5p_xfb_run(struct pvr_device *device, struct pvr_g5p_xfb_cmd *st, unsigned end) r->e.query.pool = &r->qview; } if (!ok) - return G5P_XFB_REFUSE(device, "a transform feedback buffer without a CPU mapping"); + return G5P_REFUSE(device, "a transform feedback buffer without a CPU mapping"); const char *why = g5p_xfb_execute(x, &r->e); if (why) - return G5P_XFB_REFUSE(device, "transform feedback: %s", why); + return G5P_REFUSE(device, "transform feedback: %s", why); } return VK_SUCCESS; } diff --git a/src/imagination/vulkan/pvr_arch_g9p.c b/src/imagination/vulkan/pvr_arch_g9p.c index b8b22d8b7fd..dd9f04ea3da 100644 --- a/src/imagination/vulkan/pvr_arch_g9p.c +++ b/src/imagination/vulkan/pvr_arch_g9p.c @@ -197,6 +197,7 @@ PVR_PER_ARCH(g9p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uint32_t bind_data[4 * (G9P_MAX_BUFFERS + 1) + 4] = { 0 }; uint32_t bind_code[G9P_MAX_BUFFERS + 2]; uint32_t kernel_data[G9P_KERNEL_DATA_DWORDS] = { 0 }, kernel_code[8]; + bool from_uniforms[G9P_MAX_BUFFERS] = { false }; struct pvr_pds_upload bind, kernel; unsigned bind_dwords, ctrl0, nc = 0; VkResult result; @@ -218,6 +219,17 @@ PVR_PER_ARCH(g9p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uniforms[2 * k] = pc->dev_addr.addr; uniforms[2 * k + 1] = pc->dev_addr.addr >> 32; } + if (state->buffers[k].set < PVR_MAX_DESCRIPTOR_SETS) { + /* a dynamic buffer has no descriptor in set memory (binding offset ~0) */ + const unsigned element = + (state->buffers[k].binding & ~G9P_BINDING_SIZE) >> G9P_BINDING_ELEMENT_SHIFT; + const unsigned b = state->buffers[k].binding & BITFIELD_MASK(G9P_BINDING_ELEMENT_SHIFT); + + if (!pvr_descriptor_set_dynamic_slot(desc->sets[state->buffers[k].set], b, element, + state->buffers[k].binding & G9P_BINDING_SIZE, + &uniforms[2 * k], &from_uniforms[k])) + return vk_error(cmd_buffer, VK_ERROR_UNKNOWN); + } } uniforms[2 * n] = count; result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, uniforms, @@ -236,7 +248,7 @@ PVR_PER_ARCH(g9p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, uint64_t src; uint32_t dest = 2 * k, dwords = 2; - if (k < n && state->buffers[k].set != G9P_PUSH_CONSTANTS && + if (k < n && !from_uniforms[k] && state->buffers[k].set != G9P_PUSH_CONSTANTS && state->buffers[k].set != G9P_NUM_WORKGROUPS) { const struct pvr_descriptor_set *set = desc->sets[state->buffers[k].set]; /* an array's element (Zink keeps all storage buffers in one array) */ diff --git a/src/imagination/vulkan/pvr_arch_g9p_gfx.c b/src/imagination/vulkan/pvr_arch_g9p_gfx.c index b59bd652c09..14a6b9a5310 100644 --- a/src/imagination/vulkan/pvr_arch_g9p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g9p_gfx.c @@ -1640,13 +1640,7 @@ binding_needs_copy(const struct pvr_vertex_binding *vb, uint32_t stride) * The copy reads the buffer when the job is submitted: a transfer recorded * before the draw (they run on the CPU at submission) may be what fills it. */ -struct g9p_realign { - const uint8_t *src; - uint8_t *dst; - uint64_t size, count; - uint32_t stride, new_stride; - bool gpu_written; /* the source may be written by an earlier GPU job */ -}; +/* struct g9p_realign: pvr_g9p.h (G5P's draws queue them too) */ static void realign_copy(const struct g9p_realign *r) diff --git a/src/imagination/vulkan/pvr_arch_image.c b/src/imagination/vulkan/pvr_arch_image.c index 5f92166cea5..fb4be87c5a8 100644 --- a/src/imagination/vulkan/pvr_arch_image.c +++ b/src/imagination/vulkan/pvr_arch_image.c @@ -203,6 +203,8 @@ VkResult PVR_PER_ARCH(CreateImageView)(VkDevice _device, util_format_compose_swizzles(format_swizzle, input_swizzle, info.swizzle); + info.skip_rogue_words = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p) || + PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g9p); result = pvr_arch_pack_tex_state(device, &info, &iview->image_state[info.tex_state_type]); @@ -220,6 +222,7 @@ VkResult PVR_PER_ARCH(CreateImageView)(VkDevice _device, */ if (info.is_cube && image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT) { info.tex_state_type = PVR_TEXTURE_STATE_STORAGE; + info.skip_rogue_words = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p); result = pvr_arch_pack_tex_state(device, @@ -230,6 +233,7 @@ VkResult PVR_PER_ARCH(CreateImageView)(VkDevice _device, } if (image->vk.usage & VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT) { + info.skip_rogue_words = false; /* Attachment state is created as if the mipmaps are not supported, so the * baselevel is set to zero and num_mip_levels is set to 1. Which gives an * impression that this is the only level in the image. This also requires diff --git a/src/imagination/vulkan/pvr_arch_pass.c b/src/imagination/vulkan/pvr_arch_pass.c index 8234da8b1c9..2f17922d4e5 100644 --- a/src/imagination/vulkan/pvr_arch_pass.c +++ b/src/imagination/vulkan/pvr_arch_pass.c @@ -871,6 +871,28 @@ PVR_PER_ARCH(CreateRenderPass2)(VkDevice _device, pass->max_sample_count = attachment->sample_count; } + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p)) { + /* Dependencies on commands outside the pass, merged (pvr_render_pass::ext_*) */ + pass->ext_begin_barrier = (VkMemoryBarrier2){ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2 }; + pass->ext_end_barrier = (VkMemoryBarrier2){ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2 }; + for (uint32_t i = 0; i < pCreateInfo->dependencyCount; i++) { + const VkSubpassDependency2 *dep = &pCreateInfo->pDependencies[i]; + if ((dep->srcSubpass == VK_SUBPASS_EXTERNAL) == (dep->dstSubpass == VK_SUBPASS_EXTERNAL)) + continue; + /* synchronization2: a VkMemoryBarrier2 in pNext replaces the 32-bit masks */ + const VkMemoryBarrier2 *mb2 = vk_find_struct_const(dep->pNext, MEMORY_BARRIER_2); + VkMemoryBarrier2 *b = dep->srcSubpass == VK_SUBPASS_EXTERNAL ? &pass->ext_begin_barrier : &pass->ext_end_barrier; + b->srcStageMask |= mb2 ? mb2->srcStageMask : dep->srcStageMask; + b->dstStageMask |= mb2 ? mb2->dstStageMask : dep->dstStageMask; + b->srcAccessMask |= mb2 ? mb2->srcAccessMask : dep->srcAccessMask; + b->dstAccessMask |= mb2 ? mb2->dstAccessMask : dep->dstAccessMask; + if (dep->srcSubpass == VK_SUBPASS_EXTERNAL) + pass->ext_begin = true; + else + pass->ext_end = true; + } + } + /* Count how many dependencies each subpass has. */ for (uint32_t i = 0; i < pCreateInfo->dependencyCount; i++) { const VkSubpassDependency2 *dep = &pCreateInfo->pDependencies[i]; diff --git a/src/imagination/vulkan/pvr_arch_queue.c b/src/imagination/vulkan/pvr_arch_queue.c index 86f91e2d1da..4b230c0b9a6 100644 --- a/src/imagination/vulkan/pvr_arch_queue.c +++ b/src/imagination/vulkan/pvr_arch_queue.c @@ -59,10 +59,12 @@ #include "pvr_bo.h" #include "pvr_pipeline.h" +#include "util/log.h" #include "util/macros.h" #include "util/u_atomic.h" #include "vk_alloc.h" #include "vk_common_entrypoints.h" +#include "vk_enum_to_str.h" #include "vk_fence.h" #include "vk_log.h" #include "vk_object.h" @@ -281,7 +283,7 @@ g5p_merge_syncs(struct pvr_device *device, struct vk_sync *a, struct vk_sync *b, waits[n++] = (struct vk_sync_wait){ .sync = a, .stage_mask = ~(VkPipelineStageFlags2)0 }; if (b) waits[n++] = (struct vk_sync_wait){ .sync = b, .stage_mask = ~(VkPipelineStageFlags2)0 }; - result = vk_sync_create(&device->vk, &device->pdevice->ws->syncobj_type, 0U, 0UL, out); + result = vk_sync_create(&device->vk, &device->pdevice->sync_type, 0U, 0UL, out); if (result != VK_SUCCESS) return result; const struct vk_sync_signal signal = { .sync = *out, .stage_mask = ~(VkPipelineStageFlags2)0 }; @@ -311,7 +313,7 @@ pvr_process_graphics_cmd_for_view(struct pvr_device *device, job->view_state.view_index = view_index; result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &geom_signal_sync); @@ -320,7 +322,7 @@ pvr_process_graphics_cmd_for_view(struct pvr_device *device, if (job->run_frag) { result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &frag_signal_sync); @@ -683,7 +685,7 @@ static VkResult pvr_process_compute_cmd(struct pvr_device *device, VkResult result; result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &sync); @@ -721,7 +723,7 @@ static VkResult pvr_process_transfer_cmds(struct pvr_device *device, VkResult result; result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &sync); @@ -868,7 +870,7 @@ static VkResult pvr_process_query_cmd(struct pvr_device *device, */ result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &sync); @@ -939,7 +941,7 @@ pvr_process_event_cmd_barrier(struct pvr_device *device, struct vk_sync *signal_sync; result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &signal_sync); @@ -1008,7 +1010,7 @@ pvr_process_event_cmd_set_or_reset(struct pvr_device *device, } result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &signal_sync); @@ -1119,7 +1121,7 @@ pvr_process_event_cmd_wait(struct pvr_device *device, assert(wait_count <= (sub_cmd->count + 1)); result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &signal_sync); @@ -1197,6 +1199,16 @@ static VkResult pvr_process_cmd_buffer(struct pvr_device *device, const int64_t g9p_t0 = getenv("G9P_TIME") ? os_time_get_nano() : 0; switch (sub_cmd->type) { case PVR_SUB_CMD_TYPE_GRAPHICS: { + /* a refused render pass (pvr_arch_cmd_buffer.c): nothing to submit */ + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p) && + sub_cmd->gfx.g5p_refused) { + if (!sub_cmd->is_resume) { + suspended_data->rts = NULL; + suspended_data->query_pool = NULL; + } + result = VK_SUCCESS; + break; + } /* If the fragment job utilizes queries, for data integrity * it needs to wait for the query to be processed. */ @@ -1591,7 +1603,7 @@ static VkResult pvr_process_queue_waits(struct pvr_queue *queue, } result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &queue->next_job_wait_sync[i]); @@ -1708,6 +1720,18 @@ static void pvr_signal_unsubmitted(struct vk_device *device, static VkResult pvr_driver_queue_submit(struct vk_queue *queue, struct vk_queue_submit *submit) { + struct pvr_queue *driver_queue = container_of(queue, struct pvr_queue, vk); + const bool is_g5p = + PVR_HAS_FEATURE(&driver_queue->device->pdevice->dev_info, apple_g5p); + + if (is_g5p && vk_queue_is_lost(queue)) { + for (uint32_t i = 0; i < submit->signal_count; i++) + pvr_signal_unsubmitted(&driver_queue->device->vk, + submit->signals[i].sync, + submit->signals[i].signal_value); + return VK_SUCCESS; + } + const int64_t g9p_t0 = os_time_get_nano(); VkResult result = pvr_driver_queue_submit_jobs(queue, submit); @@ -1721,8 +1745,6 @@ static VkResult pvr_driver_queue_submit(struct vk_queue *queue, * reset, freezing the desktop; signalled, it sees the device lost and can recover */ if (result != VK_SUCCESS) { - struct pvr_queue *driver_queue = container_of(queue, struct pvr_queue, vk); - for (uint32_t i = 0; i < submit->signal_count; i++) { if (PVR_HAS_FEATURE(&driver_queue->device->pdevice->dev_info, apple_g5p)) pvr_signal_unsubmitted(&driver_queue->device->vk, @@ -1732,6 +1754,7 @@ static VkResult pvr_driver_queue_submit(struct vk_queue *queue, vk_sync_signal(&driver_queue->device->vk, submit->signals[i].sync, submit->signals[i].signal_value); } + } if (result == VK_ERROR_DEVICE_LOST) return vk_queue_set_lost(queue, "GPU context lost"); diff --git a/src/imagination/vulkan/pvr_arch_tex_state.c b/src/imagination/vulkan/pvr_arch_tex_state.c index eaf373656d3..662aa274d90 100644 --- a/src/imagination/vulkan/pvr_arch_tex_state.c +++ b/src/imagination/vulkan/pvr_arch_tex_state.c @@ -121,6 +121,10 @@ VkResult pvr_arch_pack_tex_state(struct pvr_device *device, enum pvr_memlayout mem_layout; VkImageViewType iview_type; PVR_G5P_HOOK(texel_state, device, info, state) + if (info->skip_rogue_words) { + memset(state->words, 0, sizeof(state->words)); + goto metadata; + } if (info->type == VK_IMAGE_VIEW_TYPE_1D && info->mem_layout == PVR_MEMLAYOUT_LINEAR) { /* Change the memory layout to twiddled as there isn't a TEXSTATE_TEXTYPE @@ -315,6 +319,7 @@ VkResult pvr_arch_pack_tex_state(struct pvr_device *device, } } +metadata: state->meta[PCO_IMAGE_META_LAYER_SIZE] = info->layer_size; state->meta[PCO_IMAGE_META_BUFFER_ELEMS] = info->buffer_elems; state->meta[PCO_IMAGE_META_Z_SLICE] = info->z_slice; diff --git a/src/imagination/vulkan/pvr_bo.c b/src/imagination/vulkan/pvr_bo.c index 629c4f81fde..e4bf38f2f6a 100644 --- a/src/imagination/vulkan/pvr_bo.c +++ b/src/imagination/vulkan/pvr_bo.c @@ -37,6 +37,7 @@ #endif #include "hwdef/rogue_hw_utils.h" +#include "pvr_bo_log.h" #include "pvr_bo.h" #include "pvr_debug.h" #include "pvr_device.h" @@ -338,12 +339,13 @@ static inline void pvr_bo_free_bo(const struct pvr_device *const device, * * \sa #pvr_bo_free() */ -VkResult pvr_bo_alloc(struct pvr_device *device, - struct pvr_winsys_heap *heap, - uint64_t size, - uint64_t alignment, - uint64_t flags, - struct pvr_bo **const pvr_bo_out) +VkResult pvr_bo_alloc_tagged(const char *tag, + struct pvr_device *device, + struct pvr_winsys_heap *heap, + uint64_t size, + uint64_t alignment, + uint64_t flags, + struct pvr_bo **const pvr_bo_out) { struct pvr_bo *pvr_bo; VkResult result; @@ -383,6 +385,14 @@ VkResult pvr_bo_alloc(struct pvr_device *device, pvr_bo_store_insert(device->bo_store, pvr_bo); *pvr_bo_out = pvr_bo; + if (unlikely(device->bo_log)) + pvr_bo_log_driver_alloc(device, + tag, + heap, + size, + flags, + pvr_bo, + VK_SUCCESS); return VK_SUCCESS; @@ -400,6 +410,8 @@ VkResult pvr_bo_alloc(struct pvr_device *device, pvr_bo_free_bo(device, pvr_bo); err_out: + if (unlikely(device->bo_log)) + pvr_bo_log_driver_alloc(device, tag, heap, size, flags, NULL, result); return result; } @@ -479,6 +491,9 @@ void pvr_bo_free(struct pvr_device *device, struct pvr_bo *pvr_bo) vk_free(&device->vk.alloc, pvr_bo->bo->vbits); #endif /* defined(HAVE_VALGRIND) */ + if (unlikely(device->bo_log)) + pvr_bo_log_free(device, pvr_bo); + pvr_bo_store_remove(device->bo_store, pvr_bo); device->ws->ops->vma_unmap(pvr_bo->vma); @@ -538,6 +553,11 @@ static inline struct pvr_bo *pvr_bo_get_ref(struct pvr_bo *bo) return bo; } +static inline uint64_t pvr_suballoc_cache_max(const struct pvr_suballocator *a) +{ + return 2 * (uint64_t)a->default_size; +} + /** * \brief Interface to sub-allocate buffer objects. * @@ -582,6 +602,37 @@ VkResult pvr_bo_suballoc(struct pvr_suballocator *allocator, simple_mtx_lock(&allocator->mtx); + /* Large allocations must not replace the current BO or enter the cache. */ + if (aligned_size > pvr_suballoc_cache_max(allocator)) { + struct pvr_bo *bo; + + result = pvr_bo_alloc(allocator->device, + allocator->heap, + aligned_size, + align, + PVR_BO_ALLOC_FLAG_CPU_MAPPED, + &bo); + if (result != VK_SUCCESS) { + vk_free(&allocator->device->vk.alloc, suballoc_bo); + simple_mtx_unlock(&allocator->mtx); + return result; + } + + suballoc_bo->allocator = allocator; + suballoc_bo->bo = bo; + suballoc_bo->dev_addr = bo->vma->dev_addr; + suballoc_bo->offset = 0; + suballoc_bo->size = aligned_size; + + if (zero_on_alloc) + memset(pvr_bo_suballoc_get_map_addr(suballoc_bo), 0, aligned_size); + + *suballoc_bo_out = suballoc_bo; + simple_mtx_unlock(&allocator->mtx); + return VK_SUCCESS; + } + + if (allocator->bo) { uint32_t aligned_offset = ALIGN_POT(allocator->next_offset, align); @@ -667,7 +718,8 @@ void pvr_bo_suballoc_free(struct pvr_suballoc_bo *suballoc_bo) simple_mtx_lock(&suballoc_bo->allocator->mtx); if (p_atomic_read(&suballoc_bo->bo->ref_count) == 1 && - !suballoc_bo->allocator->bo_cached) { + !suballoc_bo->allocator->bo_cached && + suballoc_bo->bo->bo->size <= pvr_suballoc_cache_max(suballoc_bo->allocator)) { suballoc_bo->allocator->bo_cached = suballoc_bo->bo; } else { pvr_bo_free(suballoc_bo->allocator->device, suballoc_bo->bo); diff --git a/src/imagination/vulkan/pvr_bo.h b/src/imagination/vulkan/pvr_bo.h index bf9c22a5882..941f98b9e2b 100644 --- a/src/imagination/vulkan/pvr_bo.h +++ b/src/imagination/vulkan/pvr_bo.h @@ -117,12 +117,15 @@ struct pvr_suballoc_bo { */ #define PVR_BO_ALLOC_FLAG_PM_FW_PROTECT BITFIELD_BIT(3U) -VkResult pvr_bo_alloc(struct pvr_device *device, - struct pvr_winsys_heap *heap, - uint64_t size, - uint64_t alignment, - uint64_t flags, - struct pvr_bo **const bo_out); +VkResult pvr_bo_alloc_tagged(const char *tag, + struct pvr_device *device, + struct pvr_winsys_heap *heap, + uint64_t size, + uint64_t alignment, + uint64_t flags, + struct pvr_bo **const bo_out); +#define pvr_bo_alloc(...) pvr_bo_alloc_tagged(__func__, __VA_ARGS__) + VkResult pvr_bo_cpu_map(struct pvr_device *device, struct pvr_bo *bo); void pvr_bo_cpu_unmap(struct pvr_device *device, struct pvr_bo *bo); void pvr_bo_free(struct pvr_device *device, struct pvr_bo *bo); diff --git a/src/imagination/vulkan/pvr_bo_log.c b/src/imagination/vulkan/pvr_bo_log.c new file mode 100644 index 00000000000..3875cbb2f4f --- /dev/null +++ b/src/imagination/vulkan/pvr_bo_log.c @@ -0,0 +1,624 @@ +/* SPDX-License-Identifier: MIT */ +#include "pvr_bo_log.h" + +#include +#include +#include +#include +#include +#if defined(__linux__) +# include +#endif + +#include "pvr_bo.h" +#include "pvr_buffer.h" +#include "pvr_debug.h" +#include "pvr_device.h" +#include "pvr_image.h" +#include "util/hash_table.h" +#include "util/list.h" +#include "util/simple_mtx.h" +#include "util/u_math.h" +#include "util/log.h" +#include "util/u_debug.h" +#include "vk_enum_to_str.h" +#include "vk_format.h" +#include "vk_util.h" +#include "winsys/pvr_winsys.h" + +struct pvr_bo_log_entry { + const char *kind; + const char *tag; + const char *heap; + uint64_t size; + bool imported; + bool display; + char thread[17]; +}; + +/* Keep object records separately so padding totals include every live binding, + * while each allocation's live line shows only its four largest objects. + * Descriptions never dereference Vulkan objects at exit. + */ +struct pvr_bo_log_binding { + const void *memory; + uint64_t size; + uint64_t tight; + bool image; + char description[512]; +}; + +struct pvr_bo_log { + struct list_head link; + simple_mtx_t mutex; + struct hash_table *entries; + struct hash_table *bindings; + uint64_t min_size; + uint64_t app_live; + uint64_t app_count; + uint64_t import_live; + uint64_t display_live; + uint64_t drv_live; + uint64_t drv_count; + uint64_t peak; + uint64_t failed; + uint64_t tracking_failed; +}; + +static struct list_head logs = { &logs, &logs }; +static simple_mtx_t logs_mutex = SIMPLE_MTX_INITIALIZER; +DEBUG_GET_ONCE_NUM_OPTION(pvr_bo_log_min_kib, "PVR_BO_LOG_MIN_KIB", 64) + +static const char *thread_name(char buf[17]) +{ + buf[0] = 0; +#if defined(PR_GET_NAME) + if (prctl(PR_GET_NAME, buf) != 0) + buf[0] = 0; + buf[16] = 0; +#endif + for (char *c = buf; *c; c++) { + if (*c == ' ') + *c = '_'; + } + return buf[0] ? buf : "?"; +} + +static void +dedicated_word(const VkMemoryAllocateInfo *info, char *buf, size_t n) +{ + const VkMemoryDedicatedAllocateInfo *ded = + vk_find_struct_const(info->pNext, MEMORY_DEDICATED_ALLOCATE_INFO); + + if (ded && ded->image != VK_NULL_HANDLE) + snprintf(buf, n, "image:%#" PRIx64, (uint64_t)(uintptr_t)ded->image); + else if (ded && ded->buffer != VK_NULL_HANDLE) + snprintf(buf, n, "buffer:%#" PRIx64, (uint64_t)(uintptr_t)ded->buffer); + else + snprintf(buf, n, "none"); +} + +static const char *heap_name(const struct pvr_device *device, + const struct pvr_winsys_heap *heap) +{ + if (heap == device->heaps.general_heap) + return "general"; + if (heap == device->heaps.pds_heap) + return "pds"; + if (heap == device->heaps.rgn_hdr_heap) + return "rgn_hdr"; + if (heap == device->heaps.transfer_frag_heap) + return "transfer_frag"; + if (heap == device->heaps.usc_heap) + return "usc"; + if (heap == device->heaps.vis_test_heap) + return "vis_test"; + return "unknown"; +} + +static uint64_t live_bytes(const struct pvr_bo_log *log) +{ + return log->app_live - log->import_live + log->drv_live; +} + +struct pvr_bo_log_tag { + const char *kind; + const char *tag; + const char *heap; + uint64_t size; + uint64_t count; +}; + +static int compare_tags(const void *a, const void *b) +{ + const struct pvr_bo_log_tag *ta = a, *tb = b; + return (ta->size < tb->size) - (ta->size > tb->size); +} + +static void summary_locked(struct pvr_bo_log *log, const char *why, bool tables) +{ + const uint64_t created = live_bytes(log) - log->display_live; + const uint64_t ws_created = pvr_winsys_counted_bytes(0); + + mesa_logi("pvr bo summary why=%s pid=%ld app_live_kib=%" PRIu64 + " app_count=%" PRIu64 " import_kib=%" PRIu64 " drv_live_kib=%" PRIu64 + " drv_count=%" PRIu64 " created_live_kib=%" PRIu64 + " ws_created_kib=%" PRIu64 " ws_display_kib=%" PRIu64 + " untagged_kib=%" PRId64 " peak_kib=%" PRIu64 " failed=%" PRIu64 + " tracking_failed=%" PRIu64 "\n", + why, + (long)getpid(), + log->app_live >> 10, + log->app_count, + log->import_live >> 10, + log->drv_live >> 10, + log->drv_count, + created >> 10, + ws_created >> 10, + pvr_winsys_counted_bytes(1) >> 10, + ((int64_t)ws_created - (int64_t)created) / 1024, + log->peak >> 10, + log->failed, + log->tracking_failed); + if (!tables) { + + return; + } + + uint64_t images = 0, total = 0, tight = 0; + hash_table_foreach (log->bindings, h) { + const struct pvr_bo_log_binding *b = h->data; + if (b->image) { + images++; + total += b->size; + tight += b->tight; + } + } + mesa_logi("pvr bo summary-pad images=%" PRIu64 " total_kib=%" PRIu64 + " tight_kib=%" PRIu64 " pad_kib=%" PRId64 "\n", + images, + total >> 10, + tight >> 10, + ((int64_t)total - (int64_t)tight) / 1024); + + struct pvr_bo_log_tag *tags = + calloc(MAX2(log->entries->entries, 1), sizeof(*tags)); + unsigned num_tags = 0; + const struct pvr_bo_log_entry *largest[40] = { 0 }; + const void *keys[40] = { 0 }; + + hash_table_foreach (log->entries, h) { + const struct pvr_bo_log_entry *e = h->data; + if (tags) { + unsigned t; + for (t = 0; t < num_tags; t++) { + if (!strcmp(tags[t].kind, e->kind) && + !strcmp(tags[t].tag, e->tag) && !strcmp(tags[t].heap, e->heap)) + break; + } + if (t == num_tags) { + tags[t].kind = e->kind; + tags[t].tag = e->tag; + tags[t].heap = e->heap; + num_tags++; + } + tags[t].size += e->size; + tags[t].count++; + } + for (unsigned i = 0; i < ARRAY_SIZE(largest); i++) { + if (!largest[i] || e->size > largest[i]->size) { + for (unsigned j = ARRAY_SIZE(largest) - 1; j > i; j--) { + largest[j] = largest[j - 1]; + keys[j] = keys[j - 1]; + } + largest[i] = e; + keys[i] = h->key; + break; + } + } + } + if (tags) { + qsort(tags, num_tags, sizeof(*tags), compare_tags); + for (unsigned i = 0; i < num_tags; i++) { + mesa_logi("pvr bo summary-tag kind=%s tag=%s heap=%s count=%" PRIu64 + " kib=%" PRIu64 "\n", + tags[i].kind, + tags[i].tag, + tags[i].heap, + tags[i].count, + tags[i].size >> 10); + } + free(tags); + } + for (unsigned i = 0; i < ARRAY_SIZE(largest) && largest[i]; i++) { + const struct pvr_bo_log_entry *e = largest[i]; + const struct pvr_bo_log_binding *bound[4] = { 0 }; + unsigned count = 0; + hash_table_foreach (log->bindings, h) { + const struct pvr_bo_log_binding *b = h->data; + if (b->memory != keys[i]) + continue; + count++; + for (unsigned j = 0; j < ARRAY_SIZE(bound); j++) { + if (!bound[j] || b->size > bound[j]->size) { + for (unsigned k = ARRAY_SIZE(bound) - 1; k > j; k--) + bound[k] = bound[k - 1]; + bound[j] = b; + break; + } + } + } + char descriptions[4096] = { 0 }; + size_t used = 0; + for (unsigned j = 0; j < ARRAY_SIZE(bound) && bound[j]; j++) { + int n = snprintf(descriptions + used, sizeof(descriptions) - used, + "%s%s", j ? ";" : "", bound[j]->description); + used += MIN2(MAX2(n, 0), sizeof(descriptions) - used - 1); + } + if (count > ARRAY_SIZE(bound)) + snprintf(descriptions + used, sizeof(descriptions) - used, + ";+%u", count - (unsigned)ARRAY_SIZE(bound)); + mesa_logi("pvr bo live kind=%s size_kib=%" PRIu64 + " tag=%s heap=%s thread=%s bound=%s", + e->kind, e->size >> 10, e->tag, e->heap, e->thread, descriptions); + } + +} + +void pvr_bo_log_summary(struct pvr_device *device, const char *why) +{ + struct pvr_bo_log *log = device->bo_log; + simple_mtx_lock(&log->mutex); + summary_locked(log, why, true); + simple_mtx_unlock(&log->mutex); +} + +void pvr_bo_log_device_init(struct pvr_device *device) +{ + if (!PVR_IS_DEBUG_SET(BO_LOG)) + return; + + struct pvr_bo_log *log = calloc(1, sizeof(*log)); + if (!log) + return; + log->entries = _mesa_pointer_hash_table_create(NULL); + log->bindings = _mesa_pointer_hash_table_create(NULL); + if (!log->entries || !log->bindings) { + _mesa_hash_table_destroy(log->entries, NULL); + _mesa_hash_table_destroy(log->bindings, NULL); + free(log); + return; + } + log->min_size = MAX2(0, debug_get_option_pvr_bo_log_min_kib()); + log->min_size = MIN2(log->min_size, UINT64_MAX / 1024) * 1024; + simple_mtx_init(&log->mutex, mtx_plain); + simple_mtx_lock(&logs_mutex); + list_addtail(&log->link, &logs); + device->bo_log = log; + simple_mtx_unlock(&logs_mutex); +} + +static void free_record(struct hash_entry *entry) +{ + free(entry->data); +} + +void pvr_bo_log_device_finish(struct pvr_device *device) +{ + struct pvr_bo_log *log = device->bo_log; + simple_mtx_lock(&logs_mutex); + list_del(&log->link); + device->bo_log = NULL; + simple_mtx_unlock(&logs_mutex); + _mesa_hash_table_destroy(log->bindings, free_record); + _mesa_hash_table_destroy(log->entries, free_record); + simple_mtx_destroy(&log->mutex); + free(log); +} + +static void record_alloc(struct pvr_bo_log *log, + const void *key, + const struct pvr_bo_log_entry *entry, + const char *detail, + VkResult result) +{ + simple_mtx_lock(&log->mutex); + if (result == VK_SUCCESS) { + struct pvr_bo_log_entry *copy = malloc(sizeof(*copy)); + if (copy) { + *copy = *entry; + if (!_mesa_hash_table_insert(log->entries, key, copy)) { + free(copy); + log->tracking_failed++; + } + } else { + log->tracking_failed++; + } + if (!strcmp(entry->kind, "app")) { + log->app_live += entry->size; + log->app_count++; + if (entry->imported) + log->import_live += entry->size; + else if (entry->display) + log->display_live += entry->size; + } else { + log->drv_live += entry->size; + log->drv_count++; + } + } else { + log->failed++; + } + if (entry->size >= log->min_size || result != VK_SUCCESS) { + mesa_logi("pvr bo %s kind=%s %s=%p size_kib=%" PRIu64 + " tag=%s heap=%s %s result=%s live_kib=%" PRIu64 + " thread=%s pid=%ld\n", + result == VK_SUCCESS ? "alloc" : "alloc-failed", + entry->kind, + !strcmp(entry->kind, "app") ? "mem" : "bo", + key, + entry->size >> 10, + entry->tag, + entry->heap, + detail, + vk_Result_to_str(result), + live_bytes(log) >> 10, + entry->thread, + (long)getpid()); + + } + uint64_t old_peak = log->peak; + log->peak = MAX2(log->peak, live_bytes(log)); + if ((log->peak >> 26) > (old_peak >> 26)) + summary_locked(log, "peak", false); + if (result != VK_SUCCESS && log->failed <= 3) + summary_locked(log, "alloc-failed", true); + simple_mtx_unlock(&log->mutex); +} + +void pvr_bo_log_driver_alloc(struct pvr_device *device, + const char *tag, + struct pvr_winsys_heap *heap, + uint64_t size, + uint64_t flags, + const struct pvr_bo *bo, + VkResult result) +{ + struct pvr_bo_log_entry entry = { + .kind = "drv", + .tag = tag, + .heap = heap_name(device, heap), + .size = bo ? bo->bo->size : size, + }; + char detail[48]; + snprintf(detail, sizeof(detail), "flags=%#" PRIx64, flags); + const char *name = thread_name(entry.thread); + if (name != entry.thread) + snprintf(entry.thread, sizeof(entry.thread), "%s", name); + record_alloc(device->bo_log, bo, &entry, detail, result); +} + +void pvr_bo_log_memory_alloc(struct pvr_device *device, + const VkMemoryAllocateInfo *info, + const struct pvr_device_memory *mem, + VkResult result) +{ + const VkImportMemoryFdInfoKHR *imp = + vk_find_struct_const(info->pNext, IMPORT_MEMORY_FD_INFO_KHR); + const VkMemoryDedicatedAllocateInfo *ded = + vk_find_struct_const(info->pNext, MEMORY_DEDICATED_ALLOCATE_INFO); + const bool wsi = + vk_find_struct_const(info->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA) != NULL; + struct pvr_bo_log_entry entry = { + .kind = "app", + .tag = "app", + .heap = "general", + .size = mem ? mem->bo->size : info->allocationSize, + .imported = imp && imp->handleType, + .display = wsi && device->ws->display_fd >= 0, + }; + if (wsi) + entry.tag = "wsi"; + else if (entry.imported) + entry.tag = "import"; + else if (vk_find_struct_const(info->pNext, EXPORT_MEMORY_ALLOCATE_INFO)) + entry.tag = "export"; + else if (ded && ded->image != VK_NULL_HANDLE) + entry.tag = "dedicated-image"; + else if (ded && ded->buffer != VK_NULL_HANDLE) + entry.tag = "dedicated-buffer"; + + char dedicated[48], detail[96]; + dedicated_word(info, dedicated, sizeof(dedicated)); + snprintf(detail, + sizeof(detail), + "type=%u dedicated=%s display=%u", + info->memoryTypeIndex, + dedicated, + entry.display); + const char *name = thread_name(entry.thread); + if (name != entry.thread) + snprintf(entry.thread, sizeof(entry.thread), "%s", name); + record_alloc(device->bo_log, mem, &entry, detail, result); +} + +void pvr_bo_log_free(struct pvr_device *device, const void *key) +{ + struct pvr_bo_log *log = device->bo_log; + simple_mtx_lock(&log->mutex); + struct hash_entry *h = _mesa_hash_table_search(log->entries, key); + if (h) { + struct pvr_bo_log_entry *e = h->data; + if (!strcmp(e->kind, "app")) { + log->app_live -= e->size; + log->app_count--; + if (e->imported) + log->import_live -= e->size; + else if (e->display) + log->display_live -= e->size; + } else { + log->drv_live -= e->size; + log->drv_count--; + } + if (e->size >= log->min_size) { + mesa_logi("pvr bo free kind=%s %s=%p size_kib=%" PRIu64 + " live_kib=%" PRIu64 "\n", + e->kind, + !strcmp(e->kind, "app") ? "mem" : "bo", + key, + e->size >> 10, + live_bytes(log) >> 10); + + } + hash_table_foreach (log->bindings, b) { + struct pvr_bo_log_binding *binding = b->data; + if (binding->memory == key) { + free(binding); + _mesa_hash_table_remove(log->bindings, b); + } + } + free(e); + _mesa_hash_table_remove(log->entries, h); + } + simple_mtx_unlock(&log->mutex); +} + +static void record_binding(struct pvr_bo_log *log, + const void *object, + const struct pvr_bo_log_binding *binding) +{ + struct hash_entry *old = _mesa_hash_table_search(log->bindings, object); + if (old) { + free(old->data); + _mesa_hash_table_remove(log->bindings, old); + } + struct pvr_bo_log_binding *copy = malloc(sizeof(*copy)); + if (copy) { + *copy = *binding; + if (!_mesa_hash_table_insert(log->bindings, object, copy)) { + free(copy); + log->tracking_failed++; + } + } else { + log->tracking_failed++; + } +} + +void pvr_bo_log_unbind(struct pvr_device *device, const void *object) +{ + struct pvr_bo_log *log = device->bo_log; + simple_mtx_lock(&log->mutex); + struct hash_entry *h = _mesa_hash_table_search(log->bindings, object); + if (h) { + free(h->data); + _mesa_hash_table_remove(log->bindings, h); + } + simple_mtx_unlock(&log->mutex); +} + +static const char *layout_name(enum pvr_memlayout layout) +{ + switch (layout) { + case PVR_MEMLAYOUT_LINEAR: + return "LINEAR"; + case PVR_MEMLAYOUT_TWIDDLED: + return "TWIDDLED"; + case PVR_MEMLAYOUT_3DTWIDDLED: + return "3DTWIDDLED"; + default: + return "UNKNOWN"; + } +} + +void pvr_bo_log_image(struct pvr_device *device, + const struct pvr_image *image, + const struct pvr_device_memory *mem, + uint64_t offset) +{ + struct pvr_bo_log *log = device->bo_log; + struct pvr_bo_log_binding binding = { + .memory = mem, + .size = image->total_size, + .image = true, + }; + const uint32_t bw = vk_format_get_blockwidth(image->vk.format); + const uint32_t bh = vk_format_get_blockheight(image->vk.format); + const uint32_t bs = vk_format_get_blocksize(image->vk.format); + for (unsigned l = 0; l < image->vk.mip_levels; l++) { + binding.tight += + (uint64_t)DIV_ROUND_UP(u_minify(image->vk.extent.width, l), bw) * + DIV_ROUND_UP(u_minify(image->vk.extent.height, l), bh) * bs * + u_minify(image->vk.extent.depth, l) * image->vk.array_layers * + image->vk.samples; + } + snprintf(binding.description, + sizeof(binding.description), + "image=%p off=%" PRIu64 + " fmt=%s extent=%ux%ux%u levels=%u layers=%u" + " samples=%u usage=%#" PRIx64 " flags=%#" PRIx64 + " tiling=%s layout=%s phys=%ux%u" + " size_kib=%" PRIu64 " tight_kib=%" PRIu64 " pad_kib=%" PRId64, + (const void *)image, + offset, + vk_Format_to_str(image->vk.format), + image->vk.extent.width, + image->vk.extent.height, + image->vk.extent.depth, + image->vk.mip_levels, + image->vk.array_layers, + image->vk.samples, + (uint64_t)image->vk.usage, + image->vk.create_flags, + image->vk.tiling == VK_IMAGE_TILING_OPTIMAL ? "optimal" : "linear", + layout_name(image->memlayout), + image->planes[0].physical_extent.width, + image->planes[0].physical_extent.height, + binding.size >> 10, + binding.tight >> 10, + ((int64_t)binding.size - (int64_t)binding.tight) / 1024); + simple_mtx_lock(&log->mutex); + record_binding(log, image, &binding); + if (binding.size >= log->min_size) { + mesa_logi("pvr bo bind-image mem=%p %s\n", + (const void *)mem, + binding.description); + + } + simple_mtx_unlock(&log->mutex); +} + +void pvr_bo_log_buffer(struct pvr_device *device, + const struct pvr_buffer *buffer, + const struct pvr_device_memory *mem, + uint64_t offset) +{ + struct pvr_bo_log *log = device->bo_log; + struct pvr_bo_log_binding binding = { + .memory = mem, + .size = buffer->vk.size, + }; + /* Match GetBufferMemoryRequirements2, including its read-ahead padding. */ + uint64_t req = buffer->vk.size; + if (req % device->ws->page_size == 0 || + req % device->ws->page_size > + device->ws->page_size - PVR_BUFFER_MEMORY_PADDING_SIZE) + req += PVR_BUFFER_MEMORY_PADDING_SIZE; + req = align64(req, buffer->alignment); + snprintf(binding.description, + sizeof(binding.description), + "buffer=%p off=%" PRIu64 " size_kib=%" PRIu64 " req_kib=%" PRIu64 + " usage=%#" PRIx64, + (const void *)buffer, + offset, + binding.size >> 10, + req >> 10, + (uint64_t)buffer->vk.usage); + simple_mtx_lock(&log->mutex); + record_binding(log, buffer, &binding); + if (binding.size >= log->min_size) { + mesa_logi("pvr bo bind-buffer mem=%p %s\n", + (const void *)mem, + binding.description); + + } + simple_mtx_unlock(&log->mutex); +} diff --git a/src/imagination/vulkan/pvr_bo_log.h b/src/imagination/vulkan/pvr_bo_log.h new file mode 100644 index 00000000000..28606747031 --- /dev/null +++ b/src/imagination/vulkan/pvr_bo_log.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef PVR_BO_LOG_H +#define PVR_BO_LOG_H + +#include +#include + +struct pvr_bo; +struct pvr_buffer; +struct pvr_device; +struct pvr_device_memory; +struct pvr_image; +struct pvr_winsys_heap; + +void pvr_bo_log_device_init(struct pvr_device *device); +void pvr_bo_log_device_finish(struct pvr_device *device); +void pvr_bo_log_summary(struct pvr_device *device, const char *why); +void pvr_bo_log_driver_alloc(struct pvr_device *device, + const char *tag, + struct pvr_winsys_heap *heap, + uint64_t size, + uint64_t flags, + const struct pvr_bo *bo, + VkResult result); +void pvr_bo_log_memory_alloc(struct pvr_device *device, + const VkMemoryAllocateInfo *info, + const struct pvr_device_memory *mem, + VkResult result); +void pvr_bo_log_free(struct pvr_device *device, const void *key); +void pvr_bo_log_image(struct pvr_device *device, + const struct pvr_image *image, + const struct pvr_device_memory *mem, + uint64_t offset); +void pvr_bo_log_buffer(struct pvr_device *device, + const struct pvr_buffer *buffer, + const struct pvr_device_memory *mem, + uint64_t offset); +void pvr_bo_log_unbind(struct pvr_device *device, const void *object); + +#endif /* PVR_BO_LOG_H */ diff --git a/src/imagination/vulkan/pvr_cmd_buffer.h b/src/imagination/vulkan/pvr_cmd_buffer.h index ac93575aa4b..622d85ea9b9 100644 --- a/src/imagination/vulkan/pvr_cmd_buffer.h +++ b/src/imagination/vulkan/pvr_cmd_buffer.h @@ -153,6 +153,9 @@ struct pvr_dynamic_render_info { struct pvr_sub_cmd_gfx { /* a vk_meta blit's render pass: transfer work for barriers (pvr_arch_g5p_meta.c) */ bool g5p_transfer; + /* a render pass G5P refuses (pvr_arch_g5p_job_init(): a shape it cannot run yet, named on stderr): + * not submitted, its draws lost, the command buffer valid (pvr_arch_queue.c) */ + bool g5p_refused; struct pvr_dynamic_render_info *dr_info; struct pvr_renderpass_hwsetup_render *hw_render; uint32_t attachment_count; @@ -273,6 +276,8 @@ struct pvr_sub_cmd_compute { /* A kernel record was emitted since the last end of pass: the next one * starts with iOS's barrier task (pvr_arch_g5p_compute.c). */ bool g5p_kernel_since_end; + struct pvr_g5p_dispatch_log *g5p_log_first, *g5p_log_last; + unsigned g5p_log_count; }; struct pvr_sub_cmd_transfer { diff --git a/src/imagination/vulkan/pvr_descriptor_set.h b/src/imagination/vulkan/pvr_descriptor_set.h index 255eeb03be6..4bb2cf56ef7 100644 --- a/src/imagination/vulkan/pvr_descriptor_set.h +++ b/src/imagination/vulkan/pvr_descriptor_set.h @@ -18,6 +18,7 @@ #include "util/vma.h" #include "vk_descriptor_set_layout.h" +#include "vk_util.h" #include "pvr_common.h" #include "pvr_macros.h" @@ -85,6 +86,79 @@ struct pvr_descriptor_set { struct pvr_buffer_descriptor dynamic_buffers[]; }; +/* Dynamic uniform and storage buffers have no descriptor memory in the set: + * their binding offset is ~0, and the buffer lives in set->dynamic_buffers[] + * with the offset vkCmdBindDescriptorSets gave it. A path that reads buffer + * descriptors from the set's device memory (a DMA from set->dev_addr) must + * take these from here instead. + */ +static inline bool +pvr_descriptor_set_binding_is_dynamic( + const struct pvr_descriptor_set_layout_binding *binding) +{ + return vk_descriptor_type_is_dynamic(binding->type); +} + +/* The bound buffer of a dynamic binding's element: its address with the + * dynamic offset applied, and its range from there. Returns false when the + * element was never written (no buffer). + */ +static inline bool +pvr_descriptor_set_dynamic_buffer( + const struct pvr_descriptor_set *set, + const struct pvr_descriptor_set_layout_binding *binding, + unsigned element, + uint64_t *addr, + uint32_t *range) +{ + const struct pvr_buffer_descriptor *d = + &set->dynamic_buffers[binding->dynamic_buffer_idx + element]; + + if (!d->addr) + return false; + + *addr = d->addr + d->offset; + *range = d->size; + return true; +} + +/* For a dispatch that DMAs a buffer slot's two dwords from descriptor + * memory (the A9 and A10 compute bind tasks): when the slot's binding is + * dynamic, the dwords it would read (the address, or the size word and the + * zero after it), to be uploaded instead. *dynamic is false for a binding + * that has descriptor memory. Fails for an unbound set, an element past the + * binding or an unwritten dynamic buffer. + */ +static inline bool +pvr_descriptor_set_dynamic_slot(const struct pvr_descriptor_set *set, + unsigned binding, + unsigned element, + bool size, + uint32_t out[static 2], + bool *dynamic) +{ + *dynamic = false; + if (!set || !set->layout || binding >= set->layout->binding_count) + return false; + + const struct pvr_descriptor_set_layout_binding *lb = + &set->layout->bindings[binding]; + if (!pvr_descriptor_set_binding_is_dynamic(lb)) + return true; + if (element >= MAX2(lb->descriptor_count, 1)) + return false; + + uint64_t addr; + uint32_t range; + if (!pvr_descriptor_set_dynamic_buffer(set, lb, element, &addr, &range)) + return false; + + out[0] = size ? range : addr; + out[1] = size ? 0 : addr >> 32; + *dynamic = true; + return true; +} + struct pvr_push_descriptor_set { struct pvr_descriptor_set_layout *layout; uint8_t data[PVR_MAX_PUSH_DESCRIPTORS * PVR_MAX_DESCRIPTOR_SIZE]; /* CPU-side descriptor blob */ diff --git a/src/imagination/vulkan/pvr_device.c b/src/imagination/vulkan/pvr_device.c index f1334d55132..edf5913296e 100644 --- a/src/imagination/vulkan/pvr_device.c +++ b/src/imagination/vulkan/pvr_device.c @@ -27,6 +27,7 @@ * SOFTWARE. */ +#include "pvr_bo_log.h" #include "pvr_device.h" #include @@ -60,6 +61,7 @@ #include "pvr_winsys.h" #include "pvr_wsi.h" #include "util/log.h" +#include "util/os_time.h" #include "util/macros.h" #include "util/mesa-blake3.h" #include "util/os_misc.h" @@ -190,6 +192,22 @@ void pvr_rstate_entry_remove(struct pvr_device *device, simple_mtx_unlock(&device->rs_mtx); } +/* Inspect job errors only for blocking completion waits, on the logical + * device's winsys. Polling and availability waits retain their usual cost. */ +VkResult pvr_sync_wait_many(struct vk_device *vk, uint32_t count, + const struct vk_sync_wait *waits, + enum vk_sync_wait_flags flags, uint64_t timeout) +{ + struct pvr_device *device = container_of(vk, struct pvr_device, vk); + const bool slow = !(flags & VK_SYNC_WAIT_PENDING) && + timeout > (uint64_t)os_time_get_nano(); + VkResult result = device->pdevice->native_wait_many(vk, count, waits, flags, timeout); + if (result == VK_SUCCESS && slow && device->ws->ops->check_job_fences && + device->ws->ops->check_job_fences(device->ws, count, waits) == VK_ERROR_DEVICE_LOST) + return vk_device_check_status(vk); + return result; +} + VkResult pvr_AllocateMemory(VkDevice _device, const VkMemoryAllocateInfo *pAllocateInfo, const VkAllocationCallbacks *pAllocator, @@ -197,6 +215,15 @@ VkResult pvr_AllocateMemory(VkDevice _device, { const VkImportMemoryFdInfoKHR *fd_info = NULL; VK_FROM_HANDLE(pvr_device, device, _device); + + if (vk_device_is_lost_no_report(&device->vk)) { + if (unlikely(device->bo_log)) + pvr_bo_log_memory_alloc(device, + pAllocateInfo, + NULL, + VK_ERROR_OUT_OF_DEVICE_MEMORY); + return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); + } enum pvr_winsys_bo_type type = PVR_WINSYS_BO_TYPE_GPU; struct pvr_device_memory *mem; VkResult result; @@ -206,21 +233,31 @@ VkResult pvr_AllocateMemory(VkDevice _device, const VkMemoryType *mem_type = &device->pdevice->memory.memoryTypes[pAllocateInfo->memoryTypeIndex]; - const VkMemoryHeap *mem_heap = - &device->pdevice->memory.memoryHeaps[mem_type->heapIndex]; VkDeviceSize aligned_alloc_size = ALIGN_POT(pAllocateInfo->allocationSize, device->ws->page_size); - if (aligned_alloc_size > mem_heap->size) + if (aligned_alloc_size > device->pdevice->vk.properties.maxMemoryAllocationSize) { + if (unlikely(device->bo_log)) + pvr_bo_log_memory_alloc(device, + pAllocateInfo, + NULL, + VK_ERROR_OUT_OF_DEVICE_MEMORY); return VK_ERROR_OUT_OF_DEVICE_MEMORY; + } mem = vk_device_memory_create(&device->vk, pAllocateInfo, pAllocator, sizeof(*mem)); - if (!mem) + if (!mem) { + if (unlikely(device->bo_log)) + pvr_bo_log_memory_alloc(device, + pAllocateInfo, + NULL, + VK_ERROR_OUT_OF_HOST_MEMORY); return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + } vk_foreach_struct_const (sType, ext, pAllocateInfo->pNext) { switch ((unsigned)sType) { @@ -315,6 +352,8 @@ VkResult pvr_AllocateMemory(VkDevice _device, } *pMem = pvr_device_memory_to_handle(mem); + if (unlikely(device->bo_log)) + pvr_bo_log_memory_alloc(device, pAllocateInfo, mem, VK_SUCCESS); vk_device_memory_report_emit(&device->vk, VK_SUCCESS, /* is_alloc */ true, mem->vk.import_handle_type != 0, @@ -326,6 +365,8 @@ VkResult pvr_AllocateMemory(VkDevice _device, return VK_SUCCESS; err_vk_device_memory_destroy: + if (unlikely(device->bo_log)) + pvr_bo_log_memory_alloc(device, pAllocateInfo, NULL, result); vk_device_memory_report_emit(&device->vk, result, /* is_alloc */ true, mem->vk.import_handle_type != 0, mem->bo ? (uintptr_t)mem->bo : 0, @@ -408,6 +449,8 @@ void pvr_FreeMemory(VkDevice _device, if (mem->bo->map) device->ws->ops->buffer_unmap(mem->bo, false); + if (unlikely(device->bo_log)) + pvr_bo_log_free(device, mem); device->ws->ops->buffer_destroy(mem->bo); vk_device_memory_destroy(&device->vk, pAllocator, &mem->vk); @@ -643,11 +686,15 @@ VkResult pvr_BindBufferMemory2(VkDevice _device, if (result != VK_SUCCESS) { while (i--) { VK_FROM_HANDLE(pvr_buffer, buffer, pBindInfos[i].buffer); + if (unlikely(device->bo_log)) + pvr_bo_log_unbind(device, buffer); pvr_unbind_memory(device, buffer->vma); } return result; } + if (unlikely(device->bo_log)) + pvr_bo_log_buffer(device, buffer, mem, pBindInfos[i].memoryOffset); } return VK_SUCCESS; @@ -832,6 +879,9 @@ void pvr_DestroyBuffer(VkDevice _device, if (!buffer) return; + if (unlikely(device->bo_log)) + pvr_bo_log_unbind(device, buffer); + if (buffer->vma) pvr_unbind_memory(device, buffer->vma); diff --git a/src/imagination/vulkan/pvr_device.h b/src/imagination/vulkan/pvr_device.h index 8ebde5092b7..022b999dc3a 100644 --- a/src/imagination/vulkan/pvr_device.h +++ b/src/imagination/vulkan/pvr_device.h @@ -70,6 +70,7 @@ struct pvr_compute_query_shader { struct pvr_device { struct vk_device vk; + struct vk_device_dispatch_table *errlog_orig; struct pvr_instance *instance; struct pvr_physical_device *pdevice; @@ -216,6 +217,7 @@ struct pvr_device { struct pvr_spm_scratch_buffer_store spm_scratch_buffer_store; struct pvr_bo_store *bo_store; + struct pvr_bo_log *bo_log; struct pvr_bo *robustness_buffer; @@ -282,6 +284,10 @@ static inline struct pvr_device *vk_to_pvr_device(struct vk_device *device) return container_of(device, struct pvr_device, vk); } +VkResult pvr_sync_wait_many(struct vk_device *vk, uint32_t count, + const struct vk_sync_wait *waits, + enum vk_sync_wait_flags flags, uint64_t timeout); + VkResult pvr_device_tile_buffer_ensure_cap(struct pvr_device *device, uint32_t capacity); diff --git a/src/imagination/vulkan/pvr_drirc_gen.py b/src/imagination/vulkan/pvr_drirc_gen.py index aef105ca8e3..34b7e4f4167 100644 --- a/src/imagination/vulkan/pvr_drirc_gen.py +++ b/src/imagination/vulkan/pvr_drirc_gen.py @@ -17,7 +17,8 @@ def declare_options(): debug_options = [] features_options = [] performance_options = [] - misc_options = [] + misc_options = [drirc_gen.DrircBool("pvr_skip_refused_work", False, + "Skip unsupported work, allowing incorrect rendering", c_name="pvr_skip_refused_work")] drirc_gen.add_common_vk_options(debug_options, features_options, misc_options, valid_options=VALID_COMMON_VK_OPTIONS) diff --git a/src/imagination/vulkan/pvr_errlog.c b/src/imagination/vulkan/pvr_errlog.c new file mode 100644 index 00000000000..9a87d7953c5 --- /dev/null +++ b/src/imagination/vulkan/pvr_errlog.c @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: MIT */ +#include "pvr_errlog.h" +#include "pvr_device.h" +#include "pvr_debug.h" +#include "pvr_g5p_shader_dump.h" +#include "util/log.h" +#include "vk_alloc.h" +#include "vk_log.h" +#include "vk_command_buffer.h" +#include "vk_queue.h" +#include "vk_enum_to_str.h" +#include +#include +#include +#include + +static void pvr_errlog_result(const char *entry, VkResult result) +{ + mesa_logw("pvr: %s returned %s", entry, vk_Result_to_str(result)); + const char *dir = pvr_g5p_shader_dump_dir(); + if (!dir) + return; + char path[4096], line[640]; + snprintf(path, sizeof(path), "%s/errors.tsv", dir); + int n = snprintf(line, sizeof(line), "%lld\t%ld\tresult\t%s\t%s\n", + (long long)time(NULL), (long)getpid(), vk_Result_to_str(result), entry); + int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (fd >= 0) { + ssize_t written = write(fd, line, MIN2(n, (int)sizeof(line) - 1)); + (void)written; + close(fd); + } +} + +#include "pvr_errlog_gen.h" + +VkResult pvr_errlog_install(struct pvr_device *device) +{ + if (!PVR_IS_DEBUG_SET(ERRORS)) + return VK_SUCCESS; + device->errlog_orig = vk_alloc(&device->vk.alloc, sizeof(*device->errlog_orig), + 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); + if (!device->errlog_orig) + return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + pvr_errlog_patch(&device->vk.dispatch_table, device->errlog_orig); + return VK_SUCCESS; +} diff --git a/src/imagination/vulkan/pvr_errlog.h b/src/imagination/vulkan/pvr_errlog.h new file mode 100644 index 00000000000..8d290249a88 --- /dev/null +++ b/src/imagination/vulkan/pvr_errlog.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef PVR_ERRLOG_H +#define PVR_ERRLOG_H +#include +struct pvr_device; +VkResult pvr_errlog_install(struct pvr_device *device); +#endif diff --git a/src/imagination/vulkan/pvr_errlog_gen.py b/src/imagination/vulkan/pvr_errlog_gen.py new file mode 100644 index 00000000000..743d96da361 --- /dev/null +++ b/src/imagination/vulkan/pvr_errlog_gen.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Generate per-device error-result wrappers from the Vulkan registry.""" +import argparse +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'vulkan' / 'util')) +from vk_entrypoints import get_entrypoints_from_xml + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--xml', required=True) + parser.add_argument('--out', required=True) + args = parser.parse_args() + entries = [e for e in get_entrypoints_from_xml([args.xml], beta=False) + if e.is_device_entrypoint() and e.return_type == 'VkResult' + and e.alias is None] + out = ['/* SPDX-License-Identifier: MIT */', + '/* Generated from vk.xml; do not edit. */', ''] + for e in entries: + if e.guard: + out += [f'#ifdef {e.guard}'] + p = e.params[0] + device = {'VkDevice': f'vk_device_from_handle({p.name})', + 'VkQueue': f'vk_queue_from_handle({p.name})->base.device', + 'VkCommandBuffer': f'vk_command_buffer_from_handle({p.name})->base.device'}[p.type] + out += ['static VKAPI_ATTR VkResult VKAPI_CALL', + f'pvr_errlog_{e.name}({e.decl_params()})', '{', + f' struct vk_device *pvr_errlog_vk = {device};', + ' struct pvr_device *pvr_errlog_device = container_of(pvr_errlog_vk, struct pvr_device, vk);', + f' VkResult r = pvr_errlog_device->errlog_orig->{e.name}({e.call_params()});', + ' if (r < 0)', f' pvr_errlog_result("vk{e.name}", r);', + ' return r;', '}'] + if e.guard: + out += ['#endif'] + out += [''] + out += ['static void', 'pvr_errlog_patch(struct vk_device_dispatch_table *t,', + ' struct vk_device_dispatch_table *orig)', '{', ' *orig = *t;'] + for e in entries: + if e.guard: + out += [f'#ifdef {e.guard}'] + out += [f' if (t->{e.name})', f' t->{e.name} = pvr_errlog_{e.name};'] + if e.guard: + out += ['#endif'] + out += ['}', ''] + Path(args.out).write_text('\n'.join(out)) + + +if __name__ == '__main__': + main() diff --git a/src/imagination/vulkan/pvr_formats.c b/src/imagination/vulkan/pvr_formats.c index 5ac7f41daec..9c92664d401 100644 --- a/src/imagination/vulkan/pvr_formats.c +++ b/src/imagination/vulkan/pvr_formats.c @@ -32,6 +32,7 @@ #include "hwdef/rogue_hw_utils.h" #include "pvr_g5p_ds_hooks.h" # include "pvr_g5p_vertex.h" +#include "pvr_g5p_format_table.h" #include "pvr_common.h" #include "pvr_csb.h" #include "pvr_device.h" @@ -239,6 +240,23 @@ pvr_get_format(struct pvr_physical_device *pdevice, VkFormat vk_format) pdevice->formats.formats[format].bind != 0) { return &pdevice->formats.formats[format]; } + /* G5P: ASTC LDR and the 16-bit packed forms Rogue lacks (A1R5G5B5, + * B5G6R5, R4G4B4A4, ...) exist only in the A9's own table (Rogue's table, + * which G9P shares, has none); the G5P hooks take their features and + * texture state from that row (pvr_g5p_format_table.h) */ + if (PVR_HAS_FEATURE(&pdevice->dev_info, apple_g5p) && + (vk_format_description(vk_format)->layout == UTIL_FORMAT_LAYOUT_ASTC || + vk_format_get_blocksize(vk_format) == 2) && + pvr_g5p_format_find(vk_format)) { + static const struct pvr_format g5p_astc = { + /* Rogue's texture-format codes are not used on G5P (its own key is) */ + .tex_format = ~0u, + .depth_tex_format = ~0u, + .stencil_tex_format = ~0u, + .bind = PVR_BIND_SAMPLER_VIEW, + }; + return &g5p_astc; + } mesa_logd("Format %s(%d) not supported\n", vk_Format_to_str(vk_format), @@ -437,15 +455,17 @@ pvr_get_format_swizzle_for_tpu(const struct util_format_description *desc) } /* G5P fetches the vertex formats of pvr_g5p_vertex_format_info() (32-bit - * float and integer, 8/16-bit normalized and scaled, 16-bit float) with - * dword strides. Of those, it advertises the ones a whole number of dwords - * in size, whose tightly packed strides are dword multiples too (Zink passes - * GL's strides through), and no other, so that Zink converts the rest. */ + * float and integer, 8/16-bit normalized, scaled and pure integer, 16-bit + * float) as whole dwords and advertises all of them: a binding whose address + * or stride is off dwords (a tightly packed R8 or RGB16 array) draws from a + * dword-aligned copy, as does one whose last element sits at the end of the + * buffer's GPU mapping (pvr_arch_g5p_gfx.c). */ static bool pvr_g5p_vertex_buffer_format(VkFormat vk_format) { - return pvr_g5p_vertex_format_info(vk_format, &(struct pvr_g5p_vertex_format){ 0 }) && - vk_format_get_blocksize(vk_format) % 4 == 0; + /* Any size: the fetch reads whole dwords, and a read that could pass the end of the buffer's GPU mapping + * draws from a padded copy instead (pvr_arch_g5p_gfx.c, pvr_g5p_fetch.overread) */ + return pvr_g5p_vertex_format_info(vk_format, &(struct pvr_g5p_vertex_format){ 0 }); } static VkFormatFeatureFlags2 diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index a0560f31ac5..3f1b72011e9 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -20,8 +20,39 @@ #include #include +/* Zink retries device-memory errors; wgpu survives only OOM failures. + * Keep refusal internal and map it at creation/recording API boundaries. */ +static inline VkResult pvr_refusal_to_api(VkResult result) +{ + return result == VK_ERROR_FEATURE_NOT_PRESENT ? VK_ERROR_OUT_OF_HOST_MEMORY : result; +} + +#include "util/log.h" +#include "util/u_atomic.h" +#include "vk_log.h" + +/* Refusal kinds are keyed by their call site (pipeline validation or pass/draw + * validation). Count atomically across recording threads; warn on occurrences + * 1, 2, 4, ... so tolerant workloads cannot emit one warning per draw. Keep + * vk_errorf for debug callbacks and preserve the internal refusal result. */ +#define G5P_REFUSE(obj, ...) \ + ({ \ + static uint32_t refusal_count; \ + uint32_t occurrence = p_atomic_inc_return(&refusal_count); \ + if (occurrence && !(occurrence & (occurrence - 1))) \ + mesa_logw("G5P refuses: " __VA_ARGS__); \ + vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__); \ + }) + #include "pvr_common.h" #include "pvr_limits.h" + +/* User clip distances are written to the A9 TA user planes. */ +static inline bool +pvr_g5p_clip_distance_probe(void) +{ + return true; +} #include "pvr_macros.h" #include "util/simple_mtx.h" #include "vk_meta.h" @@ -35,8 +66,9 @@ struct pvr_renderpass_hwsetup_render; struct pvr_sub_cmd_gfx; struct pvr_suballoc_bo; -/* Colour-target counts with programs: 1, 2 and 4, the counts with an Apple witness (others are refused) */ -#define PVR_G5P_MRT_VARIANTS 3 +/* Colour-target counts with programs: 1, 2 and 4 (Apple's packed-f16 pass), then 5..8 compact UNORM8 + * with a BGRA8 clear (variants 3..6) and an RGBA8 clear (7..10). 3 runs as 4. */ +#define PVR_G5P_MRT_VARIANTS 11 /* * The device-level programs every G5P render uses, uploaded once per @@ -84,11 +116,15 @@ struct pvr_g5p_programs { unsigned load_temps, load_common_dwords, load_uploads; struct { unsigned kind; /* PCO_G5P_UPLOAD_IMAGE, _SAMPLER or _IMAGE_SIZE */ - unsigned dwords, common; - } load_upload[3]; + unsigned dwords, common, binding; /* binding: the target an IMAGE upload reads (5..8 targets) */ + } load_upload[32]; char load_refusal[128]; /* Lazy integer variants: float device allocation order remains identical. */ struct pvr_g5p_programs *raw_load[11]; /* scalar R8 uses10/11, separate fromRGBA8 */ + /* Lazy compact LOAD programs: 5..8 targets, BGRA8 [0..3] and RGBA8 [4..7] */ + struct pvr_g5p_programs *mrt_load[8]; + /* Lazy 4x LOAD (one UNORM/float target, pvr_g5p_compile_load_ms()), kicked at sample rate */ + struct pvr_g5p_programs *ms_load; /* raw clear programs by total output words: 2, 4, 8 (two 32-bit integer targets) */ struct pvr_suballoc_bo *raw_clear_usc[4]; /*2/4/8 words, then scalar1 */ /* raw end-of-tile programs by target count: 1, 2 */ @@ -177,6 +213,9 @@ enum pvr_g5p_fetch_rate { struct pvr_g5p_fetch { uint8_t binding, dwords, ao, rate; uint32_t offset, divisor; + /* bytes the fetch reads past the end of its binding's last attribute in the element (the fetch reads whole + * dwords, the ICD layout 16 bytes a location): the draw keeps that inside the buffer's GPU mapping */ + uint16_t overread; }; /* One common-store upload of the fragment program (the backend's @@ -218,6 +257,11 @@ struct pvr_g5p_pipeline { * pointers and sizes (6 words) */ struct pvr_suballoc_bo *static_vary, *static_frag; uint32_t static_vary_words[4], static_frag_words[6]; + /* flat varyings from the provoking vertex 1 and 2 (pvr_arch_g5p_gfx.c, PVR_G5P_FLATSHADE_CONTROL()): the + * coefficient program with FLAT_VERTEX1/2 and the fragment pointers naming it; [0] is varying/static_frag */ + struct pvr_pds_upload varying_pv[2]; + struct pvr_suballoc_bo *static_frag_pv[2]; + uint32_t static_frag_pv_words[2][6]; /* the vertex program: DOUTU temps, output dwords, unified-store size, * system values the fetch writes (bit 0 vertex index, 1 instance index) * and their registers */ @@ -235,6 +279,8 @@ struct pvr_g5p_pipeline { * Original colour-only programs remain byte-identical. */ struct pvr_g5p_pipeline *punch; bool punch_variant; + const char *punch_why; /* no punch variant: why (static string; g5p_pipeline_build_punch()) */ + bool fs_mask_upass; /* Apple's masked form: OPAQUE / FAST_PUNCH_THROUGH, upass 7 (pco_g5p_fs_key::mask_upass) */ /* the program writes memory (pco_g5p_program::side_effects): a fragment * program's draws bound the shaded pixels with the ISP scissor; a vertex * program's draws run even when no pixel is left (pvr_arch_g5p_gfx.c) */ @@ -248,6 +294,7 @@ struct pvr_g5p_pipeline { * ISP object type, as AGXMetalA9 encodeISPPrimitiveType picks it from * its triangle fill mode (desktop-gl-polygon-mode.md) */ VkPolygonMode polygon_mode; + unsigned vs_clip_count; /* user clip distances: TA user planes 0..n-1 (Apple's VertexProgramVariant) */ /* the fragment program's tile read (pco_g5p_program::blend_mode: 0 none) * and colour write mask, for the ISP pass type (pvr_g5p_blend.h) */ unsigned fs_blend_mode; @@ -310,6 +357,12 @@ bool pvr_g5p_compile_load_rgb10a2ui(struct pco_g5p_program *out); bool pvr_g5p_compile_load_rgba8ui(struct pco_g5p_program *out); bool pvr_g5p_compile_load_rgba16ui(struct pco_g5p_program *out); bool pvr_g5p_compile_load_raw(struct pco_g5p_program *out, bool is_signed); +/* the 4x LOAD background (Apple's loader at sample rate, pvr_g5p_nir.c) */ +#define PVR_G5P_HAVE_LOAD_MS 1 +bool pvr_g5p_compile_load_ms(struct pco_g5p_program *out); +/* The compact LOAD background of 5..8 UNORM8 targets: texelFetch of every + * target (set 0, binding t), packed as the compact output. */ +bool pvr_g5p_compile_load_mrt(struct pco_g5p_program *out, unsigned targets, bool bgra); /* The clear quad's stages, built with nir_builder (pvr_g5p_nir.c): the * vertex stage passes its vec4 position at location 0 through, the * fragment stage writes the push constant's colour (bytes 0..15) to colour @@ -423,4 +476,12 @@ uint32_t PVR_PER_ARCH(g5p_compute_stream)(const struct pvr_device *device, uint6 #endif /* PVR_PER_ARCH */ +/* VDM_STATE0 flatshade_control (Rogue's field, bits 1-2): the vertex of a strip's or fan's triangle whose flat + * varyings the triangle takes. 2 for the last provoking vertex, 1 for fans, 0 otherwise (G9P's 5a7204430d1 ("pvr/g9p: the flat colour's vertex in VDM_STATE0"): without it odd strip triangles took another vertex's colour, dEQP + * rasterization.flatshading.triangle_strip). @dyn: a vk_dynamic_graphics_state. */ +#define PVR_G5P_FLATSHADE_CONTROL(dyn) \ + ((dyn)->rs.provoking_vertex == VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT ? 2u \ + : (dyn)->ia.primitive_topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN ? 1u \ + : 0u) + #endif /* PVR_G5P_H */ diff --git a/src/imagination/vulkan/pvr_g5p_compute.h b/src/imagination/vulkan/pvr_g5p_compute.h index 9cf9bb40d54..a2f54cf550f 100644 --- a/src/imagination/vulkan/pvr_g5p_compute.h +++ b/src/imagination/vulkan/pvr_g5p_compute.h @@ -2,6 +2,7 @@ #ifndef PVR_G5P_COMPUTE_H #define PVR_G5P_COMPUTE_H #include +#include #include #include "pvr_macros.h" #include "pvr_winsys.h" @@ -11,9 +12,16 @@ struct pvr_cmd_buffer; struct pvr_sub_cmd_compute; struct pvr_compute_ctx; struct vk_sync; +/* Owned by its command-buffer sub-command; immutable across submits. */ +struct pvr_g5p_dispatch_log { + struct pvr_g5p_dispatch_log *next; + char *line; + char pipeline[13]; + bool stub; +}; #ifdef PVR_PER_ARCH VkResult PVR_PER_ARCH(g5p_compute_pipeline_compile)(struct pvr_device *, const VkComputePipelineCreateInfo *, struct pvr_compute_pipeline *); -VkResult PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *, struct pvr_sub_cmd_compute *, uint64_t, const uint32_t[3], const uint32_t[3]); +VkResult PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *, struct pvr_sub_cmd_compute *, uint64_t, uint64_t, const uint32_t[3], const uint32_t[3]); VkResult PVR_PER_ARCH(g5p_compute_end_pass)(struct pvr_cmd_buffer *, struct pvr_sub_cmd_compute *); VkResult PVR_PER_ARCH(g5p_compute_job_submit)(struct pvr_compute_ctx *, struct pvr_sub_cmd_compute *, struct vk_sync *, struct vk_sync *); VkResult PVR_PER_ARCH(g5p_compute_ctx_create)(struct pvr_device *, enum pvr_winsys_ctx_priority, struct pvr_compute_ctx **); diff --git a/src/imagination/vulkan/pvr_g5p_compute_hooks.h b/src/imagination/vulkan/pvr_g5p_compute_hooks.h index ffad1f784df..8081d307603 100644 --- a/src/imagination/vulkan/pvr_g5p_compute_hooks.h +++ b/src/imagination/vulkan/pvr_g5p_compute_hooks.h @@ -24,7 +24,7 @@ } #define PVR_G5P_HOOK_compute_dispatch() \ if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g5p)) { \ - result = pvr_arch_g5p_cmd_dispatch(cmd_buffer, sub_cmd, indirect_addr.addr, base_group, workgroup_size); \ + result = pvr_arch_g5p_cmd_dispatch(cmd_buffer, sub_cmd, indirect_addr.addr, indirect_offset, base_group, workgroup_size); \ if (result != VK_SUCCESS) pvr_cmd_buffer_set_error_unwarned(cmd_buffer, result); \ goto end_cmd_dispatch; \ } diff --git a/src/imagination/vulkan/pvr_g5p_compute_state.h b/src/imagination/vulkan/pvr_g5p_compute_state.h index 8e2bb1afceb..727502925bb 100644 --- a/src/imagination/vulkan/pvr_g5p_compute_state.h +++ b/src/imagination/vulkan/pvr_g5p_compute_state.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: MIT */ #ifndef PVR_G5P_COMPUTE_STATE_H #define PVR_G5P_COMPUTE_STATE_H -#define PVR_G5P_COMPUTE_STATE_FIELDS bool a9_bodyless; bool a9_unit_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; +#define PVR_G5P_COMPUTE_STATE_FIELDS unsigned shared_spill_stride; int shared_spill_logged; char a9_pipeline_id[13]; bool a9_capture_stub; bool a9_compute_log; unsigned a9_log_local[3], a9_log_shared; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ + unsigned a9_num_uploads; struct { unsigned char kind, set, binding, element, combined, level, common, dwords; uint32_t value; } a9_uploads[10]; #define PVR_G5P_NATIVE_STORAGE(d) PVR_HAS_FEATURE((d), apple_g5p) #endif diff --git a/src/imagination/vulkan/pvr_g5p_draw_hooks.h b/src/imagination/vulkan/pvr_g5p_draw_hooks.h index 8e2ee175711..2f0c62a2336 100644 --- a/src/imagination/vulkan/pvr_g5p_draw_hooks.h +++ b/src/imagination/vulkan/pvr_g5p_draw_hooks.h @@ -22,8 +22,7 @@ result = pvr_flush_push_descriptors(cmd_buffer, &state->gfx_desc_state); \ if (result == VK_SUCCESS) \ result = pvr_arch_g5p_cmd_draw_indirect(cmd_buffer, buffer, offset, drawCount, stride, true); \ - if (result != VK_SUCCESS) \ - vk_command_buffer_set_error(&cmd_buffer->vk, result); \ + g5p_draw_result(cmd_buffer, result); /* a refusal skips the draw */ \ goto end_cmd_draw_indexed_indirect; \ } #define PVR_G5P_HOOK_g5p_indirect() \ @@ -31,8 +30,7 @@ result = pvr_flush_push_descriptors(cmd_buffer, &state->gfx_desc_state); \ if (result == VK_SUCCESS) \ result = pvr_arch_g5p_cmd_draw_indirect(cmd_buffer, buffer, offset, drawCount, stride, false); \ - if (result != VK_SUCCESS) \ - vk_command_buffer_set_error(&cmd_buffer->vk, result); \ + g5p_draw_result(cmd_buffer, result); /* a refusal skips the draw */ \ goto end_cmd_draw_indirect; \ } diff --git a/src/imagination/vulkan/pvr_g5p_ds.c b/src/imagination/vulkan/pvr_g5p_ds.c index 911ce58e565..5b05cb80f5d 100644 --- a/src/imagination/vulkan/pvr_g5p_ds.c +++ b/src/imagination/vulkan/pvr_g5p_ds.c @@ -21,6 +21,7 @@ #include "csbgen/rogue/ppp.h" #include "pvr_g5p_ds.h" +#include "pvr_g5p_shader_dump.h" static uint32_t g5p_pot(uint32_t v) @@ -131,8 +132,9 @@ pvr_g5p_ds_zls(const struct pvr_g5p_ds_target *t, struct pvr_g5p_zls_words *out) const bool d = t->depth && f->depth, s = t->stencil && f->stencil; const bool zload = d && t->zload, zstore = d && t->zstore; const bool sload = s && t->sload, sstore = s && t->sstore; - if (t->samples == 4 && (zload || zstore || sload || sstore)) - return "a load or store of a 4-sample depth/stencil attachment (no multisample ZLS capture: it stays on chip)"; + /* 4 samples: the same words over the 4x plane (iOS captures of 4x depth store, depth load + * and stencil: ZLSCTL 0xe2004 / 0xaa006 / 0x26002 and ZLS_PIXELS 0x1f803f at 64x64, as at one sample; the + * stencil plane at depth + 0x10000, four times the one-sample plane) */ const struct ROGUE_CR_ISP_ZLSCTL ctl = { .loadtwiddled = true, diff --git a/src/imagination/vulkan/pvr_g5p_format_table.h b/src/imagination/vulkan/pvr_g5p_format_table.h index a4d4e57ffe4..68797f3d243 100644 --- a/src/imagination/vulkan/pvr_g5p_format_table.h +++ b/src/imagination/vulkan/pvr_g5p_format_table.h @@ -16,13 +16,13 @@ * 0x21 32, 0x31 32_32, 0x38 32_32_32_32, 0x26 10_10_10_2, 0x25 11_11_10, * 0x27 9_9_9_e5, 0x0b 5_6_5, 0x0d 1_5_5_5, 0x0c 4_4_4_4; block compressed * 0x58 ETC2 RGB8, 0x5a ETC2 RGB8A1, 0x59 ETC2 RGBA8 (EAC alpha), 0x5b EAC R11, - * 0x5c EAC RG11. Selectors: 0-3 a component (component 0 is the lowest + * 0x5c EAC RG11, 0x60..0x6d ASTC LDR 4x4..12x12. Selectors: 0-3 a component (component 0 is the lowest * field of a packed format), 4 one, 5 zero. * * What a row advertises is what has run on the A9 through this driver: every - * row's rungs of the formats hardware test (src/imagination/g5p/tests/formats) - * passed on the device, every pixel exact (2026-11-04: 118 rungs of the rows - * advertised then, and 39 rungs of the rows decoded but not yet run, the + * row's cases of the formats hardware test (src/imagination/g5p/tests/formats) + * passed on the device, every pixel exact (2026-11-04: 118 cases of the rows + * advertised then, and 39 cases of the rows decoded but not yet run, the * probes, promoted since). Before that: * - sampling: the texture state is format-independent (lane b's * pvr_arch_g5p_tex.c); each key below was sampled exactly on the device by @@ -36,7 +36,7 @@ * packs R8, RG8, R16F, RG16F and RG11B10F from the two-word program too: * exact on the device. Blending is the fragment program's, on the tile's * f16 values before the PBE, for every rendered row; - * - probes (for a device rung only): rows decoded or + * - probes (for a device case only): rows decoded or * derived but not yet run, not advertised by default. None at present. */ @@ -62,9 +62,10 @@ enum pvr_g5p_fmt_flags { PVR_G5P_FMT_FILTER = 1u << 1, /* linear filtering */ PVR_G5P_FMT_RENDER = 1u << 2, /* colour attachment: the two-word program + this PBE key */ PVR_G5P_FMT_BLEND = 1u << 3, /* blending into it (the fragment program's) */ - PVR_G5P_FMT_COMPRESSED = 1u << 4, /* 4x4 blocks, twiddled only */ + PVR_G5P_FMT_COMPRESSED = 1u << 4, /* blocks (4x4 ETC2/EAC, 4x4..12x12 ASTC), twiddled only */ PVR_G5P_FMT_INTEGER = 1u << 5, /* integer texel: texelFetch/unfiltered */ - /* not advertised (none at present) */ + PVR_G5P_FMT_STORAGE = 1u << 6, /* compute writes with a declared format */ + /* narrow integer render rows remain unadvertised probes */ PVR_G5P_FMT_PROBE_SAMPLED = 1u << 8, /* the sampled bits are a probe */ PVR_G5P_FMT_PROBE_RENDER = 1u << 9, /* the render (and blend) bits are a probe */ PVR_G5P_FMT_PROBE_BLEND = 1u << 10, /* the blend bit alone is a probe */ @@ -72,16 +73,17 @@ enum pvr_g5p_fmt_flags { /* Where a row's key and its advertised use come from */ enum pvr_g5p_fmt_evidence { - PVR_G5P_FMT_EV_G5P1B, /* device-proven on g5p-1b (tex-blend 22/22, formats 118 + 39 rungs) */ + PVR_G5P_FMT_EV_G5P1B, /* device-proven on g5p-1b (tex-blend 22/22, formats 118 + 39 cases) */ PVR_G5P_FMT_EV_ICD, /* captured key, device-proven by our ICD (gles3tex, textypes) */ PVR_G5P_FMT_EV_RULE, /* derived from the decoded code rule, not captured */ PVR_G5P_FMT_EV_CAPTURE, /* captured key, never run on the device */ + PVR_G5P_FMT_EV_AGX, /* AGXMetalA9's own texFormat row (code and swizzle words), never run on the device */ }; struct pvr_g5p_format { VkFormat format; uint32_t key; - uint8_t cpp; /* bytes per texel, or per 4x4 block */ + uint8_t cpp; /* bytes per texel, or per block */ bool srgb; uint16_t flags; uint8_t evidence; @@ -114,7 +116,7 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { /* 8_8_8_8: pass/pass-rgba (BGRA8, RGBA8), fmt-srgb8 (q1 bit 45). The * A8B8G8R8 packed formats are the R8G8B8A8 bytes on a little-endian GPU. */ G5P_FMT(B8G8R8A8_UNORM, 4, false, G5P_SF | G5P_RT, G5P1B, 0x28, G5P_SEL_BGRA), - G5P_FMT(R8G8B8A8_UNORM, 4, false, G5P_SF | G5P_RT, G5P1B, 0x28, G5P_SEL_RGBA), + G5P_FMT(R8G8B8A8_UNORM, 4, false, G5P_SF | G5P_RT | PVR_G5P_FMT_STORAGE, G5P1B, 0x28, G5P_SEL_RGBA), G5P_FMT(A8B8G8R8_UNORM_PACK32, 4, false, G5P_SF | G5P_RT, G5P1B, 0x28, G5P_SEL_RGBA), G5P_FMT(B8G8R8A8_SRGB, 4, true, G5P_SF | G5P_RT, G5P1B, 0x28, G5P_SEL_BGRA), G5P_FMT(R8G8B8A8_SRGB, 4, true, G5P_SF | G5P_RT, G5P1B, 0x28, G5P_SEL_RGBA), @@ -143,13 +145,13 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { G5P_FMT(R16G16B16A16_SNORM, 8, false, G5P_SF, G5P1B, 0xb2, G5P_SEL_RGBA), /* half float (t9-t11, fmt-rg16f, fmt-rgba16f), rendered with the two-word * program (RGBA16F: an 8-byte PBE pixel) */ - G5P_FMT(R16_SFLOAT, 2, false, G5P_SF | G5P_RT, G5P1B, 0x209, G5P_SEL_R001), - G5P_FMT(R16G16_SFLOAT, 4, false, G5P_SF | G5P_RT, G5P1B, 0x223, G5P_SEL_RG01), - G5P_FMT(R16G16B16A16_SFLOAT, 8, false, G5P_SF | G5P_RT, G5P1B, 0x232, G5P_SEL_RGBA), + G5P_FMT(R16_SFLOAT, 2, false, G5P_SF | G5P_RT | PVR_G5P_FMT_STORAGE, G5P1B, 0x209, G5P_SEL_R001), + G5P_FMT(R16G16_SFLOAT, 4, false, G5P_SF | G5P_RT | PVR_G5P_FMT_STORAGE, G5P1B, 0x223, G5P_SEL_RG01), + G5P_FMT(R16G16B16A16_SFLOAT, 8, false, G5P_SF | G5P_RT | PVR_G5P_FMT_STORAGE, G5P1B, 0x232, G5P_SEL_RGBA), /* 32-bit float (fmt-r32f, t12-t13): the A9 does not filter 32-bit float */ - G5P_FMT(R32_SFLOAT, 4, false, G5P_S, G5P1B, 0x221, G5P_SEL_R001), + G5P_FMT(R32_SFLOAT, 4, false, G5P_S | PVR_G5P_FMT_STORAGE, G5P1B, 0x221, G5P_SEL_R001), G5P_FMT(R32G32_SFLOAT, 8, false, G5P_S, G5P1B, 0x231, G5P_SEL_RG01), - G5P_FMT(R32G32B32A32_SFLOAT, 16, false, G5P_S, G5P1B, 0x238, G5P_SEL_RGBA), + G5P_FMT(R32G32B32A32_SFLOAT, 16, false, G5P_S | PVR_G5P_FMT_STORAGE, G5P1B, 0x238, G5P_SEL_RGBA), /* packed float (t14-t15). Apple packs RG11B10F in a 66-byte program; the * PBE packs it from the two-word program's f16 too, exactly. */ G5P_FMT(B10G11R11_UFLOAT_PACK32, 4, false, G5P_SF | G5P_RT, G5P1B, 0x225, G5P_SEL_RGB1), @@ -161,9 +163,26 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { * with alpha low). */ G5P_FMT(R5G6B5_UNORM_PACK16, 2, false, G5P_SF, G5P1B, 0x0b, 2, 1, 0, G5P_ONE), G5P_FMT(B4G4R4A4_UNORM_PACK16, 2, false, G5P_SF, G5P1B, 0x0c, 1, 2, 3, 0), + /* The other 16-bit packed forms (PPSSPP's A1R5G5B5 among them), sampled and filtered, from AGXMetalA9's + * texFormat rows (word 0 the code, word 16 the sampling swizzle, + * 3-bit selectors from bit 6: 2 + component, 0 zero, 1 one) and chooseTextureFormatOBJC's MTLPixelFormat map: + * B5G6R5Unorm (40) -> RGB565 0x0b [c2 c1 c0 1] A1BGR5Unorm (41) -> RGB5A1 0x0d [c3 c2 c1 c0] + * ABGR4Unorm (42) -> RGBA4 0x0c [c3 c2 c1 c0] BGR5A1Unorm (43) -> A1RGB5 0x0e [c2 c1 c0 c3] + * (no MTLPixelFormat) BGRA4 0x0c [c2 c1 c0 c3] + * Component 0 is the lowest field: the 565 and 4444 rows above, which assume it, are exact on the device. So + * 0x0d is 1_5_5_5 (alpha bit 0) and 0x0e 5_5_5_1 (alpha bit 15); the Vulkan forms below are those rows' own + * swizzles or a reordering of them. Advertised by default since the tex16 cases (packed-texture format checks) were exact + * on the device (2026-11-14); these formats are always enabled. */ + G5P_FMT(A1R5G5B5_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0e, 2, 1, 0, 3), + G5P_FMT(R5G5B5A1_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0d, 3, 2, 1, 0), + G5P_FMT(B5G5R5A1_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0d, 1, 2, 3, 0), + G5P_FMT(B5G6R5_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0b, 0, 1, 2, G5P_ONE), + G5P_FMT(R4G4B4A4_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0c, 3, 2, 1, 0), + G5P_FMT(A4R4G4B4_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0c, 2, 1, 0, 3), + G5P_FMT(A4B4G4R4_UNORM_PACK16, 2, false, G5P_SF | G5P_PS, AGX, 0x0c, 0, 1, 2, 3), /* integer (g3b-sample-b t5-t18, fmt-rgba8ui; textypes int-fetch and * int-sample on the device). RGBA32 UINT/SINT clear/draw/LOAD/masks - * passed the integer-r4 core/compat device rungs. Narrow integer render + * passed the integer-r4 core/compat device cases. Narrow integer render * targets remain absent or probe-only; integer blending is not exposed. */ G5P_FMT(R8_UINT, 1, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x100, G5P_SEL_R001), G5P_FMT(R8G8_UINT, 2, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x10a, G5P_SEL_RG01), @@ -172,9 +191,9 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { G5P_FMT(R16_UINT, 2, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x109, G5P_SEL_R001), G5P_FMT(R16G16_UINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x123, G5P_SEL_RG01), G5P_FMT(R16G16B16A16_UINT, 8, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x132, G5P_SEL_RGBA), - G5P_FMT(R32_UINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x121, G5P_SEL_R001), + G5P_FMT(R32_UINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_STORAGE, G5P1B, 0x121, G5P_SEL_R001), G5P_FMT(R32G32_UINT, 8, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x131, G5P_SEL_RG01), - G5P_FMT(R32G32B32A32_UINT, 16, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x138, G5P_SEL_RGBA), + G5P_FMT(R32G32B32A32_UINT, 16, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_STORAGE, G5P1B, 0x138, G5P_SEL_RGBA), G5P_FMT(R8_SINT, 1, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x180, G5P_SEL_R001), G5P_FMT(R8G8_SINT, 2, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x18a, G5P_SEL_RG01), G5P_FMT(R8G8B8A8_SINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x1a8, G5P_SEL_RGBA), @@ -182,9 +201,9 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { G5P_FMT(R16_SINT, 2, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x189, G5P_SEL_R001), G5P_FMT(R16G16_SINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x1a3, G5P_SEL_RG01), G5P_FMT(R16G16B16A16_SINT, 8, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_PROBE_RENDER, G5P1B, 0x1b2, G5P_SEL_RGBA), - G5P_FMT(R32_SINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x1a1, G5P_SEL_R001), + G5P_FMT(R32_SINT, 4, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_STORAGE, G5P1B, 0x1a1, G5P_SEL_R001), G5P_FMT(R32G32_SINT, 8, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x1b1, G5P_SEL_RG01), - G5P_FMT(R32G32B32A32_SINT, 16, false, G5P_SI | PVR_G5P_FMT_RENDER, G5P1B, 0x1b8, G5P_SEL_RGBA), + G5P_FMT(R32G32B32A32_SINT, 16, false, G5P_SI | PVR_G5P_FMT_RENDER | PVR_G5P_FMT_STORAGE, G5P1B, 0x1b8, G5P_SEL_RGBA), /* ETC2/EAC (g3b-compressed t0-t7; textypes etc2, etc2-8x8 and eac exact * on the device), twiddled 4x4 blocks; sRGB is q1 bit 45 as everywhere. * Uploads are upstream's block-unit copies through the CPU twiddle; blits @@ -199,6 +218,40 @@ static const struct pvr_g5p_format pvr_g5p_format_table[] = { G5P_FMT(EAC_R11_SNORM_BLOCK, 8, false, G5P_CT, G5P1B, 0xdb, G5P_SEL_R001), G5P_FMT(EAC_R11G11_UNORM_BLOCK, 16, false, G5P_CT, G5P1B, 0x5c, G5P_SEL_RG01), G5P_FMT(EAC_R11G11_SNORM_BLOCK, 16, false, G5P_CT, G5P1B, 0xdc, G5P_SEL_RG01), + /* ASTC LDR, all 14 block sizes, 16 bytes per block: AGXMetalA9's texFormat + * rows (chooseTextureFormatOBJC 0x207dc25b4): codes 0x60..0x6d in Vulkan's + * size order, sRGB as gamma 3 (q1 bit 45); -[AGXA9FamilyDevice + * isASTCPixelFormatsSupported] returns 1 on the A9. + * Twiddled over power-of-two block grids + * (pvr_g5p_image.c); copies and blits decode with util's ASTC decoder. */ + G5P_FMT(ASTC_4x4_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x60, G5P_SEL_RGBA), + G5P_FMT(ASTC_4x4_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x60, G5P_SEL_RGBA), + G5P_FMT(ASTC_5x4_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x61, G5P_SEL_RGBA), + G5P_FMT(ASTC_5x4_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x61, G5P_SEL_RGBA), + G5P_FMT(ASTC_5x5_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x62, G5P_SEL_RGBA), + G5P_FMT(ASTC_5x5_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x62, G5P_SEL_RGBA), + G5P_FMT(ASTC_6x5_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x63, G5P_SEL_RGBA), + G5P_FMT(ASTC_6x5_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x63, G5P_SEL_RGBA), + G5P_FMT(ASTC_6x6_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x64, G5P_SEL_RGBA), + G5P_FMT(ASTC_6x6_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x64, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x5_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x65, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x5_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x65, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x6_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x66, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x6_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x66, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x8_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x67, G5P_SEL_RGBA), + G5P_FMT(ASTC_8x8_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x67, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x5_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x68, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x5_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x68, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x6_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x69, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x6_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x69, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x8_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x6a, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x8_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x6a, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x10_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x6b, G5P_SEL_RGBA), + G5P_FMT(ASTC_10x10_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x6b, G5P_SEL_RGBA), + G5P_FMT(ASTC_12x10_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x6c, G5P_SEL_RGBA), + G5P_FMT(ASTC_12x10_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x6c, G5P_SEL_RGBA), + G5P_FMT(ASTC_12x12_UNORM_BLOCK, 16, false, G5P_CT, CAPTURE, 0x6d, G5P_SEL_RGBA), + G5P_FMT(ASTC_12x12_SRGB_BLOCK, 16, true, G5P_CT, CAPTURE, 0x6d, G5P_SEL_RGBA), }; #undef G5P_FMT @@ -246,7 +299,8 @@ pvr_g5p_format_flags(const struct pvr_g5p_format *f, bool probe) * compressed sources decoded per block (pvr_arch_g5p_layout.c); * - COLOR_ATTACHMENT (BLEND, BLIT_DST) for rendered rows, in either tiling * (a linear 2D target is what the PBE writes); - * - nothing else: no storage images (no compute on G5P), no texel buffers. + * - STORAGE_IMAGE for the declared-format compute pack paths, optimal only; + * no formatless image writes/reads or storage atomics, no texel buffers. * linear: the features of VK_IMAGE_TILING_LINEAR. */ static inline VkFormatFeatureFlags2 @@ -270,6 +324,8 @@ pvr_g5p_format_features(const struct pvr_g5p_format *f, bool linear, bool probe) if (flags & PVR_G5P_FMT_BLEND) r |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT; } + if ((flags & PVR_G5P_FMT_STORAGE) && !linear) + r |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT; return r; } diff --git a/src/imagination/vulkan/pvr_g5p_formats.c b/src/imagination/vulkan/pvr_g5p_formats.c index 559ca7054c1..09f1d2919dd 100644 --- a/src/imagination/vulkan/pvr_g5p_formats.c +++ b/src/imagination/vulkan/pvr_g5p_formats.c @@ -18,6 +18,7 @@ #include #include "util/log.h" +#include "util/u_debug.h" #include "vk_enum_to_str.h" #include "vk_format.h" @@ -31,22 +32,44 @@ pvr_g5p_formats_probe(void) return false; } +/* Qualified 16-bit packed texture formats are always enabled. */ +static bool +tex16_row(VkFormat format) +{ + const struct pvr_g5p_format *f = pvr_g5p_format_find(format); + return f && f->evidence == PVR_G5P_FMT_EV_AGX && f->cpp == 2; +} + +static bool +row_probe(VkFormat format) +{ + return pvr_g5p_formats_probe() || tex16_row(format); +} + bool pvr_g5p_format_tex(VkFormat format, uint32_t *key, bool *srgb) { const struct pvr_g5p_format *f = pvr_g5p_format_find(format); - if (!f || !(pvr_g5p_format_flags(f, pvr_g5p_formats_probe()) & PVR_G5P_FMT_SAMPLED)) + if (!f || !(pvr_g5p_format_flags(f, row_probe(format)) & PVR_G5P_FMT_SAMPLED)) return false; *key = f->key; *srgb = f->srgb; return true; } +/* The physical one-dword store of already packed bytes (5..8 compact + * targets): R32UI's PBE key, no format admission of its own. */ +uint32_t +pvr_g5p_packed_word_pbe_key(void) +{ + return pvr_g5p_format_find(VK_FORMAT_R32_UINT)->key; +} + bool pvr_g5p_format_pbe(VkFormat format, uint32_t *key, bool *srgb) { - return pvr_g5p_format_pbe_key(format, pvr_g5p_formats_probe(), key, srgb); + return pvr_g5p_format_pbe_key(format, row_probe(format), key, srgb); } VkFormatFeatureFlags2 @@ -56,7 +79,7 @@ pvr_g5p_color_format_features(VkFormat format, VkImageTiling tiling, VkFormatFea /* the A9's own answer; upstream's flags only say the format exists */ (void)flags; - return pvr_g5p_format_features(f, tiling == VK_IMAGE_TILING_LINEAR, pvr_g5p_formats_probe()); + return pvr_g5p_format_features(f, tiling == VK_IMAGE_TILING_LINEAR, row_probe(format)); } VkResult @@ -74,6 +97,13 @@ pvr_g5p_image_format_check(const VkPhysicalDeviceImageFormatInfo2 *info, VkImage why = "3D block-compressed images (the A9's 3D order is built for texels only)"; else if (info->type == VK_IMAGE_TYPE_3D && tiling != VK_IMAGE_TILING_OPTIMAL) why = "linear 3D images (Apple samples 3D twiddled only)"; + if (!why && (info->usage & VK_IMAGE_USAGE_STORAGE_BIT)) { + const struct pvr_g5p_format *f = pvr_g5p_format_find(info->format); + if (!(f->flags & PVR_G5P_FMT_STORAGE)) + why = "no A9 storage pack path for this format"; + else if (info->type != VK_IMAGE_TYPE_2D || tiling != VK_IMAGE_TILING_OPTIMAL) + why = "A9 storage images require optimal 2D images (including arrays/cube faces)"; + } if (!why) return VK_SUCCESS; mesa_logd("G5P: no %s image of %s, tiling %s, usage %#x, flags %#x: %s", vk_ImageType_to_str(info->type), diff --git a/src/imagination/vulkan/pvr_g5p_formats.h b/src/imagination/vulkan/pvr_g5p_formats.h index 5fcaa3a389f..9beb01175f9 100644 --- a/src/imagination/vulkan/pvr_g5p_formats.h +++ b/src/imagination/vulkan/pvr_g5p_formats.h @@ -32,6 +32,8 @@ bool pvr_g5p_format_tex(VkFormat format, uint32_t *key, bool *srgb); /* The PBE key and sRGB bit of a colour target format, or false: the A9 * doesn't render it (the pipeline and the render job refuse it by name) */ bool pvr_g5p_format_pbe(VkFormat format, uint32_t *key, bool *srgb); +/* Physical one-dword stores for already packed bytes; no public format admission. */ +uint32_t pvr_g5p_packed_word_pbe_key(void); /* pvr_get_image_format_features2(): the features of a colour format on * the A9, in place of upstream's (flags) */ diff --git a/src/imagination/vulkan/pvr_g5p_hooks.h b/src/imagination/vulkan/pvr_g5p_hooks.h index 78725bbb67e..97682e7cd79 100644 --- a/src/imagination/vulkan/pvr_g5p_hooks.h +++ b/src/imagination/vulkan/pvr_g5p_hooks.h @@ -47,6 +47,13 @@ pvr_arch_g5p_pack_image_view( \ iview, &(iview)->image_state[PVR_TEXTURE_STATE_SAMPLE]); +/* Only storage descriptors reinterpret cube faces as a 2D array. */ +#define PVR_G5P_HOOK_storage_image(dev_info, iview, desc) \ + if (PVR_HAS_FEATURE(dev_info, apple_g5p) && \ + ((iview)->vk.view_type == VK_IMAGE_VIEW_TYPE_CUBE || \ + (iview)->vk.view_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY)) \ + pvr_arch_g5p_pack_storage_image_view(iview, desc); + /* Release integration of the vertex-store capability, preserving shared lines. */ /* Also the polygon-mode and indirect lanes' feature lines (their * pvr_g5p_publish_hooks.h, which this base no longer has): line and point @@ -142,7 +149,9 @@ (surface)->g5p_aspect = (aspect); \ } \ if (PVR_G5P_IMAGE_IS_SLICED_3D(image)) { \ - (surface)->dev_addr.addr += (info)->depthPitch * (uint32_t)(fdepth); \ + /* the slice's offset, info->depthPitch * z, is added once after this \ + * hook by pvr_setup_transfer_surface() itself, as for any non-3D \ + * surface: adding it here too put slice z at 2z (observed in 3D transfers) */ \ (surface)->mem_layout = PVR_MEMLAYOUT_TWIDDLED; \ (surface)->depth = 1; \ } @@ -292,7 +301,7 @@ if (PVR_HAS_FEATURE(&(cmd_buffer)->device->pdevice->dev_info, apple_g5p)) { \ VkResult r_ = pvr_arch_g5p_clear_attachments(cmd_buffer, count, attachments, rect_count, rects); \ if (r_ != VK_SUCCESS) \ - pvr_cmd_buffer_set_error_unwarned(cmd_buffer, r_); \ + pvr_cmd_buffer_set_error_unwarned(cmd_buffer, pvr_refusal_to_api(r_)); \ return; \ } diff --git a/src/imagination/vulkan/pvr_g5p_image.c b/src/imagination/vulkan/pvr_g5p_image.c index 0fd3452a1c6..78a04852c15 100644 --- a/src/imagination/vulkan/pvr_g5p_image.c +++ b/src/imagination/vulkan/pvr_g5p_image.c @@ -62,6 +62,7 @@ pvr_g5p_image_layout(struct pvr_image *image, unsigned stride_align) return; const bool twiddled = is3d || image->vk.mip_levels > 1 || + (image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT) || (image->vk.create_flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) || (image->vk.array_layers > 1 && (image->vk.usage & VK_IMAGE_USAGE_SAMPLED_BIT)) || vk_format_is_compressed(image->vk.format); @@ -89,10 +90,31 @@ pvr_g5p_image_layout(struct pvr_image *image, unsigned stride_align) const uint32_t chain = image->vk.mip_levels > 1 ? util_logbase2(MAX2(pw, ph)) + 1 : 1; VkDeviceSize size = 0, own = 0; + /* Blocks other than 4x4 (ASTC 5x4..12x12): each level's block grid is a + * power of two per side, pow2(ceil(level / block)), as AGXMetalA9's + * TextureGen2 lays them out (0x207dbfe4c: ceil(width / bw), then the + * next power of two, per plane level) and as the Morton order needs (a + * 7-block side is addressed as 8: g5p_twiddle2d()). Upstream's level + * offsets minify the block count instead, so these images take their + * offsets from here. 4x4 grids are already powers of two (or one block): + * ETC2/EAC and ASTC 4x4 keep upstream's offsets, which are the same. */ + const bool pow2_blocks = !is3d && vk_format_is_compressed(image->vk.format) && (bw != 4 || bh != 4); for (uint32_t l = 0; l < MAX2(chain, image->vk.mip_levels); l++) { - const uint32_t ew = DIV_ROUND_UP(u_minify(pw, l), bw), eh = DIV_ROUND_UP(u_minify(ph, l), bh); + uint32_t ew = DIV_ROUND_UP(u_minify(pw, l), bw), eh = DIV_ROUND_UP(u_minify(ph, l), bh); + if (pow2_blocks) { + ew = util_next_power_of_two(ew); + eh = util_next_power_of_two(eh); + } const VkDeviceSize level_size = (VkDeviceSize)cpp * ew * eh; + if (pow2_blocks && l < image->vk.mip_levels) { + struct pvr_mip_level *level = &plane->mip_levels[l]; + + level->pitch = cpp * ew; + level->height_pitch = eh; + level->size = level_size; + level->offset = own; + } if (is3d && l < image->vk.mip_levels) { /* a 3D image's levels are one slice's: upstream's 3DTWIDDLED levels hold every slice */ struct pvr_mip_level *level = &plane->mip_levels[l]; @@ -193,7 +215,9 @@ pvr_g5p_msaa_limits(struct pvr_physical_device *pdevice, struct vk_properties *p p->framebufferStencilSampleCounts = fb; p->framebufferNoAttachmentsSampleCounts = fb; p->framebufferIntegerColorSampleCounts = VK_SAMPLE_COUNT_1_BIT; - p->sampledImageColorSampleCounts = VK_SAMPLE_COUNT_1_BIT; + /* 4-sample colour images are sampled (texelFetch of a 2D multisample texture, + * pvr_arch_g5p_tex.c's 2Wx2H view state and the compiler's txf_ms lowering) */ + p->sampledImageColorSampleCounts = fb; p->sampledImageIntegerSampleCounts = VK_SAMPLE_COUNT_1_BIT; p->sampledImageDepthSampleCounts = VK_SAMPLE_COUNT_1_BIT; p->sampledImageStencilSampleCounts = VK_SAMPLE_COUNT_1_BIT; @@ -203,22 +227,27 @@ pvr_g5p_msaa_limits(struct pvr_physical_device *pdevice, struct vk_properties *p /* * An image format's sample counts: 4 as well only for what a 4x pass * renders (our ICD's rule): a 2D optimal colour attachment of 4-byte - * samples, or one of the depth/stencil formats the ZLS keeps on chip, used - * neither as a sampled, storage nor input attachment image. Upstream's Rogue answer is every count up to 8. + * samples, or one of the depth/stencil formats the ZLS keeps on chip, not + * used as a storage image, nor as a sampled depth/stencil image. Upstream's Rogue answer is every count up to 8. */ void pvr_g5p_msaa_image_samples(struct pvr_physical_device *pdevice, const VkPhysicalDeviceImageFormatInfo2 *info, VkImageTiling tiling, VkImageFormatProperties *props) { const VkImageUsageFlags attach = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT; - const VkImageUsageFlags read = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT | - VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT; + /* Vulkan bounds sampleCounts by the framebuffer, sampled and storage limits of the usage; input + * attachments have no limit of their own and do not change the 4x layout. Multisample subpass + * reads stay refused by the compiler (tex_shape_refusal()). */ + const VkImageUsageFlags read = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT; const bool format_ok = pvr_g5p_ds_format(info->format) || (vk_format_is_color(info->format) && !vk_format_is_compressed(info->format) && !vk_format_is_int(info->format) && vk_format_get_blocksize(info->format) == 4); + /* a 4-sample colour attachment may also be sampled (its view is the 2Wx2H twiddled texture of + * its samples, pvr_arch_g5p_tex.c); sampled depth/stencil and storage images stay 1x */ + const VkImageUsageFlags colour_read = vk_format_is_color(info->format) ? VK_IMAGE_USAGE_SAMPLED_BIT : 0; const bool four = tiling == VK_IMAGE_TILING_OPTIMAL && info->type == VK_IMAGE_TYPE_2D && !(info->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) && - (info->usage & attach) && !(info->usage & read) && format_ok; + (info->usage & attach) && !(info->usage & read & ~colour_read) && format_ok; props->sampleCounts &= four ? VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT : VK_SAMPLE_COUNT_1_BIT; } diff --git a/src/imagination/vulkan/pvr_g5p_mrt.h b/src/imagination/vulkan/pvr_g5p_mrt.h index 1e1b8120680..ce241bc6d59 100644 --- a/src/imagination/vulkan/pvr_g5p_mrt.h +++ b/src/imagination/vulkan/pvr_g5p_mrt.h @@ -26,8 +26,9 @@ struct pvr_cmd_buffer; struct pvr_device; struct pvr_renderpass_hwsetup_render; -/* the most colour targets a G5P pipeline or pass has (Apple's 4-target pass) */ -#define PVR_G5P_MRT_MAX_COLORS 4 +/* the most colour targets a G5P pipeline or pass has: Apple's 4-target pass, then 5..8 compact UNORM8 + * targets (one format, full write mask, no blending) */ +#define PVR_G5P_MRT_MAX_COLORS 8 /* The formats of Apple's 2- and 4-target pass */ static inline bool diff --git a/src/imagination/vulkan/pvr_g5p_msaa.c b/src/imagination/vulkan/pvr_g5p_msaa.c index c685a67b06d..a12cb40091e 100644 --- a/src/imagination/vulkan/pvr_g5p_msaa.c +++ b/src/imagination/vulkan/pvr_g5p_msaa.c @@ -117,6 +117,18 @@ pvr_g5p_msaa_pbe_store(uint64_t q[3], uint32_t key, uint64_t va, uint32_t width, q[2] = 0; } +/* Texture state q0 bit 55: the 4-sample image (Apple's 4x LOAD state, the PBE store state has it clear) */ +#define G5P_TEX_Q0_MS4 (1ull << 55) + +void +pvr_g5p_msaa_tex_state(uint64_t q[3], uint32_t key, uint64_t va, uint32_t width, uint32_t height, bool srgb) +{ + q[0] = (uint64_t)(height - 1) << 41 | (uint64_t)(width - 1) << 27 | + ((key & ~G5P_PBE_LAYOUT_MASK) | G5P_PBE_LAYOUT_MS_TWIDDLED) | G5P_TEX_Q0_MS4; + q[1] = va >> 1 | (srgb ? G5P_PBE_Q1_SRGB : 0); + q[2] = 0; +} + void pvr_g5p_msaa_pbe_resolve(uint64_t q[3]) { diff --git a/src/imagination/vulkan/pvr_g5p_msaa.h b/src/imagination/vulkan/pvr_g5p_msaa.h index 67e153aba61..ef80653dae9 100644 --- a/src/imagination/vulkan/pvr_g5p_msaa.h +++ b/src/imagination/vulkan/pvr_g5p_msaa.h @@ -59,6 +59,11 @@ uint64_t pvr_g5p_msaa_sample_offset(uint32_t width, uint32_t height, uint32_t cp void pvr_g5p_msaa_pbe_store(uint64_t q[3], uint32_t key, uint64_t va, uint32_t width, uint32_t height, bool srgb); void pvr_g5p_msaa_pbe_resolve(uint64_t q[3]); +/* The texture state the 4x LOAD background samples (the 2D-multisample fetch, g5p_usc.h + * G5P_TEX_2D_MS): the store's surface, W x H pixels, layout 4 twiddled and q0 bit 55, no PBE bit + * (Apple's 4x colour load of a 64x64 BGRA8 target differs from its PBE store words in exactly those + * fields). */ +void pvr_g5p_msaa_tex_state(uint64_t q[3], uint32_t key, uint64_t va, uint32_t width, uint32_t height, bool srgb); /* The end-of-tile program of a one-target 4x store: the one-target program * (g5p_mrt_eot_usc(out, 1)) with the emitpix's sample field 2 instead of 1 @@ -100,8 +105,8 @@ struct pvr_cmd_buffer; struct pvr_renderpass_hwsetup_render; /* job_init: refuses, by name, a multisample render this lane does not - * build (only 1 and 4 samples; at 4: one colour target, no LOAD, no - * multisample depth/stencil load or store). VK_SUCCESS at one sample. */ + * build (only 1 and 4 samples; at 4: one colour target, stored or resolved; + * its LOAD is Apple's 4x loader, pvr_arch_g5p.c). VK_SUCCESS at one sample. */ VkResult PVR_PER_ARCH(g5p_msaa_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_renderpass_hwsetup_render *hw_render); diff --git a/src/imagination/vulkan/pvr_g5p_nir.c b/src/imagination/vulkan/pvr_g5p_nir.c index 3547f5dcee4..4bc078ac35f 100644 --- a/src/imagination/vulkan/pvr_g5p_nir.c +++ b/src/imagination/vulkan/pvr_g5p_nir.c @@ -21,6 +21,7 @@ #include "compiler/spirv/spirv_info.h" #include "util/u_call_once.h" #include "util/u_math.h" +#include "vk_nir.h" #include "vk_shader_module.h" #include "vk_util.h" @@ -62,6 +63,20 @@ g5p_init_options(void) * shifts, masks and selects. Keep PCO's BFE/BFI capabilities disabled. */ g5p_options.lower_bitfield_extract = true; g5p_options.lower_bitfield_insert = true; + /* GLSL's norm packings (packUnorm2x16() ...) lower to the measured float clamps, rounds, conversions and + * shifts (G9P's 26b4dcbc83d ("pvr/g9p: packing built-ins"); the half packings: pco_g5p_general.c lower_half_packing()) */ + g5p_options.lower_pack_snorm_2x16 = true; + g5p_options.lower_unpack_snorm_2x16 = true; + g5p_options.lower_pack_unorm_2x16 = true; + g5p_options.lower_unpack_unorm_2x16 = true; + g5p_options.lower_pack_snorm_4x8 = true; + g5p_options.lower_unpack_snorm_4x8 = true; + g5p_options.lower_pack_unorm_4x8 = true; + g5p_options.lower_unpack_unorm_4x8 = true; + /* saturating integer add/subtract (naga's bounds clamps: usub_sat(arrayLength, 1)): compare and select */ + g5p_options.lower_usub_sat = true; + g5p_options.lower_uadd_sat = true; + g5p_options.lower_iadd_sat = true; g5p_vertex_options = g5p_options; g5p_vertex_options.max_unroll_iterations = 16; /* The isolated Apple iv_reg_pop/reverse witnesses qualify native VS @@ -289,6 +304,67 @@ clamp_dyn_ubo(nir_shader *nir) static nir_shader *g5p_nir_prepare(nir_shader *nir); +/* Storage-buffer accesses the backend encodes: a 32-bit scalar, or a vec4 + * load at a 16-byte-aligned offset (one 16-byte addressed load). Wider or + * misaligned vectors become scalars at offset + 4k (WGSL's vec2/vec3 and + * struct members), and stores become one scalar store per written + * component. A constant offset keeps a vec4 that the fixed load already + * encodes, so earlier programs are unchanged. */ +static bool +split_ssbo_access(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + if (in->intrinsic == nir_intrinsic_load_ssbo) { + const unsigned n = in->def.num_components; + if (in->def.bit_size != 32 || n == 1) + return false; + if (n == 4 && (nir_intrinsic_align(in) >= 16 || + (nir_src_is_const(in->src[1]) && !(nir_src_as_uint(in->src[1]) % 16)))) + return false; + b->cursor = nir_before_instr(&in->instr); + nir_def *comp[NIR_MAX_VEC_COMPONENTS]; + for (unsigned k = 0; k < n; k++) + comp[k] = nir_load_ssbo(b, 1, 32, in->src[0].ssa, nir_iadd_imm(b, in->src[1].ssa, 4 * k), + .access = nir_intrinsic_access(in), .align_mul = 4, .align_offset = 0); + nir_def_replace(&in->def, nir_vec(b, comp, n)); + return true; + } + if (in->intrinsic == nir_intrinsic_store_ssbo) { + const unsigned n = in->src[0].ssa->num_components; + const unsigned mask = nir_intrinsic_write_mask(in); + if (in->src[0].ssa->bit_size != 32 || (n == 1 && mask == 1)) + return false; + b->cursor = nir_before_instr(&in->instr); + u_foreach_bit(k, mask) + nir_store_ssbo(b, nir_channel(b, in->src[0].ssa, k), in->src[1].ssa, + nir_iadd_imm(b, in->src[2].ssa, 4 * k), .write_mask = 1, + .access = nir_intrinsic_access(in), .align_mul = 4, .align_offset = 0); + nir_instr_remove(&in->instr); + return true; + } + return false; +} + +/* After constant folding: a register-indexed vec2/vec3 UBO load becomes + * scalars at offset + 4k (the backend encodes scalar and vec4 forms; + * vec4_dyn_ubo() then merges complete quads). Constant-offset loads are + * preloaded and stay as they are. */ +static bool +split_dyn_ubo(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + const unsigned n = in->def.num_components; + if (in->intrinsic != nir_intrinsic_load_ubo || in->def.bit_size != 32 || (n != 2 && n != 3) || + nir_src_is_const(in->src[1])) + return false; + b->cursor = nir_before_instr(&in->instr); + nir_def *comp[NIR_MAX_VEC_COMPONENTS]; + for (unsigned k = 0; k < n; k++) + comp[k] = nir_load_ubo(b, 1, 32, in->src[0].ssa, nir_iadd_imm(b, in->src[1].ssa, 4 * k), + .access = nir_intrinsic_access(in), .align_mul = 4, .align_offset = 0, + .range_base = 0, .range = ~0u); + nir_def_replace(&in->def, nir_vec(b, comp, n)); + return true; +} + /* SPIR-V -> optimised NIR (the ICD's g5p_nir()) */ static nir_shader * g5p_nir(const uint32_t *spv, size_t words, mesa_shader_stage stage, const char *entry) @@ -330,6 +406,14 @@ g5p_nir_prepare(nir_shader *nir) * as its location (glslang's gl_PointSize became slot 1, COL0) */ nir_split_var_copies(nir); nir_split_per_member_structs(nir); + /* gl_ClipDistance/gl_CullDistance sizes, before IO lowering, after the + * dead outputs are gone (vk_nir.c: glslang declares an unwritten + * gl_ClipDistance[1] and gl_CullDistance[1] in every gl_PerVertex) */ + if (stage == MESA_SHADER_VERTEX) { + const nir_remove_dead_variables_options dead = { .can_remove_var = nir_vk_is_not_xfb_output }; + nir_remove_dead_variables(nir, nir_var_shader_out, &dead); + nir_gather_clip_cull_distance_sizes_from_vars(nir); + } nir_lower_vars_to_ssa(nir); nir_opt_deref(nir); nir_lower_system_values(nir); @@ -339,6 +423,7 @@ g5p_nir_prepare(nir_shader *nir) nir_shader_gather_xfb_info(nir); nir_lower_io_passes(nir, false); nir_lower_explicit_io(nir, nir_var_mem_ssbo | nir_var_mem_ubo, nir_address_format_32bit_index_offset); + nir_shader_intrinsics_pass(nir, split_ssbo_access, nir_metadata_control_flow, NULL); bool progress; do { progress = false; @@ -351,6 +436,11 @@ g5p_nir_prepare(nir_shader *nir) NIR_PASS(progress, nir, nir_opt_dead_cf); } while (progress); progress = false; + if (nir_shader_intrinsics_pass(nir, split_dyn_ubo, nir_metadata_control_flow, NULL)) { + NIR_PASS(_, nir, nir_opt_constant_folding); + NIR_PASS(_, nir, nir_opt_algebraic); + NIR_PASS(_, nir, nir_opt_cse); + } NIR_PASS(progress, nir, vec4_dyn_ubo); if (stage == MESA_SHADER_FRAGMENT && pco_g5p_fs_dyn_ubo()) NIR_PASS(progress, nir, clamp_dyn_ubo); @@ -471,6 +561,50 @@ pad_color_slots(nir_shader *nir, unsigned color_targets) nir_index_ssa_defs(impl); } +/* The clip distance components a vertex program stores (a bit per + * distance; compact CLIP_DIST0/1, constant offsets) */ +static bool +clip_written(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + if (in->intrinsic != nir_intrinsic_store_output) + return false; + const unsigned loc = nir_intrinsic_io_semantics(in).location; + if (loc != VARYING_SLOT_CLIP_DIST0 && loc != VARYING_SLOT_CLIP_DIST1) + return false; + nir_src *off = nir_get_io_offset_src(in); + const unsigned first = 4 * (loc - VARYING_SLOT_CLIP_DIST0 + (nir_src_is_const(*off) ? nir_src_as_uint(*off) : 0)) + + nir_intrinsic_component(in); + *(unsigned *)data |= nir_intrinsic_write_mask(in) << first; + return false; +} + +/* A declared clip distance the program never stores is 0 (on the plane, + * not clipped): the TA reads every declared one (plane0..n-1), and an + * unwritten output dword would clip by a stale register. GL's disabled + * planes are already stored as 0 (Zink's nir_lower_clip_disable). */ +static void +fill_clip_distances(nir_shader *nir) +{ + const unsigned n = nir->info.clip_distance_array_size; + unsigned written = 0; + if (!n) + return; + nir_shader_intrinsics_pass(nir, clip_written, nir_metadata_all, &written); + const unsigned missing = BITFIELD_MASK(n) & ~written; + if (!missing) + return; + nir_function_impl *impl = nir_shader_get_entrypoint(nir); + nir_builder b = nir_builder_at(nir_after_impl(impl)); + u_foreach_bit(i, missing) { + const gl_varying_slot loc = i < 4 ? VARYING_SLOT_CLIP_DIST0 : VARYING_SLOT_CLIP_DIST1; + nir_store_output(&b, nir_imm_float(&b, 0.0f), nir_imm_int(&b, 0), .base = 0, .component = i % 4, + .write_mask = 1, .src_type = nir_type_float32, + .io_semantics = (nir_io_semantics){ .location = loc, .num_slots = 1 }); + nir->info.outputs_written |= BITFIELD64_BIT(loc); + } + nir_index_ssa_defs(impl); +} + /* G5P renders layer 0 of single-layer views only (no layered rendering is * exposed): a vertex shader's gl_Layer is dropped and a fragment shader's * gl_Layer reads 0. vk_meta's rectangle and blit shaders use both. */ @@ -555,6 +689,8 @@ graphics_lower(nir_shader *nir, unsigned color_targets, bool *sample_mask, bool nir_opt_dce(nir); nir_index_ssa_defs(nir_shader_get_entrypoint(nir)); } + if (nir && stage == MESA_SHADER_VERTEX) + fill_clip_distances(nir); bool writes_psize = false; if (nir && stage == MESA_SHADER_VERTEX && point_size) nir_shader_intrinsics_pass(nir, find_point_size, nir_metadata_all, &writes_psize); @@ -669,6 +805,10 @@ xfb_lower(nir_shader *nir, struct g5p_xfb_layout *layout, char *refusal, size_t snprintf(refusal, refusal_size, "transform feedback of 16-bit data"); return false; } + if (o->location == VARYING_SLOT_CLIP_DIST0 || o->location == VARYING_SLOT_CLIP_DIST1) { + snprintf(refusal, refusal_size, "transform feedback of gl_ClipDistance (capture not qualified)"); + return false; + } const unsigned r = layout->nranges++; layout->ranges[r].buffer = o->buffer; layout->ranges[r].offset = o->offset; @@ -898,6 +1038,88 @@ pvr_g5p_compile_load_raw(struct pco_g5p_program *out, bool is_signed) return g5p_compile_load_type(out, is_signed ? 2 : 1); } +/* + * The 4x LOAD background (a loaded 4-sample colour attachment), Apple's loader (an iOS capture + * of a 4x colour load: the 1x loader plus the own-sample operand, dimension f9, kicked at + * sample rate): a sample of the target's 2D-multisample state at the pixel centre in normalized + * coordinates, color = texture(prev, gl_FragCoord.xy / textureSize(prev, 0)), compiled with the key's + * ms_own_sample, so the sample reads the sample the task shades (pco_g5p_general.c + * emit_tex_ms_own_sample()). Its uploads: the image state, the sampler, the size block. + */ +bool +pvr_g5p_compile_load_ms(struct pco_g5p_program *out) +{ + static util_once_flag once = UTIL_ONCE_FLAG_INIT; + util_call_once(&once, g5p_init_options); + memset(out, 0, sizeof(*out)); + glsl_type_singleton_init_or_ref(); + + nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, &g5p_options, "g5p-load-ms4"); + nir_variable *prev = nir_variable_create(b.shader, nir_var_uniform, + glsl_sampler_type(GLSL_SAMPLER_DIM_2D, false, false, GLSL_TYPE_FLOAT), + "prev"); + prev->data.descriptor_set = 0; + prev->data.binding = 0; + nir_variable *color = nir_variable_create(b.shader, nir_var_shader_out, glsl_vec4_type(), "color"); + color->data.location = FRAG_RESULT_DATA0; + nir_deref_instr *deref = nir_build_deref_var(&b, prev); + nir_def *size = nir_i2f32(&b, nir_trim_vector(&b, nir_txs(&b, .texture_deref = deref, .lod = nir_imm_int(&b, 0), + .dim = GLSL_SAMPLER_DIM_2D, + .dest_type = nir_type_int32), 2)); + nir_def *uv = nir_fdiv(&b, nir_trim_vector(&b, nir_load_frag_coord(&b), 2), size); + nir_def *texel = nir_tex(&b, uv, .texture_deref = deref, .sampler_deref = deref, .dim = GLSL_SAMPLER_DIM_2D, + .dest_type = nir_type_float32); + nir_store_var(&b, color, texel, 0xf); + + bool sample_mask = false; + nir_shader *nir = graphics_lower(g5p_nir_prepare(b.shader), 1, &sample_mask, false, NULL); + struct pco_g5p_fs_key key = PCO_G5P_FS_KEY_DEFAULT; + key.last_read_groups = true; + key.ms_own_sample = true; + bool ok = nir && pco_g5p_compile_general_link(nir, true, NULL, &key, NULL, out); + if (!nir) + snprintf(out->refusal, sizeof(out->refusal), "the 4x load program's NIR"); + ralloc_free(nir); + glsl_type_singleton_decref(); + return ok; +} + +bool +pvr_g5p_compile_load_mrt(struct pco_g5p_program *out, unsigned targets, bool bgra) +{ + static util_once_flag once = UTIL_ONCE_FLAG_INIT; + util_call_once(&once, g5p_init_options); + memset(out, 0, sizeof(*out)); + glsl_type_singleton_init_or_ref(); + + nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, &g5p_options, "g5p-load-mrt"); + nir_def *xy = nir_f2i32(&b, nir_trim_vector(&b, nir_load_frag_coord(&b), 2)); + for (unsigned t = 0; t < targets; t++) { + nir_variable *prev = nir_variable_create(b.shader, nir_var_uniform, + glsl_sampler_type(GLSL_SAMPLER_DIM_2D, false, false, GLSL_TYPE_FLOAT), + "prev"); + prev->data.descriptor_set = 0; + prev->data.binding = t; + nir_variable *color = nir_variable_create(b.shader, nir_var_shader_out, glsl_vec4_type(), "color"); + color->data.location = FRAG_RESULT_DATA0 + t; + nir_def *texel = nir_txf(&b, xy, .texture_deref = nir_build_deref_var(&b, prev), .lod = nir_imm_int(&b, 0), + .dim = GLSL_SAMPLER_DIM_2D, .dest_type = nir_type_float32); + nir_store_var(&b, color, texel, 0xf); + } + bool sample_mask = false; + nir_shader *nir = graphics_lower(g5p_nir_prepare(b.shader), targets, &sample_mask, false, NULL); + struct pco_g5p_fs_key key = PCO_G5P_FS_KEY_DEFAULT; + key.last_read_groups = true; + key.color_targets = targets; + key.color_output = bgra ? PCO_G5P_COLOR_PACKED_BGRA8 : PCO_G5P_COLOR_PACKED_RGBA8; + bool ok = nir && pco_g5p_compile_general_link(nir, true, NULL, &key, NULL, out); + if (!nir) + snprintf(out->refusal, sizeof(out->refusal), "the compact load program's NIR"); + ralloc_free(nir); + glsl_type_singleton_decref(); + return ok; +} + nir_shader * pvr_g5p_clear_nir(bool vertex) { diff --git a/src/imagination/vulkan/pvr_g5p_shader_dump.c b/src/imagination/vulkan/pvr_g5p_shader_dump.c new file mode 100644 index 00000000000..1565566f1f1 --- /dev/null +++ b/src/imagination/vulkan/pvr_g5p_shader_dump.c @@ -0,0 +1,321 @@ +/* SPDX-License-Identifier: MIT */ +/* Shader capture controlled by PVR_DEBUG (pvr_g5p_shader_dump.h). */ +#include "pvr_g5p_shader_dump.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util/macros.h" +#include "util/log.h" +#include "pvr_debug.h" +#include "util/u_debug.h" +#include "c11/threads.h" + +DEBUG_GET_ONCE_OPTION(pvr_shader_dump_dir, "PVR_SHADER_DUMP_DIR", "/tmp/pvr-shaders") +#include "util/mesa-blake3.h" +#include "util/u_atomic.h" +#include "util/u_call_once.h" +#include "vk_pipeline_layout.h" +#include "vk_shader_module.h" +#include "vk_util.h" +#include "pvr_descriptor_set.h" +#include "pvr_g5p_stub_spv.h" + +static const char *dump_dir; +static bool stub_on; +static bool compute_log; +static util_once_flag env_once = UTIL_ONCE_FLAG_INIT; + +static void +read_env(void) +{ + const char *d = PVR_IS_DEBUG_SET(SHADER_DUMP) ? debug_get_option_pvr_shader_dump_dir() : NULL; + if (d && *d) { + mkdir(d, 0755); /* may exist */ + dump_dir = d; + /* once per process: a sandboxed process may not write there (Firefox: MOZ_DISABLE_GPU_SANDBOX=1) */ + if (access(d, W_OK)) + mesa_logw("G5P shader dump: cannot write %s (pid %ld): %s", d, (long)getpid(), strerror(errno)); + else + mesa_logi("G5P shader dump: %s (pid %ld)", d, (long)getpid()); + } + compute_log = PVR_IS_DEBUG_SET(COMPUTE_LOG); + stub_on = PVR_IS_DEBUG_SET(CAPTURE_STUB); + if (stub_on) + mesa_logw("G5P capture stub: ON (refused shaders are replaced; pictures will be wrong)"); +} + +const char * +pvr_g5p_shader_dump_dir(void) +{ + util_call_once(&env_once, read_env); + return dump_dir; +} + +bool +pvr_g5p_capture_stub(void) +{ + util_call_once(&env_once, read_env); + return stub_on; +} + +bool +pvr_g5p_compute_log(void) +{ + util_call_once(&env_once, read_env); + return compute_log; +} + +static bool +stage_spirv(const VkPipelineShaderStageCreateInfo *info, const uint32_t **words, size_t *bytes) +{ + VK_FROM_HANDLE(vk_shader_module, module, info->module); + const VkShaderModuleCreateInfo *inline_ci = vk_find_struct_const(info->pNext, SHADER_MODULE_CREATE_INFO); + if (module && !module->nir) { + *words = (const uint32_t *)module->data; + *bytes = module->size; + return true; + } + if (inline_ci) { + *words = inline_ci->pCode; + *bytes = inline_ci->codeSize; + return true; + } + return false; /* a driver-internal NIR module */ +} + +void +pvr_g5p_shader_id(const VkPipelineShaderStageCreateInfo *info, char id[13]) +{ + const uint32_t *words; + size_t bytes; + memcpy(id, "internal-nir", 13); + if (stage_spirv(info, &words, &bytes)) { + blake3_hash hash; + char hex[BLAKE3_OUT_LEN * 2 + 1]; + _mesa_blake3_compute(words, bytes, hash); + _mesa_blake3_format(hex, hash); + memcpy(id, hex, 12); + id[12] = 0; + } +} + +/* write @size bytes to @path through a temporary name and a rename */ +static void +write_atomic(const char *path, const void *data, size_t size, bool keep_existing) +{ + static uint32_t serial; + if (keep_existing && access(path, F_OK) == 0) + return; + char tmp[4096]; + snprintf(tmp, sizeof(tmp), "%s.tmp.%ld.%u", path, (long)getpid(), p_atomic_inc_return(&serial)); + int fd = open(tmp, O_WRONLY | O_CREAT | O_EXCL, 0644); + if (fd < 0) + return; + const char *p = data; + size_t left = size; + while (left) { + ssize_t n = write(fd, p, left); + if (n <= 0) { + if (n < 0 && errno == EINTR) + continue; + break; + } + p += n, left -= n; + } + close(fd); + if (left || rename(tmp, path)) + unlink(tmp); +} + +/* a growing string */ +struct sbuf { + char *s; + size_t len, cap; +}; + +static void +sb_printf(struct sbuf *b, const char *fmt, ...) PRINTFLIKE(2, 3); + +static void +sb_printf(struct sbuf *b, const char *fmt, ...) +{ + for (;;) { + va_list ap; + va_start(ap, fmt); + int n = vsnprintf(b->s ? b->s + b->len : NULL, b->s ? b->cap - b->len : 0, fmt, ap); + va_end(ap); + if (n < 0) + return; + if (b->s && b->len + n < b->cap) { + b->len += n; + return; + } + size_t cap = MAX2(b->cap * 2, b->len + n + 256); + char *s = realloc(b->s, cap); + if (!s) + return; + b->s = s, b->cap = cap; + } +} + +static void +sb_json_string(struct sbuf *b, const char *s) +{ + sb_printf(b, "\""); + for (; s && *s; s++) { + const unsigned char c = *s; + if (c == '"' || c == '\\') + sb_printf(b, "\\%c", c); + else if (c < 0x20) + sb_printf(b, "\\u%04x", c); + else + sb_printf(b, "%c", c); + } + sb_printf(b, "\""); +} + +static void +sb_hex(struct sbuf *b, const void *data, size_t size) +{ + sb_printf(b, "\""); + for (size_t i = 0; i < size; i++) + sb_printf(b, "%02x", ((const uint8_t *)data)[i]); + sb_printf(b, "\""); +} + +static const char * +descriptor_type_name(VkDescriptorType t) +{ + switch (t) { + case VK_DESCRIPTOR_TYPE_SAMPLER: return "sampler"; + case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: return "combined_image_sampler"; + case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: return "sampled_image"; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: return "storage_image"; + case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: return "uniform_texel_buffer"; + case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: return "storage_texel_buffer"; + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: return "uniform_buffer"; + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: return "storage_buffer"; + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: return "uniform_buffer_dynamic"; + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: return "storage_buffer_dynamic"; + case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT: return "input_attachment"; + default: return "other"; + } +} + +void +pvr_g5p_shader_dump(const VkPipelineShaderStageCreateInfo *info, VkPipelineLayout layout_handle, + const char *stage, const struct pvr_g5p_dump_blob *blobs, unsigned nblobs, + const char *result, const char *reason) +{ + const char *dir = pvr_g5p_shader_dump_dir(); + const uint32_t *words; + size_t bytes; + if (!dir || !info || !stage_spirv(info, &words, &bytes)) + return; + + /* named by the first 20 bytes (40 hex digits) of the module's BLAKE3 */ + blake3_hash sha; + char hex[BLAKE3_HEX_LEN]; + _mesa_blake3_compute(words, bytes, sha); + _mesa_blake3_format(hex, sha); + hex[40] = 0; + char path[4096]; + snprintf(path, sizeof(path), "%s/%s.spv", dir, hex); + write_atomic(path, words, bytes, true); + + /* everything that decides the compile, then the result */ + struct sbuf b = { 0 }; + sb_printf(&b, "{\n \"spirv\": \"%s.spv\",\n \"stage\": \"%s\",\n \"entry\": ", hex, stage); + sb_json_string(&b, info->pName); + const VkSpecializationInfo *spec = info->pSpecializationInfo; + sb_printf(&b, ",\n \"specialization\": {\"entries\": ["); + for (uint32_t i = 0; spec && i < spec->mapEntryCount; i++) + sb_printf(&b, "%s[%u, %u, %zu]", i ? ", " : "", spec->pMapEntries[i].constantID, spec->pMapEntries[i].offset, + spec->pMapEntries[i].size); + sb_printf(&b, "], \"data\": "); + sb_hex(&b, spec ? spec->pData : NULL, spec ? spec->dataSize : 0); + sb_printf(&b, "},\n \"layout\": ["); + VK_FROM_HANDLE(vk_pipeline_layout, layout, layout_handle); + for (uint32_t s = 0; layout && s < layout->set_count; s++) { + const struct vk_descriptor_set_layout *vs = layout->set_layouts[s]; + sb_printf(&b, "%s\n [", s ? "," : ""); + if (vs) { + const struct pvr_descriptor_set_layout *ps = container_of(vs, struct pvr_descriptor_set_layout, vk); + bool first = true; + for (uint32_t i = 0; i < ps->binding_count; i++) { + const struct pvr_descriptor_set_layout_binding *pb = &ps->bindings[i]; + if (!pb->descriptor_count) + continue; + sb_printf(&b, "%s{\"binding\": %u, \"type\": \"%s\", \"count\": %u, \"stages\": %u}", first ? "" : ", ", + i, descriptor_type_name(pb->type), pb->descriptor_count, pb->stage_flags); + first = false; + } + } + sb_printf(&b, "]"); + } + sb_printf(&b, "],\n \"push_constants\": ["); + for (uint32_t i = 0; layout && i < layout->push_range_count; i++) + sb_printf(&b, "%s[%u, %u, %u]", i ? ", " : "", layout->push_ranges[i].stageFlags, + layout->push_ranges[i].offset, layout->push_ranges[i].size); + sb_printf(&b, "],\n \"keys\": {"); + for (unsigned i = 0; i < nblobs; i++) { + sb_printf(&b, "%s\"%s\": ", i ? ", " : "", blobs[i].name); + sb_hex(&b, blobs[i].data, blobs[i].size); + } + sb_printf(&b, "}"); + if (!b.s) + return; + /* the variant: everything above */ + blake3_hash vsha; + char vhex[BLAKE3_HEX_LEN]; + _mesa_blake3_compute(b.s, b.len, vsha); + _mesa_blake3_format(vhex, vsha); + vhex[12] = 0; + sb_printf(&b, ",\n \"result\": \"%s\",\n \"reason\": ", result); + sb_json_string(&b, reason ? reason : ""); + sb_printf(&b, "\n}\n"); + snprintf(path, sizeof(path), "%s/%s.%s.%s.json", dir, hex, stage, vhex); + if (b.s) + write_atomic(path, b.s, b.len, false); + free(b.s); + + /* one index line, one append */ + char line[1024]; + int n = snprintf(line, sizeof(line), "%lld\t%ld\t%s\t%s\t%s\t%s\t", (long long)time(NULL), (long)getpid(), + hex, stage, vhex, result); + for (const char *r = reason ? reason : ""; *r && n < (int)sizeof(line) - 2; r++) + line[n++] = *r == '\t' || *r == '\n' ? ' ' : *r; + line[MIN2(n, (int)sizeof(line) - 2)] = '\n'; + n = MIN2(n + 1, (int)sizeof(line) - 1); + snprintf(path, sizeof(path), "%s/index.tsv", dir); + int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (fd >= 0) { + ssize_t w = write(fd, line, n); + (void)w; + close(fd); + } +} + +const VkPipelineShaderStageCreateInfo * +pvr_g5p_stub_stage(VkShaderStageFlagBits stage, VkPipelineShaderStageCreateInfo *info, VkShaderModuleCreateInfo *module) +{ + const uint32_t *code = stage == VK_SHADER_STAGE_FRAGMENT_BIT ? pvr_g5p_stub_frag : + stage == VK_SHADER_STAGE_VERTEX_BIT ? pvr_g5p_stub_vert : pvr_g5p_stub_comp; + const size_t size = stage == VK_SHADER_STAGE_FRAGMENT_BIT ? sizeof(pvr_g5p_stub_frag) : + stage == VK_SHADER_STAGE_VERTEX_BIT ? sizeof(pvr_g5p_stub_vert) : sizeof(pvr_g5p_stub_comp); + *module = (VkShaderModuleCreateInfo){ .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, + .codeSize = size, .pCode = code }; + *info = (VkPipelineShaderStageCreateInfo){ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .pNext = module, .stage = stage, .pName = "main" }; + return info; +} + +thread_local unsigned pvr_g5p_internal_pipeline; diff --git a/src/imagination/vulkan/pvr_g5p_shader_dump.h b/src/imagination/vulkan/pvr_g5p_shader_dump.h new file mode 100644 index 00000000000..3d3ddab56c0 --- /dev/null +++ b/src/imagination/vulkan/pvr_g5p_shader_dump.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Shader capture for compatibility triage (diagnostic; off unless set). + * + * PVR_DEBUG=shader_dump (PVR_SHADER_DUMP_DIR=): every shader stage the G5P backend compiles is + * recorded in : + * .spv the SPIR-V module, once (first 40 hex digits of + * the BLAKE3 of its words) + * ...json the compile: stage, entry point, specialization + * constants, pipeline layout, the backend's key + * structs as hex (fragment key, vertex layout, + * link; enough for an offline admission check + * replay), and the result: accepted, or refused + * with the backend's reason. = a hash of + * everything but the result. + * index.tsv one line per compile: unix time, pid, hash, + * stage, v, ADMIT/REFUSE/STUB, the reason + * Files are written to a temporary name and renamed (several threads and + * processes may compile at once); index lines are single appends. + * + * PVR_DEBUG=capture_stub: a refused stage is replaced by a stub with the + * same interface (fragment: magenta at location 0; vertex: a point at the + * origin, nothing drawn; compute: an empty kernel, the dispatch omitted), so + * the pipeline is created and the application keeps creating the rest. The + * refusal is still logged ("G5P capture stub: ...") and dumped. Never on by + * default: stubbed pipelines draw wrong pictures. + */ +#ifndef PVR_G5P_SHADER_DUMP_H +#define PVR_G5P_SHADER_DUMP_H + +#include +#include "c11/threads.h" +#include +#include + +struct pvr_g5p_dump_blob { + const char *name; + const void *data; + size_t size; +}; + +/* NULL when PVR_DEBUG=shader_dump is unset (read once) */ +const char *pvr_g5p_shader_dump_dir(void); +bool pvr_g5p_capture_stub(void); +bool pvr_g5p_compute_log(void); +void pvr_g5p_shader_id(const VkPipelineShaderStageCreateInfo *info, char id[13]); + +/* Record one compile of @stage_info (no-op without PVR_DEBUG=shader_dump). + * @result: "ADMIT", "REFUSE" or "STUB" (refused, then stubbed). */ +void pvr_g5p_shader_dump(const VkPipelineShaderStageCreateInfo *stage_info, VkPipelineLayout layout, + const char *stage, const struct pvr_g5p_dump_blob *blobs, unsigned nblobs, + const char *result, const char *reason); + +/* A stage create info for the stub of @stage, in caller storage */ +const VkPipelineShaderStageCreateInfo *pvr_g5p_stub_stage(VkShaderStageFlagBits stage, + VkPipelineShaderStageCreateInfo *info, + VkShaderModuleCreateInfo *module); + +/* Pipelines the driver makes for itself (clear quads, vk_meta blits) are created with this + * raised: a refused one fails as before instead of being admitted without programs */ +extern thread_local unsigned pvr_g5p_internal_pipeline; + +#endif diff --git a/src/imagination/vulkan/pvr_g5p_stub_spv.h b/src/imagination/vulkan/pvr_g5p_stub_spv.h new file mode 100644 index 00000000000..4dbf80640ec --- /dev/null +++ b/src/imagination/vulkan/pvr_g5p_stub_spv.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +/* PVR_DEBUG=capture_stub substitutes (pvr_g5p_shader_dump.c), glslangValidator -V --target-env vulkan1.1: + * frag: layout(location=0) out vec4 c; c = vec4(1, 0, 1, 1) (magenta) + * vert: gl_Position = vec4(0, 0, 0, 1) (a point: draws nothing) + * comp: local_size 1, empty (a bodyless dispatch: omitted) */ +#ifndef PVR_G5P_STUB_SPV_H +#define PVR_G5P_STUB_SPV_H +#include +static const uint32_t pvr_g5p_stub_frag[] = { + 0x07230203,0x00010300,0x0008000b,0x0000000d,0x00000000,0x00020011,0x00000001,0x0006000b, + 0x00000001,0x4c534c47,0x6474732e,0x3035342e,0x00000000,0x0003000e,0x00000000,0x00000001, + 0x0006000f,0x00000004,0x00000004,0x6e69616d,0x00000000,0x00000009,0x00030010,0x00000004, + 0x00000007,0x00030003,0x00000002,0x000001c2,0x00040005,0x00000004,0x6e69616d,0x00000000, + 0x00030005,0x00000009,0x00000063,0x00040047,0x00000009,0x0000001e,0x00000000,0x00020013, + 0x00000002,0x00030021,0x00000003,0x00000002,0x00030016,0x00000006,0x00000020,0x00040017, + 0x00000007,0x00000006,0x00000004,0x00040020,0x00000008,0x00000003,0x00000007,0x0004003b, + 0x00000008,0x00000009,0x00000003,0x0004002b,0x00000006,0x0000000a,0x3f800000,0x0004002b, + 0x00000006,0x0000000b,0x00000000,0x0007002c,0x00000007,0x0000000c,0x0000000a,0x0000000b, + 0x0000000a,0x0000000a,0x00050036,0x00000002,0x00000004,0x00000000,0x00000003,0x000200f8, + 0x00000005,0x0003003e,0x00000009,0x0000000c,0x000100fd,0x00010038 +}; +static const uint32_t pvr_g5p_stub_vert[] = { + 0x07230203,0x00010300,0x0008000b,0x00000015,0x00000000,0x00020011,0x00000001,0x0006000b, + 0x00000001,0x4c534c47,0x6474732e,0x3035342e,0x00000000,0x0003000e,0x00000000,0x00000001, + 0x0006000f,0x00000000,0x00000004,0x6e69616d,0x00000000,0x0000000d,0x00030003,0x00000002, + 0x000001c2,0x00040005,0x00000004,0x6e69616d,0x00000000,0x00060005,0x0000000b,0x505f6c67, + 0x65567265,0x78657472,0x00000000,0x00060006,0x0000000b,0x00000000,0x505f6c67,0x7469736f, + 0x006e6f69,0x00070006,0x0000000b,0x00000001,0x505f6c67,0x746e696f,0x657a6953,0x00000000, + 0x00070006,0x0000000b,0x00000002,0x435f6c67,0x4470696c,0x61747369,0x0065636e,0x00070006, + 0x0000000b,0x00000003,0x435f6c67,0x446c6c75,0x61747369,0x0065636e,0x00030005,0x0000000d, + 0x00000000,0x00030047,0x0000000b,0x00000002,0x00050048,0x0000000b,0x00000000,0x0000000b, + 0x00000000,0x00050048,0x0000000b,0x00000001,0x0000000b,0x00000001,0x00050048,0x0000000b, + 0x00000002,0x0000000b,0x00000003,0x00050048,0x0000000b,0x00000003,0x0000000b,0x00000004, + 0x00020013,0x00000002,0x00030021,0x00000003,0x00000002,0x00030016,0x00000006,0x00000020, + 0x00040017,0x00000007,0x00000006,0x00000004,0x00040015,0x00000008,0x00000020,0x00000000, + 0x0004002b,0x00000008,0x00000009,0x00000001,0x0004001c,0x0000000a,0x00000006,0x00000009, + 0x0006001e,0x0000000b,0x00000007,0x00000006,0x0000000a,0x0000000a,0x00040020,0x0000000c, + 0x00000003,0x0000000b,0x0004003b,0x0000000c,0x0000000d,0x00000003,0x00040015,0x0000000e, + 0x00000020,0x00000001,0x0004002b,0x0000000e,0x0000000f,0x00000000,0x0004002b,0x00000006, + 0x00000010,0x00000000,0x0004002b,0x00000006,0x00000011,0x3f800000,0x0007002c,0x00000007, + 0x00000012,0x00000010,0x00000010,0x00000010,0x00000011,0x00040020,0x00000013,0x00000003, + 0x00000007,0x00050036,0x00000002,0x00000004,0x00000000,0x00000003,0x000200f8,0x00000005, + 0x00050041,0x00000013,0x00000014,0x0000000d,0x0000000f,0x0003003e,0x00000014,0x00000012, + 0x000100fd,0x00010038 +}; +static const uint32_t pvr_g5p_stub_comp[] = { + 0x07230203,0x00010300,0x0008000b,0x0000000a,0x00000000,0x00020011,0x00000001,0x0006000b, + 0x00000001,0x4c534c47,0x6474732e,0x3035342e,0x00000000,0x0003000e,0x00000000,0x00000001, + 0x0005000f,0x00000005,0x00000004,0x6e69616d,0x00000000,0x00060010,0x00000004,0x00000011, + 0x00000001,0x00000001,0x00000001,0x00030003,0x00000002,0x000001c2,0x00040005,0x00000004, + 0x6e69616d,0x00000000,0x00040047,0x00000009,0x0000000b,0x00000019,0x00020013,0x00000002, + 0x00030021,0x00000003,0x00000002,0x00040015,0x00000006,0x00000020,0x00000000,0x00040017, + 0x00000007,0x00000006,0x00000003,0x0004002b,0x00000006,0x00000008,0x00000001,0x0006002c, + 0x00000007,0x00000009,0x00000008,0x00000008,0x00000008,0x00050036,0x00000002,0x00000004, + 0x00000000,0x00000003,0x000200f8,0x00000005,0x000100fd,0x00010038 +}; +#endif diff --git a/src/imagination/vulkan/pvr_g5p_tex.h b/src/imagination/vulkan/pvr_g5p_tex.h index 5e96d790931..4a130bfdcfd 100644 --- a/src/imagination/vulkan/pvr_g5p_tex.h +++ b/src/imagination/vulkan/pvr_g5p_tex.h @@ -40,7 +40,7 @@ struct pvr_sampler_descriptor; /* What the texture state needs of a format (pvr_arch_g5p_tex.c looks it * up). Lane (f)'s format table, pvr_arch_g5p_formats.c, is to provide it; * until then pvr_arch_g5p_tex.c has the formats of the rendered and - * textured rungs only. */ + * textured cases only. */ struct pvr_g5p_tex_format { VkFormat format; uint32_t key; /* PVR_G5P_TEX_KEY(): format code and channel selectors */ @@ -51,12 +51,15 @@ struct pvr_g5p_tex_format { const char *PVR_PER_ARCH(g5p_pack_image_view)(const struct pvr_image_view *iview, struct pvr_image_descriptor *desc); +const char *PVR_PER_ARCH(g5p_pack_storage_image_view)(const struct pvr_image_view *iview, + struct pvr_image_descriptor *desc); const char *PVR_PER_ARCH(g5p_pack_sampler)(const VkSamplerCreateInfo *info, struct pvr_sampler_descriptor *desc); void PVR_PER_ARCH(g5p_combine_sampler)(struct pvr_combined_image_sampler_descriptor *desc, const struct pvr_image_view *iview); # define pvr_arch_g5p_pack_image_view PVR_PER_ARCH(g5p_pack_image_view) +# define pvr_arch_g5p_pack_storage_image_view PVR_PER_ARCH(g5p_pack_storage_image_view) # define pvr_arch_g5p_pack_sampler PVR_PER_ARCH(g5p_pack_sampler) # define pvr_arch_g5p_combine_sampler PVR_PER_ARCH(g5p_combine_sampler) diff --git a/src/imagination/vulkan/pvr_g5p_tex_encode.h b/src/imagination/vulkan/pvr_g5p_tex_encode.h index 80d7f35e206..bc875761bec 100644 --- a/src/imagination/vulkan/pvr_g5p_tex_encode.h +++ b/src/imagination/vulkan/pvr_g5p_tex_encode.h @@ -326,6 +326,37 @@ pvr_g5p_tex_size(const uint32_t state[4], unsigned level, uint32_t out[4]) /* A descriptor's state as the draw path uploads it: a valid image state * never has q0 = 0 (its layout bits), a valid sampler word is never 0. */ +/* + * The texture state an upload's sample reads (PCO_G5P_UPLOAD_IMAGE's offset + * flags), in place over the 8 copied dwords; NULL, or the reason there is none. + * - AS_2D (an array sample at layer 0 drawn as 2D): a 2D_ARRAY state becomes + * its base layer's 2D state (layout 2, no layer count, no big-layer bit; + * the address is already the base layer's, as a 2D view of that layer + * gets); a 2D or linear state is used as it is; 3D/cube have no 2D form. + * - ARRAY: the state must be an array's (a single-layer array view of a + * linear image has only its 2D state: sampled at a non-constant layer). + */ +static inline const char * +pvr_g5p_tex_state_for(uint32_t dw[8], unsigned flags) +{ + const unsigned layout = dw[0] & 7; + if (flags & 1u /* PCO_G5P_UPLOAD_IMAGE_AS_2D */) { + if (layout == PVR_G5P_TEX_LAYOUT_2D_ARRAY) { + uint64_t q0 = dw[0] | (uint64_t)dw[1] << 32, q1 = dw[2] | (uint64_t)dw[3] << 32; + q0 = (q0 & ~UINT64_C(7)) | PVR_G5P_TEX_LAYOUT_2D; + q1 &= ~(UINT64_C(0x3fff) << PVR_G5P_TEX_DEPTH_SHIFT); + q1 &= ~(UINT64_C(1) << PVR_G5P_TEX_BIG_LAYER_BIT); + dw[0] = (uint32_t)q0, dw[1] = (uint32_t)(q0 >> 32), dw[2] = (uint32_t)q1, dw[3] = (uint32_t)(q1 >> 32); + } else if (layout != PVR_G5P_TEX_LAYOUT_2D) { + return "a layer-0 array sample of a 3D or cube view"; + } + } else if ((flags & 2u /* PCO_G5P_UPLOAD_IMAGE_ARRAY */) && layout != PVR_G5P_TEX_LAYOUT_2D_ARRAY) { + return "an array sample at a non-constant layer of a view without array state (a single-layer view of a " + "linear image: Apple samples arrays twiddled only)"; + } + return NULL; +} + static inline bool pvr_g5p_tex_state_valid(const uint32_t dwords[2]) { diff --git a/src/imagination/vulkan/pvr_g5p_upload.h b/src/imagination/vulkan/pvr_g5p_upload.h index f535e124e70..909420a2fb0 100644 --- a/src/imagination/vulkan/pvr_g5p_upload.h +++ b/src/imagination/vulkan/pvr_g5p_upload.h @@ -33,7 +33,10 @@ struct pco_g5p_program; struct pvr_cmd_buffer; struct pvr_device; -#define PVR_G5P_MAX_STAGE_UPLOADS 16 +/* 8+ storage-buffer pointers, 13 UBOs, push constants and the XFB block + * (WebGPU core per-stage maxima together). The loader's data is one 16-byte + * unit per upload; VDMCTRL_PDS_STATE0.pds_data_size holds up to 63. */ +#define PVR_G5P_MAX_STAGE_UPLOADS 24 /* One upload of a stage: what the backend's pco_g5p_upload names */ struct pvr_g5p_stage_upload { diff --git a/src/imagination/vulkan/pvr_g5p_vertex.h b/src/imagination/vulkan/pvr_g5p_vertex.h index e65e9e8fbd6..99abbcf4329 100644 --- a/src/imagination/vulkan/pvr_g5p_vertex.h +++ b/src/imagination/vulkan/pvr_g5p_vertex.h @@ -26,8 +26,11 @@ struct nir_shader; struct vk_vertex_input_state; -/* The backend's unified store for vertex inputs: dwords r0..r31 */ +/* The backend's unified store for vertex inputs: dwords r0..r31 (the one-location-per-vec4 layout's range) */ #define PVR_G5P_VERTEX_INPUT_DWORDS 32 +/* The packed layout's limit: attributes above r31 are read through a move (pco_g5p_general.c VS_INPUT_DWORDS, + * Apple's vertex form for attributes above r31: 44 dwords from r0 in applegpu_g5p) */ +#define PVR_G5P_VERTEX_INPUT_MAX_DWORDS 64 /* * A pipeline's vertex input layout. pvr_g5p_vertex_layout_init() takes the @@ -71,6 +74,7 @@ struct pvr_g5p_vertex_format { uint8_t components, bytes; int8_t convert; bool packed_1010102; /* one raw dword; components unpacked in NIR */ + bool raw_int; /* 8/16-bit pure integers: the dwords fetched raw, each component shifted out in NIR */ uint8_t swizzle[4]; }; diff --git a/src/imagination/vulkan/pvr_g5p_vertex_nir.c b/src/imagination/vulkan/pvr_g5p_vertex_nir.c index 5facbc525b5..ffebebbc7b3 100644 --- a/src/imagination/vulkan/pvr_g5p_vertex_nir.c +++ b/src/imagination/vulkan/pvr_g5p_vertex_nir.c @@ -26,7 +26,9 @@ #include "compiler/nir/nir.h" #include "compiler/nir/nir_builder.h" #include "compiler/shader_enums.h" +#include "util/log.h" #include "util/macros.h" +#include "util/u_debug.h" #include "util/format/u_format.h" #include "util/u_math.h" #include "vk_format.h" @@ -81,8 +83,25 @@ pvr_g5p_vertex_format_info(VkFormat format, struct pvr_g5p_vertex_format *f) if (!d || d->layout != UTIL_FORMAT_LAYOUT_PLAIN || d->block.width != 1 || d->block.height != 1 || d->nr_channels < 1 || d->nr_channels > 4) return false; - /* Packed 10:10:10:2 formats (a 32-bit fetch followed by scalar NIR - * operations, f->packed_1010102) have not run on the device: not taken. */ + /* Packed 10:10:10:2 formats: a 32-bit fetch followed by ordinary scalar + * NIR operations (f->packed_1010102); no hardware packed conversion is + * assumed. Tested on the A9 before the powervr interface: all 42 + * dEQP-GLES3 vertex_arrays.single_attribute *2_10_10_10 cases fixed + * (int and uint, wrong result before), 24 controls kept, none lost. */ + if (d->nr_channels == 4 && d->block.bits == 32 && + d->channel[0].size == 10 && d->channel[1].size == 10 && + d->channel[2].size == 10 && d->channel[3].size == 2) { + f->components = 4; + f->bytes = 1; /* four logical components occupy one dword */ + f->convert = -1; + f->packed_1010102 = true; + for (unsigned c = 0; c < 4; c++) { + if (d->swizzle[c] > PIPE_SWIZZLE_W) + return false; + f->swizzle[c] = d->swizzle[c]; + } + return true; + } const struct util_format_channel_description *ch = &d->channel[0]; const unsigned bits = ch->size; if ((bits != 8 && bits != 16 && bits != 32) || d->block.bits != bits * d->nr_channels) @@ -118,9 +137,16 @@ pvr_g5p_vertex_format_info(VkFormat format, struct pvr_g5p_vertex_format *f) f->convert = PCO_G5P_VSF_F16; return true; } - /* to float: normalized or scaled; 8- and 16-bit integer inputs are not - * converted yet */ - if (!(is_signed || is_unsigned) || ch->pure_integer) + /* 8- and 16-bit pure integers (glVertexAttribIPointer with GL_BYTE ... GL_UNSIGNED_SHORT): the dwords + * that hold them fetched raw, each component shifted out (and sign-extended) in NIR, no conversion + * (G9P's 4a8c71666d9 ("pvr/g9p: Qt Quick on Zink: packed vertex formats, texelFetch") for the A10) */ + if ((is_signed || is_unsigned) && ch->pure_integer) { + f->convert = -1; + f->raw_int = true; + return true; + } + /* to float: normalized or scaled */ + if (!(is_signed || is_unsigned)) return false; f->convert = (bits == 16 ? PCO_G5P_VSF_U16 : PCO_G5P_VSF_U8) + (ch->normalized ? 2 : 0) + is_signed; return true; @@ -259,14 +285,29 @@ packed_fetches(struct pvr_g5p_vertex_layout *l, struct attrib *a, unsigned n, un } slot += a[i].dwords; } - if (slot > PVR_G5P_VERTEX_INPUT_DWORDS) - return REFUSE(l, "vertex attributes of %u dwords (the unified store holds %u)", slot, - PVR_G5P_VERTEX_INPUT_DWORDS); + if (slot > PVR_G5P_VERTEX_INPUT_MAX_DWORDS) + return REFUSE(l, "vertex attributes of %u dwords (%u are built)", slot, PVR_G5P_VERTEX_INPUT_MAX_DWORDS); for (unsigned v = 0; 4 * v < slot; v++) l->components[v] = MIN2(4, slot - 4 * v); return true; } +/* Each fetch's overread: how far it reads past the end of the last attribute of its binding's element. Vulkan + * keeps every fetched vertex's attributes inside the bound range, so the draw's reads end at most that many bytes + * past it (pvr_arch_g5p_gfx.c keeps them inside the buffer's GPU mapping, or draws from a padded copy). */ +static void +fetch_overreads(struct pvr_g5p_vertex_layout *l, const struct attrib *a, unsigned n) +{ + uint32_t end[MESA_VK_MAX_VERTEX_BINDINGS] = { 0 }; + for (unsigned i = 0; i < n; i++) + end[a[i].binding] = MAX2(end[a[i].binding], a[i].offset + a[i].lead + a[i].f.bytes * a[i].f.components); + for (unsigned f = 0; f < l->fetch_count; f++) { + struct pvr_g5p_fetch *fe = &l->fetch_list[f]; + const uint32_t fetch_end = fe->offset + 4 * fe->dwords; + fe->overread = fetch_end > end[fe->binding] ? MIN2(fetch_end - end[fe->binding], UINT16_MAX) : 0; + } +} + struct rewrite { const struct pvr_g5p_vertex_layout *l; const unsigned *base; @@ -295,6 +336,20 @@ rewrite_load(nir_builder *b, nir_intrinsic_instr *in, void *data) c[k] = comp != 3 ? nir_imm_int(b, 0) : is_float ? nir_imm_float(b, 1.0f) : nir_imm_int(b, 1); continue; } + if (f.raw_int) { + /* the component's bytes in the attribute's fetched dwords */ + const unsigned at = lead + f.bytes * f.swizzle[comp], reg = rw->base[loc] + at / 4; + const unsigned shift = 8 * (at % 4), bits = 8 * f.bytes; + nir_def *word = nir_load_input(b, 1, 32, nir_imm_int(b, 0), .base = reg / 4, + .component = reg % 4, .dest_type = nir_type_uint32, + .io_semantics = (nir_io_semantics){ + .location = VERT_ATTRIB_GENERIC0 + reg / 4, .num_slots = 1 }); + const bool is_signed = vk_format_description(rw->l->vi->attributes[loc].format)->channel[0].type == + UTIL_FORMAT_TYPE_SIGNED; + c[k] = is_signed ? nir_ishr_imm(b, nir_ishl_imm(b, word, 32 - shift - bits), 32 - bits) + : nir_iand_imm(b, nir_ushr_imm(b, word, shift), BITFIELD_MASK(bits)); + continue; + } if (f.packed_1010102) { unsigned reg = rw->base[loc]; nir_def *word = nir_load_input(b, 1, 32, nir_imm_int(b, 0), .base = reg / 4, @@ -323,6 +378,19 @@ rewrite_load(nir_builder *b, nir_intrinsic_instr *in, void *data) /* the component's byte in the fetched dwords; a converted one names * its format and byte in the load's base */ const unsigned at = lead + f.bytes * f.swizzle[comp], reg = rw->base[loc] + at / 4; + /* a 32-bit component off dwords (an attribute at an unaligned offset): the two dwords it straddles, + * funnel-shifted (the binding itself is dword-aligned, pvr_arch_g5p_gfx.c g5p_realign_binding()) */ + if (f.convert < 0 && at % 4) { + nir_def *w[2]; + for (unsigned i = 0; i < 2; i++) + w[i] = nir_load_input(b, 1, 32, nir_imm_int(b, 0), .base = (reg + i) / 4, .component = (reg + i) % 4, + .dest_type = nir_type_uint32, + .io_semantics = (nir_io_semantics){ .location = VERT_ATTRIB_GENERIC0 + (reg + i) / 4, + .num_slots = 1 }); + const unsigned sh = 8 * (at % 4); + c[k] = nir_ior(b, nir_ushr_imm(b, w[0], sh), nir_ishl_imm(b, w[1], 32 - sh)); + continue; + } const unsigned base = f.convert < 0 ? reg / 4 : PCO_G5P_VS_CONVERT_BASE + 4 * f.convert + at % 4; c[k] = nir_load_input(b, 1, 32, nir_imm_int(b, 0), .base = base, .component = reg % 4, .dest_type = type, .io_semantics = (nir_io_semantics){ .location = VERT_ATTRIB_GENERIC0 + reg / 4, @@ -332,11 +400,44 @@ rewrite_load(nir_builder *b, nir_intrinsic_instr *in, void *data) return true; } +/* A load of a location the pipeline gives no attribute: (0, 0, 0, 1), GL's generic-attribute default that Zink's GL + * relies on (G9P's 4a8c71666d9 ("pvr/g9p: Qt Quick on Zink: packed vertex formats, texelFetch")). Diagnostic: each pipeline logs the locations it substitutes. */ +struct unbound_state { + uint32_t valid, substituted; +}; + +static bool +unbound_input(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + struct unbound_state *u = data; + if (in->intrinsic != nir_intrinsic_load_input) + return false; + const int loc = generic_location(in); + if (loc < 0 || (u->valid & BITFIELD_BIT(loc)) || in->def.bit_size != 32) + return false; + u->substituted |= BITFIELD_BIT(loc); + const bool is_float = nir_alu_type_get_base_type(nir_intrinsic_dest_type(in)) == nir_type_float; + b->cursor = nir_before_instr(&in->instr); + nir_def *c[4]; + for (unsigned k = 0; k < in->def.num_components; k++) + c[k] = nir_intrinsic_component(in) + k != 3 ? nir_imm_int(b, 0) : is_float ? nir_imm_float(b, 1.0f) + : nir_imm_int(b, 1); + nir_def_replace(&in->def, nir_vec(b, c, in->def.num_components)); + return true; +} + bool pvr_g5p_vertex_lower(nir_shader *vs, struct pvr_g5p_vertex_layout *l) { const struct vk_vertex_input_state *vi = l->vi; + { + struct unbound_state u = { .valid = vi ? vi->attributes_valid : 0 }; + nir_shader_intrinsics_pass(vs, unbound_input, nir_metadata_control_flow, &u); + u_foreach_bit (loc, u.substituted) + mesa_logw("G5P vertex input %u has no attribute: reading (0,0,0,1)", loc); + } + /* the locations the program reads */ l->read = 0; nir_foreach_function_impl (impl, vs) { @@ -378,14 +479,12 @@ pvr_g5p_vertex_lower(nir_shader *vs, struct pvr_g5p_vertex_layout *l) /* the fetch is dword-granular: a 32-bit attribute starts on a dword, * a converted one is read from the dwords that hold its bytes */ const unsigned lead = at->offset % 4; - if (f.convert < 0 && lead) - return REFUSE(l, "vertex attribute %u at offset %u (the fetch is dword-granular)", loc, at->offset); - if (lead % f.bytes) + /* a 32-bit attribute at an offset off dwords: read from the two dwords it straddles (rewrite_load()) */ + const bool straddle = f.convert < 0 && !f.raw_int && !f.packed_1010102 && lead; + if (!straddle && lead % f.bytes) return REFUSE(l, "vertex attribute %u: %u-byte components at offset %u", loc, f.bytes, at->offset); - if (!l->dynamic_strides && vi->bindings[at->binding].stride % 4) - return REFUSE(l, "vertex binding %u: stride %u (the fetch is dword-granular)", at->binding, - vi->bindings[at->binding].stride); - converted |= f.convert >= 0 || f.packed_1010102; + /* a stride off dwords: the draw's copy (pvr_arch_g5p_gfx.c g5p_realign_binding()) */ + converted |= f.convert >= 0 || f.packed_1010102 || f.raw_int || straddle; a[n++] = (struct attrib){ loc, at->binding, DIV_ROUND_UP(lead + f.bytes * f.components, 4), at->offset - lead, lead, f }; } @@ -395,12 +494,14 @@ pvr_g5p_vertex_lower(nir_shader *vs, struct pvr_g5p_vertex_layout *l) l->packed = converted || l->read >= BITFIELD_BIT(ICD_LOCATIONS); if (!l->packed) { icd_fetches(l, a, n); + fetch_overreads(l, a, n); return fetch_divisors(l); } unsigned base[PCO_G5P_MAX_ATTRIBS] = { 0 }; if (!packed_fetches(l, a, n, base) || !fetch_divisors(l)) return false; + fetch_overreads(l, a, n); const struct rewrite rw = { l, base }; nir_shader_intrinsics_pass(vs, rewrite_load, nir_metadata_control_flow, (void *)&rw); nir_opt_constant_folding(vs); diff --git a/src/imagination/vulkan/pvr_g9p.h b/src/imagination/vulkan/pvr_g9p.h index baeac2dd543..4aeae805fa9 100644 --- a/src/imagination/vulkan/pvr_g9p.h +++ b/src/imagination/vulkan/pvr_g9p.h @@ -7,6 +7,7 @@ #ifndef PVR_G9P_H #define PVR_G9P_H +#include #include #include @@ -23,6 +24,18 @@ struct pvr_graphics_pipeline; struct pvr_render_job; struct pvr_renderpass_hwsetup_render; struct pvr_sub_cmd_compute; +/* A vertex binding copied to a dword-aligned stride when its job is submitted (pvr_arch_g9p_gfx.c; G5P's draws + * queue them too, pvr_arch_g5p_gfx.c): src's count elements of stride bytes to dst at new_stride */ +struct g9p_realign { + const uint8_t *src; + uint8_t *dst; + uint64_t size, count; + uint32_t stride, new_stride; + bool gpu_written; /* the source may be written by an earlier GPU job */ + uint64_t dst_addr; /* the copy's device address (G5P reuses a copy across the sub-command's draws) */ + uint32_t extra; /* G5P: zero bytes after the copy's last element, for its fetches' overread */ +}; + struct pvr_sub_cmd_gfx; struct vk_graphics_pipeline_state; diff --git a/src/imagination/vulkan/pvr_image.c b/src/imagination/vulkan/pvr_image.c index cef6854870c..55693cdd2d4 100644 --- a/src/imagination/vulkan/pvr_image.c +++ b/src/imagination/vulkan/pvr_image.c @@ -21,6 +21,7 @@ * SOFTWARE. */ +#include "pvr_bo_log.h" #include "pvr_image.h" #include @@ -430,6 +431,9 @@ VkResult pvr_image_init(struct pvr_device *device, void pvr_image_fini(struct pvr_device *device, struct pvr_image *image) { + if (unlikely(device->bo_log)) + pvr_bo_log_unbind(device, image); + if (image->vma) pvr_unbind_memory(device, image->vma); } @@ -509,11 +513,15 @@ VkResult pvr_BindImageMemory2(VkDevice _device, while (i--) { VK_FROM_HANDLE(pvr_image, image, pBindInfos[i].image); + if (unlikely(device->bo_log)) + pvr_bo_log_unbind(device, image); pvr_unbind_memory(device, image->vma); } return result; } + if (unlikely(device->bo_log)) + pvr_bo_log_image(device, image, mem, offset); } return VK_SUCCESS; diff --git a/src/imagination/vulkan/pvr_instance.c b/src/imagination/vulkan/pvr_instance.c index da1f495bfdd..dc54f7023c1 100644 --- a/src/imagination/vulkan/pvr_instance.c +++ b/src/imagination/vulkan/pvr_instance.c @@ -371,6 +371,7 @@ VkResult pvr_CreateInstance(const VkInstanceCreateInfo *pCreateInfo, goto err_free_instance; pvr_process_debug_variable(); + instance->vk.enable_debug_logging = PVR_IS_DEBUG_SET(ERRORS); pvr_init_dri_options(instance); instance->active_device_count = 0; diff --git a/src/imagination/vulkan/pvr_pass.h b/src/imagination/vulkan/pvr_pass.h index 1b52787e9f3..c290ab9519a 100644 --- a/src/imagination/vulkan/pvr_pass.h +++ b/src/imagination/vulkan/pvr_pass.h @@ -101,6 +101,13 @@ struct pvr_render_subpass { struct pvr_render_pass { struct vk_object_base base; + /* The pass's dependencies on commands outside it (srcSubpass or + * dstSubpass VK_SUBPASS_EXTERNAL), merged: G5P records them as pipeline + * barriers when the pass begins and ends (a copy after a render pass that + * declares 0 -> EXTERNAL must see its colour). */ + bool ext_begin, ext_end; + VkMemoryBarrier2 ext_begin_barrier, ext_end_barrier; + /* Saved information from pCreateInfo. */ uint32_t attachment_count; diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index f5812d1016c..d3c25d5fd1c 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -268,7 +268,8 @@ static void pvr_physical_device_get_supported_features( .multiViewport = false, .samplerAnisotropy = true, .textureCompressionETC2 = true, - .textureCompressionASTC_LDR = false, + /* G5P: all 28 ASTC LDR formats sampled, filtered and blit sources (pvr_g5p_format_table.h) */ + .textureCompressionASTC_LDR = PVR_HAS_FEATURE(dev_info, apple_g5p), .textureCompressionBC = false, .occlusionQueryPrecise = true, .pipelineStatisticsQuery = false, @@ -285,7 +286,10 @@ static void pvr_physical_device_get_supported_features( .shaderStorageBufferArrayDynamicIndexing = true, .shaderStorageImageArrayDynamicIndexing = true, .shaderClipDistance = true, - .shaderCullDistance = true, + /* G5P: the A9 TA's user planes clip; Apple's A9 driver sets no cull plane, and a cull + * distance is no clip distance (it drops whole primitives): not supported, so applications + * (PPSSPP) do not write gl_CullDistance */ + .shaderCullDistance = !PVR_HAS_FEATURE(dev_info, apple_g5p), .shaderFloat64 = false, .shaderInt64 = false, .shaderInt16 = false, @@ -605,6 +609,9 @@ static uint32_t get_api_version(void) static unsigned get_custom_border_color_samplers(const struct pvr_device_info *dev_info); +/* Minimum required by the Vulkan spec Limits (maxMemoryAllocationSize) */ +#define PVR_MAX_MEMORY_ALLOCATION_SIZE (1ull << 30) + static bool pvr_physical_device_get_properties( const struct pvr_physical_device *const pdevice, struct vk_properties *const properties) @@ -656,7 +663,7 @@ static bool pvr_physical_device_get_properties( assert(pdevice->memory.memoryHeapCount == 1); const VkDeviceSize max_memory_alloc_size = - pdevice->memory.memoryHeaps[0].size; + MAX2(pdevice->memory.memoryHeaps[0].size, PVR_MAX_MEMORY_ALLOCATION_SIZE); const uint32_t line_sub_pixel_precision_bits = PVR_HAS_FEATURE(dev_info, simple_internal_parameter_format) ? 4U : 8U; @@ -671,13 +678,15 @@ static bool pvr_physical_device_get_properties( .deviceType = VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU, /* deviceName and pipelineCacheUUID are filled below .*/ - .maxImageDimension1D = 4096U, - .maxImageDimension2D = 4096U, + /* G5P: 8192, the Orchard kernel's render-target bound (G5P_RU_MAX_DIM) and inside the A9's + * 14-bit width/height fields of the texture and PBE states (pvr_g5p_tex_encode.h, g5p_pbe.h) */ + .maxImageDimension1D = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, + .maxImageDimension2D = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, .maxImageDimension3D = pvr_get_texture_extent_max_z(dev_info), - .maxImageDimensionCube = 4096U, + .maxImageDimensionCube = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, .maxImageArrayLayers = rogue_get_render_size_max_z(dev_info), .maxTexelBufferElements = 64U * 1024U, - .maxUniformBufferRange = 16U * 1024U, + .maxUniformBufferRange = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 64U * 1024U : 16U * 1024U, .maxStorageBufferRange = 128U * 1024U * 1024U, .maxPushConstantsSize = 128U, .maxMemoryAllocationCount = 4096U, @@ -695,7 +704,7 @@ static bool pvr_physical_device_get_properties( .maxDescriptorSetSamplers = 3U * 16U, .maxDescriptorSetUniformBuffers = 3U * 12U, .maxDescriptorSetUniformBuffersDynamic = 8U, - .maxDescriptorSetStorageBuffers = 3U * 4U, + .maxDescriptorSetStorageBuffers = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 16U : 3U * 4U, .maxDescriptorSetStorageBuffersDynamic = 4U, .maxDescriptorSetSampledImages = 3U * 16U, .maxDescriptorSetStorageImages = 3U * 4U, @@ -707,7 +716,8 @@ static bool pvr_physical_device_get_properties( .maxVertexInputAttributeOffset = 2048U - 1U, .maxVertexInputBindingStride = 2048U, /* G9P: its 30 varyings (Zink exposes an eighth of this as GL varyings) */ - .maxVertexOutputComponents = PVR_HAS_FEATURE(dev_info, apple_g9p) ? 120U : 64U, + .maxVertexOutputComponents = PVR_HAS_FEATURE(dev_info, apple_g9p) ? 120U : + PVR_HAS_FEATURE(dev_info, apple_g5p) ? 112U : 64U, /* G5P: 4 * PCO_G5P_MAX_FS_VARYINGS */ /* Tessellation Limits */ /* Requires tessellationShader */ @@ -729,10 +739,11 @@ static bool pvr_physical_device_get_properties( .maxGeometryTotalOutputComponents = 0U, /* Fragment Shader Limits */ - .maxFragmentInputComponents = PVR_HAS_FEATURE(dev_info, apple_g9p) ? 120U : 64U, - .maxFragmentOutputAttachments = 4U, + .maxFragmentInputComponents = PVR_HAS_FEATURE(dev_info, apple_g9p) ? 120U : + PVR_HAS_FEATURE(dev_info, apple_g5p) ? 112U : 64U, /* G5P: 4 * PCO_G5P_MAX_FS_VARYINGS */ + .maxFragmentOutputAttachments = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8U : 4U, /* G5P: 5..8 compact UNORM8 targets */ .maxFragmentDualSrcAttachments = 0U, /* Requires dualSrcBlend */ - .maxFragmentCombinedOutputResources = 4U, + .maxFragmentCombinedOutputResources = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8U : 4U, /* Compute Shader Limits */ .maxComputeSharedMemorySize = 16U * 1024U, @@ -741,10 +752,14 @@ static bool pvr_physical_device_get_properties( [1] = (64U * 1024U) - 1, [2] = (64U * 1024U) - 1, }, - .maxComputeWorkGroupInvocations = 128U, + /* G5P: 256 (AGXMetalA9: 512 per threadgroup, maxThreadsPerThreadgroup; Rogue's CDM: 32 lanes x 16 + * tasks per USC). Barriers above 32 lanes are Apple's software barrier (g5p_barrier.h); a pipeline whose + * temporaries cannot keep every task of a barriered workgroup resident is refused at creation + * (pvr_arch_g5p_compute.c). */ + .maxComputeWorkGroupInvocations = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 256U : 128U, .maxComputeWorkGroupSize = { - [0] = 128U, - [1] = 128U, + [0] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 256U : 128U, + [1] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 256U : 128U, [2] = 64U, }, @@ -759,10 +774,11 @@ static bool pvr_physical_device_get_properties( .maxSamplerAnisotropy = 16.0f, /* Requires samplerAnisotropy */ .maxViewports = 1U, /* Requires multiViewport */ - .maxViewportDimensions[0] = 4096U, - .maxViewportDimensions[1] = 4096U, - .viewportBoundsRange[0] = -8192.0f, - .viewportBoundsRange[1] = 8191.0f, + .maxViewportDimensions[0] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, + .maxViewportDimensions[1] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, + /* at least [-2 * maxViewportDimensions, 2 * maxViewportDimensions - 1] */ + .viewportBoundsRange[0] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? -16384.0f : -8192.0f, + .viewportBoundsRange[1] = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 16383.0f : 8191.0f, .viewportSubPixelBits = 0U, .minMemoryMapAlignment = pdevice->ws->page_size, @@ -782,8 +798,8 @@ static bool pvr_physical_device_get_properties( .maxInterpolationOffset = 0.5f, .subPixelInterpolationOffsetBits = 4U, - .maxFramebufferWidth = 4096U, - .maxFramebufferHeight = 4096U, + .maxFramebufferWidth = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, + .maxFramebufferHeight = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8192U : 4096U, .maxFramebufferLayers = 256U, /* Note: update nir_shader_compiler_options.max_samples when changing this. */ @@ -791,7 +807,7 @@ static bool pvr_physical_device_get_properties( .framebufferDepthSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, .framebufferStencilSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, .framebufferNoAttachmentsSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, - .maxColorAttachments = 4U, + .maxColorAttachments = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 8U : 4U, /* G5P: 5..8 compact UNORM8 targets (pvr_g5p_mrt.h) */ .sampledImageColorSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, .sampledImageIntegerSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, .sampledImageDepthSampleCounts = VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_4_BIT, @@ -802,7 +818,7 @@ static bool pvr_physical_device_get_properties( .timestampPeriod = 0.0f, .maxClipDistances = PVR_MAX_USER_PLANES, - .maxCullDistances = PVR_MAX_USER_PLANES, + .maxCullDistances = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 0U : PVR_MAX_USER_PLANES, /* no shaderCullDistance */ .maxCombinedClipAndCullDistances = PVR_MAX_USER_PLANES, .discreteQueuePriorities = 2U, @@ -822,7 +838,7 @@ static bool pvr_physical_device_get_properties( .nonCoherentAtomSize = 1U, /* Vulkan 1.1 */ - .subgroupSize = ROGUE_MAX_INSTANCES_PER_TASK, + .subgroupSize = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 1U : ROGUE_MAX_INSTANCES_PER_TASK, .subgroupSupportedStages = VK_SHADER_STAGE_COMPUTE_BIT, .subgroupSupportedOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_VOTE_BIT | @@ -962,9 +978,10 @@ static bool pvr_physical_device_get_properties( .integerDotProductAccumulatingSaturating64BitMixedSignednessAccelerated = false, /* Vulkan 1.3 / VK_EXT_subgroup_size_control */ - .minSubgroupSize = ROGUE_MAX_INSTANCES_PER_TASK, - .maxSubgroupSize = ROGUE_MAX_INSTANCES_PER_TASK, - .maxComputeWorkgroupSubgroups = 128U / ROGUE_MAX_INSTANCES_PER_TASK, + .minSubgroupSize = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 1U : ROGUE_MAX_INSTANCES_PER_TASK, + .maxSubgroupSize = PVR_HAS_FEATURE(dev_info, apple_g5p) ? 1U : ROGUE_MAX_INSTANCES_PER_TASK, + .maxComputeWorkgroupSubgroups = PVR_HAS_FEATURE(dev_info, apple_g5p) ? + 256U : 128U / ROGUE_MAX_INSTANCES_PER_TASK, .requiredSubgroupSizeStages = VK_SHADER_STAGE_COMPUTE_BIT, /* Vulkan 1.2 / VK_KHR_timeline_semaphore */ @@ -1157,14 +1174,25 @@ static bool pvr_device_is_conformant(const struct pvr_device_info *info) return false; } -/* Minimum required by the Vulkan spec Limits (maxMemoryAllocationSize) */ -#define PVR_MAX_MEMORY_ALLOCATION_SIZE (1ull << 30) -static inline uint64_t pvr_compute_heap_size(struct pvr_instance *instance) + +static inline uint64_t +pvr_compute_heap_size(const struct pvr_physical_device *pdevice, + struct pvr_instance *instance) { - uint64_t available_ram = - os_get_gpu_heap_size(instance->drirc.misc.heap_memory_percent, NULL); - return MAX2(available_ram, PVR_MAX_MEMORY_ALLOCATION_SIZE); + uint64_t total; + if (!os_get_total_physical_memory(&total)) + return PVR_MAX_MEMORY_ALLOCATION_SIZE; + const float percent = instance->drirc.misc.heap_memory_percent; + if (percent == OS_GPU_HEAP_SIZE_HEURISTIC && + PVR_HAS_FEATURE(&pdevice->dev_info, apple_g5p)) { + uint64_t page_size; + if (!os_get_page_size(&page_size)) + return PVR_MAX_MEMORY_ALLOCATION_SIZE; + /* This device class limits the GPU working set to two thirds of RAM. */ + return ALIGN_POT(total * 2 / 3, page_size); + } + return os_gpu_heap_size_calculate(total, percent, NULL); } static void @@ -1257,7 +1285,7 @@ VkResult pvr_physical_device_init(struct pvr_physical_device *pdevice, /* Setup available memory heaps and types */ pdevice->memory.memoryHeapCount = 1; - pdevice->memory.memoryHeaps[0].size = pvr_compute_heap_size(instance); + pdevice->memory.memoryHeaps[0].size = pvr_compute_heap_size(pdevice, instance); pdevice->memory.memoryHeaps[0].flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; pdevice->memory.memoryTypeCount = 1; @@ -1294,6 +1322,11 @@ VkResult pvr_physical_device_init(struct pvr_physical_device *pdevice, supported_properties.transformFeedbackDraw = false; supported_properties.transformFeedbackQueries = true; supported_extensions.EXT_color_write_enable = false; + /* G5P samples every cube seamlessly and ignores VkSamplerCreateFlags' non-seamless bit; not advertising + * the extension leaves GL's non-seamless cubes to Zink (G9P's 82abaf9be98 ("pvr/g9p: non-seamless cube maps") builds them natively + * on the A10) */ + supported_extensions.EXT_non_seamless_cube_map = false; + supported_features.nonSeamlessCubeMap = false; supported_features.dualSrcBlend = true; supported_properties.maxFragmentDualSrcAttachments = 1; supported_features.colorWriteEnable = false; PVR_G5P_HOOK(release9_features, supported_features) @@ -1356,7 +1389,23 @@ VkResult pvr_physical_device_init(struct pvr_physical_device *pdevice, goto err_vk_physical_device_finish; } - pdevice->vk.supported_sync_types = ws->sync_types; + pdevice->skip_refused_work = PVR_HAS_FEATURE(&pdevice->dev_info, apple_g5p) && + instance->drirc.misc.pvr_skip_refused_work; + + pdevice->sync_type = ws->syncobj_type; + if (ws->job_sync_size) { + pdevice->sync_type.size = MAX2(pdevice->sync_type.size, ws->job_sync_size); + pdevice->native_wait_many = pdevice->sync_type.wait_many; + pdevice->sync_type.wait_many = pvr_sync_wait_many; + } + pdevice->sync_types[0] = &pdevice->sync_type; + pdevice->sync_types[1] = NULL; + pdevice->sync_types[2] = NULL; + if (ws->sync_types[1]) { + pdevice->timeline_sync_type = vk_sync_timeline_get_type(&pdevice->sync_type); + pdevice->sync_types[1] = &pdevice->timeline_sync_type.sync; + } + pdevice->vk.supported_sync_types = pdevice->sync_types; pdevice->pco_ctx = pco_ctx_create(&pdevice->dev_info, &pdevice->dev_runtime_info, NULL); diff --git a/src/imagination/vulkan/pvr_physical_device.h b/src/imagination/vulkan/pvr_physical_device.h index 20f410ba890..d17cd312128 100644 --- a/src/imagination/vulkan/pvr_physical_device.h +++ b/src/imagination/vulkan/pvr_physical_device.h @@ -26,6 +26,8 @@ #include "pvr_device_info.h" #include "pvr_formats.h" +#include "vk_sync.h" +#include "vk_sync_timeline.h" #if defined(VK_USE_PLATFORM_DISPLAY_KHR) || \ defined(VK_USE_PLATFORM_WAYLAND_KHR) || \ @@ -54,6 +56,7 @@ struct pvr_physical_device { char *render_path; char *display_path; bool has_primary; + bool skip_refused_work; /* primary node (cardN) of the render device * Only valid when has_primary is true. @@ -63,6 +66,12 @@ struct pvr_physical_device { dev_t render_devid; struct pvr_winsys *ws; + struct vk_sync_type sync_type; + const struct vk_sync_type *sync_types[3]; + struct vk_sync_timeline_type timeline_sync_type; + VkResult (*native_wait_many)(struct vk_device *, uint32_t, + const struct vk_sync_wait *, + enum vk_sync_wait_flags, uint64_t); struct pvr_device_info dev_info; struct pvr_device_runtime_info dev_runtime_info; diff --git a/src/imagination/vulkan/pvr_tex_state.h b/src/imagination/vulkan/pvr_tex_state.h index fe48892f234..7764ef79824 100644 --- a/src/imagination/vulkan/pvr_tex_state.h +++ b/src/imagination/vulkan/pvr_tex_state.h @@ -48,6 +48,7 @@ /** Parameters for #pvr_pack_tex_state(). */ struct pvr_texture_state_info { + bool skip_rogue_words; VkFormat format; enum pvr_memlayout mem_layout; uint32_t flags; diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm.c b/src/imagination/vulkan/winsys/powervr/pvr_drm.c index c45a5385e62..9231b94c7de 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm.c @@ -42,6 +42,77 @@ #include "vk_alloc.h" #include "vk_drm_syncobj.h" #include "vk_log.h" +#include "util/log.h" +#include "util/u_atomic.h" +#include "util/os_time.h" +#include +#include +#include + +/* Terminal loss diagnostics keep the first failing job's errno. */ +void pvr_drm_report_loss(struct pvr_drm_winsys *ws, const char *job_type, + const char *source, int status) +{ + if (p_atomic_cmpxchg(&ws->loss_reported, 0, 1) == 0) { + ws->base.loss_source = source; + p_atomic_set(&ws->lost, 1); + const char *message = strerror(status < 0 ? -status : status); + mesa_loge("%s: device lost: job=%s %s=%d (%s)", + ws->is_g5p ? "G5P" : "pvr", job_type, source, + status, message); + } +} + +VkResult pvr_drm_submit_jobs(struct pvr_drm_winsys *ws, + struct drm_pvr_ioctl_submit_jobs_args *args, + const char *job_type) +{ + if (drmIoctl(ws->base.render_fd, DRM_IOCTL_PVR_SUBMIT_JOBS, args) == 0) + return VK_SUCCESS; + const int err = errno; + if (err == EIO) + pvr_drm_report_loss(ws, job_type, "SUBMIT_JOBS errno", err); + const char *message = strerror(err); + return vk_errorf(NULL, err == EIO ? VK_ERROR_DEVICE_LOST : VK_ERROR_OUT_OF_DEVICE_MEMORY, + "Failed to submit %s job (errno %d: %s)", job_type, err, message); +} + +static bool pvr_drm_winsys_is_lost(struct pvr_winsys *ws) +{ + struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ws); + return p_atomic_read(&drm_ws->lost); +} + +static VkResult pvr_drm_check_job_fences(struct pvr_winsys *base, + uint32_t count, + const struct vk_sync_wait *waits) +{ + struct pvr_drm_winsys *ws = to_pvr_drm_winsys(base); + for (uint32_t i = 0; i < count; i++) { + /* Job out-syncobjs are binary. A timeline export would not name the + * specific point that was waited, so leave those to the runtime. + */ + if ((waits[i].sync->flags & VK_SYNC_IS_TIMELINE) || + !ws->base.job_sync_size || + waits[i].sync->type->size != ws->base.job_sync_size) + continue; + const char *job_type = ((struct pvr_drm_job_sync *)waits[i].sync)->job_type; + if (!job_type) + continue; + struct vk_drm_syncobj *sync = vk_sync_as_drm_syncobj(waits[i].sync); + int fd = -1; + if (!sync || drmSyncobjExportSyncFile(ws->base.render_fd, sync->syncobj, &fd)) + continue; + struct sync_file_info info = { 0 }; + const int ret = ioctl(fd, SYNC_IOC_FILE_INFO, &info); + close(fd); + if (!ret && info.status < 0) { + pvr_drm_report_loss(ws, job_type, "fence status", info.status); + return VK_ERROR_DEVICE_LOST; + } + } + return VK_SUCCESS; +} static void pvr_drm_finish_heaps(struct pvr_drm_winsys *const drm_ws) { @@ -391,6 +462,8 @@ static void pvr_drm_winsys_get_heaps_info(struct pvr_winsys *ws, static const struct pvr_winsys_ops drm_winsys_ops = { .destroy = pvr_drm_winsys_destroy, + .is_lost = pvr_drm_winsys_is_lost, + .check_job_fences = pvr_drm_check_job_fences, .device_info_init = pvr_drm_winsys_device_info_init, .get_heaps_info = pvr_drm_winsys_get_heaps_info, .buffer_create = pvr_drm_winsys_buffer_create, @@ -723,6 +796,15 @@ VkResult pvr_drm_winsys_create(const int render_fd, drm_ws->bvnc = gpu_info.gpu_id; + struct pvr_device_info info; + if (!pvr_device_info_init(&info, drm_ws->bvnc)) { + drm_ws->is_g5p = PVR_HAS_FEATURE(&info, apple_g5p); + drm_ws->job_fence_errors = drm_ws->is_g5p || + PVR_HAS_FEATURE(&info, apple_g9p); + drm_ws->base.job_sync_size = drm_ws->job_fence_errors + ? sizeof(struct pvr_drm_job_sync) : 0; + } + result = pvr_ioctl(render_fd, DRM_IOCTL_PVR_CREATE_VM_CONTEXT, &create_vm_context_args, diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm.h b/src/imagination/vulkan/winsys/powervr/pvr_drm.h index c8b1aa0b8d8..2e5379f26fb 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm.h +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm.h @@ -29,10 +29,17 @@ #include #include "pvr_winsys.h" +#include "vk_drm_syncobj.h" #include "util/macros.h" #include "util/rwlock.h" #include "util/sparse_array.h" +/* Identify job out-fences without inspecting ordinary poll waits. */ +struct pvr_drm_job_sync { + struct vk_drm_syncobj base; + const char *job_type; +}; + struct pvr_drm_winsys_heap { struct pvr_winsys_heap base; }; @@ -43,6 +50,15 @@ struct pvr_drm_winsys { /* Packed bvnc */ uint64_t bvnc; + bool is_g5p; + /* The kernel sets a job fence's error when it resets (pvr_apple.c); + * upstream pvr_queue.c does not. + */ + bool job_fence_errors; + int loss_reported; + int lost; /* Atomic and terminal. */ + int unmap_loss_reported; + /* Protects imported BOs creation/freeing, which includes the bo_map member * below. */ @@ -85,6 +101,21 @@ struct pvr_drm_winsys { uint32_t vm_context; }; +struct drm_pvr_ioctl_submit_jobs_args; +VkResult pvr_drm_submit_jobs(struct pvr_drm_winsys *ws, + struct drm_pvr_ioctl_submit_jobs_args *args, + const char *job_type); +void pvr_drm_report_loss(struct pvr_drm_winsys *ws, const char *job_type, + const char *source, int status); + +static inline void pvr_drm_tag_job_sync(struct pvr_drm_winsys *ws, + struct vk_sync *sync, const char *job_type) +{ + if (ws->job_fence_errors && sync && ws->base.job_sync_size && + sync->type->size == ws->base.job_sync_size) + ((struct pvr_drm_job_sync *)sync)->job_type = job_type; +} + /******************************************* helper macros *******************************************/ diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c index 04f8309f04a..75ad551a814 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c @@ -36,6 +36,7 @@ #include "drm-uapi/pvr_drm.h" #include "pvr_drm.h" +#include "pvr_debug.h" #include "pvr_drm_bo.h" #include "pvr_macros.h" #include "pvr_winsys_helper.h" @@ -43,6 +44,7 @@ #include "util/macros.h" #include "vk_alloc.h" #include "vk_log.h" +#include "util/log.h" static VkResult pvr_drm_create_gem_bo(struct pvr_drm_winsys *drm_ws, uint32_t drm_flags, @@ -149,6 +151,9 @@ static void pvr_drm_buffer_release(struct pvr_drm_winsys_bo *drm_bo) if (p_atomic_dec_return(&drm_bo->ref_count) == 0) { uint32_t handle = drm_bo->handle; + if (PVR_IS_DEBUG_SET(BO_LOG) && drm_bo->count_kind < 2) + pvr_winsys_count_bo(drm_bo->count_kind, -(int64_t)drm_bo->base.size); + /* Our BO structs are stored in a sparse array in the winsys structure, * so we don't want to free the BO pointer, instead we want to reset it * to 0, to signal that array entry as being free. @@ -193,6 +198,10 @@ pvr_drm_display_buffer_create(struct pvr_drm_winsys *drm_ws, assert((*bo_out)->size >= size); + to_pvr_drm_winsys_bo(*bo_out)->count_kind = 1; + if (PVR_IS_DEBUG_SET(BO_LOG)) + pvr_winsys_count_bo(1, (*bo_out)->size); + return VK_SUCCESS; } @@ -250,6 +259,9 @@ VkResult pvr_drm_winsys_buffer_create(struct pvr_winsys *ws, drm_bo->base.ws = ws; drm_bo->handle = handle; drm_bo->flags = drm_flags; + drm_bo->count_kind = 0; + if (PVR_IS_DEBUG_SET(BO_LOG)) + pvr_winsys_count_bo(0, size); p_atomic_set(&drm_bo->ref_count, 1); g9p_bolog('+', handle, size); @@ -294,6 +306,7 @@ pvr_drm_winsys_buffer_create_from_fd(struct pvr_winsys *ws, drm_bo->base.ws = ws; drm_bo->base.size = (uint64_t)size; drm_bo->base.is_imported = true; + drm_bo->count_kind = 2; drm_bo->handle = handle; p_atomic_set(&drm_bo->ref_count, 1); @@ -525,12 +538,14 @@ VkResult pvr_drm_winsys_vma_map(struct pvr_winsys_vma *vma, return vk_error(NULL, VK_ERROR_MEMORY_MAP_FAILED); } - result = pvr_ioctl(drm_ws->base.render_fd, - DRM_IOCTL_PVR_VM_MAP, - &args, - VK_ERROR_MEMORY_MAP_FAILED); - if (result != VK_SUCCESS) - return result; + if (drmIoctl(drm_ws->base.render_fd, DRM_IOCTL_PVR_VM_MAP, &args)) { + const int err = errno; + const char *message = strerror(err); + return vk_errorf(NULL, + err == EIO ? VK_ERROR_OUT_OF_HOST_MEMORY + : VK_ERROR_OUT_OF_DEVICE_MEMORY, + "DRM_IOCTL_PVR_VM_MAP failed (errno %d: %s)", err, message); + } pvr_drm_buffer_acquire(drm_bo); @@ -558,11 +573,21 @@ void pvr_drm_winsys_vma_unmap(struct pvr_winsys_vma *vma) /* Address should be mapped. */ assert(vma->bo); - pvr_ioctlf(drm_ws->base.render_fd, - DRM_IOCTL_PVR_VM_UNMAP, - &args, - VK_ERROR_UNKNOWN, - "Unmap failed"); + if (p_atomic_read(&drm_ws->lost)) { + if (drmIoctl(drm_ws->base.render_fd, DRM_IOCTL_PVR_VM_UNMAP, &args) && + p_atomic_cmpxchg(&drm_ws->unmap_loss_reported, 0, 1) == 0) { + const int err = errno; + const char *message = strerror(err); + mesa_logw("%s: VM_UNMAP after loss ignored (errno %d: %s)", + drm_ws->is_g5p ? "G5P" : "pvr", err, message); + } + } else { + pvr_ioctlf(drm_ws->base.render_fd, + DRM_IOCTL_PVR_VM_UNMAP, + &args, + VK_ERROR_UNKNOWN, + "Unmap failed"); + } pvr_drm_buffer_release(drm_bo); diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h index b7879b17354..f102325df8a 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h @@ -40,6 +40,8 @@ struct pvr_drm_winsys_bo { struct pvr_winsys_bo base; uint32_t ref_count; + /* 0: CREATE_BO, 1: display, 2: plain import (not counted). */ + uint8_t count_kind; uint32_t handle; uint64_t flags; }; diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c index b25c66f28dc..98990b7cf18 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_compute.c @@ -22,6 +22,9 @@ */ #include +#include +#include +#include "pvr_g5p_compute.h" #include #include #include @@ -45,6 +48,7 @@ struct pvr_drm_winsys_compute_ctx { /* Handle to kernel context. */ uint32_t handle; + uint64_t submit_seq; }; #define to_pvr_drm_winsys_compute_ctx(ctx) \ @@ -122,6 +126,7 @@ VkResult pvr_drm_winsys_compute_ctx_create( drm_ctx->base.ws = ws; drm_ctx->handle = ctx_args.handle; + drm_ctx->submit_seq = 0; *ctx_out = &drm_ctx->base; @@ -138,7 +143,7 @@ void pvr_drm_winsys_compute_ctx_destroy(struct pvr_winsys_compute_ctx *ctx) { struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); struct pvr_drm_winsys_compute_ctx *drm_ctx = - to_pvr_drm_winsys_compute_ctx(ctx); + to_pvr_drm_winsys_compute_ctx((struct pvr_winsys_compute_ctx *)ctx); struct drm_pvr_ioctl_destroy_context_args args = { .handle = drm_ctx->handle, }; @@ -172,9 +177,9 @@ VkResult pvr_drm_winsys_compute_submit( UNUSED const struct pvr_device_info *dev_info, struct vk_sync *signal_sync) { - const struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); - const struct pvr_drm_winsys_compute_ctx *drm_ctx = - to_pvr_drm_winsys_compute_ctx(ctx); + struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); + struct pvr_drm_winsys_compute_ctx *drm_ctx = + to_pvr_drm_winsys_compute_ctx((struct pvr_winsys_compute_ctx *)ctx); struct drm_pvr_sync_op sync_ops[2]; struct drm_pvr_job job_args = { @@ -215,9 +220,25 @@ VkResult pvr_drm_winsys_compute_submit( } /* Returns VK_ERROR_OUT_OF_DEVICE_MEMORY to match pvrsrv. */ - return pvr_ioctlf(drm_ws->base.render_fd, - DRM_IOCTL_PVR_SUBMIT_JOBS, - &args, - VK_ERROR_OUT_OF_DEVICE_MEMORY, - "Failed to submit compute job"); + const VkResult result = pvr_drm_submit_jobs(drm_ws, &args, "compute"); + if (result == VK_SUCCESS) + pvr_drm_tag_job_sync(drm_ws, signal_sync, "compute"); + if (drm_ws->is_g5p && result == VK_SUCCESS) { + const uint64_t seq = ++drm_ctx->submit_seq; + if (submit_info->g5p_log) { + flockfile(stderr); + fprintf(stderr, "G5P compute ctx=%u submit=%" PRIu64 " out_syncobj=%u dispatches=%u\n", + drm_ctx->handle, seq, + signal_sync ? vk_sync_as_drm_syncobj(signal_sync)->syncobj : 0, + submit_info->g5p_log_count); + unsigned i = 0; + for (const struct pvr_g5p_dispatch_log *l = submit_info->g5p_log; l; l = l->next, i++) { + fprintf(stderr, "G5P compute #%u %s\n", i, l->line); + if (l->stub) + fprintf(stderr, "G5P compute omitted pipe=%s\n", l->pipeline); + } + funlockfile(stderr); + } + } + return result; } diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_null.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_null.c index 4c93d183de3..d4b70229850 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_null.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_null.c @@ -44,6 +44,8 @@ VkResult pvr_drm_winsys_null_job_submit(struct pvr_winsys *ws, struct vk_sync_signal *signal_sync) { const struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ws); + pvr_drm_tag_job_sync(to_pvr_drm_winsys(ws), signal_sync->sync, + "queue completion (geometry/fragment/compute)"); uint32_t tmp_syncobj; VkResult result; int ret; diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c index 0c543a3eeb4..ddf43186e42 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_render.c @@ -474,7 +474,7 @@ VkResult pvr_drm_winsys_render_submit( struct vk_sync *signal_sync_frag) { - const struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); + struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); const struct pvr_drm_winsys_render_ctx *drm_ctx = to_pvr_drm_winsys_render_ctx(ctx); const struct pvr_winsys_geometry_state *const geom_state = @@ -643,12 +643,12 @@ VkResult pvr_drm_winsys_render_submit( flock(lock_fd, LOCK_EX); /* Returns VK_ERROR_OUT_OF_DEVICE_MEMORY to match pvrsrv. */ - const VkResult result = pvr_ioctlf(drm_ws->base.render_fd, - DRM_IOCTL_PVR_SUBMIT_JOBS, - &args, - VK_ERROR_OUT_OF_DEVICE_MEMORY, - "Failed to submit render job"); + const VkResult result = pvr_drm_submit_jobs(drm_ws, &args, "render"); + if (result == VK_SUCCESS) { + pvr_drm_tag_job_sync(drm_ws, signal_sync_geom, "geometry"); + pvr_drm_tag_job_sync(drm_ws, signal_sync_frag, "fragment"); + } if (lock_fd >= 0) flock(lock_fd, LOCK_UN); return result; diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c index 9ae0f64dade..9e31fdaa504 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_job_transfer.c @@ -128,7 +128,7 @@ VkResult pvr_drm_winsys_transfer_submit( UNUSED const struct pvr_device_info *const dev_info, struct vk_sync *signal_sync) { - const struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); + struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ctx->ws); const struct pvr_drm_winsys_transfer_ctx *drm_ctx = to_pvr_drm_winsys_transfer_ctx(ctx); @@ -172,9 +172,8 @@ VkResult pvr_drm_winsys_transfer_submit( } /* Returns VK_ERROR_OUT_OF_DEVICE_MEMORY to match pvrsrv. */ - return pvr_ioctlf(drm_ws->base.render_fd, - DRM_IOCTL_PVR_SUBMIT_JOBS, - &args, - VK_ERROR_OUT_OF_DEVICE_MEMORY, - "Failed to submit transfer job"); + const VkResult result = pvr_drm_submit_jobs(drm_ws, &args, "transfer"); + if (result == VK_SUCCESS) + pvr_drm_tag_job_sync(drm_ws, signal_sync, "transfer"); + return result; } diff --git a/src/imagination/vulkan/winsys/pvr_winsys.c b/src/imagination/vulkan/winsys/pvr_winsys.c index 64c5c76b175..fb4387576e7 100644 --- a/src/imagination/vulkan/winsys/pvr_winsys.c +++ b/src/imagination/vulkan/winsys/pvr_winsys.c @@ -29,11 +29,27 @@ #include "powervr/pvr_drm_public.h" #include "pvr_winsys.h" #include "vk_log.h" +#include "util/u_atomic.h" #if defined(PVR_SUPPORT_SERVICES_DRIVER) # include "pvrsrvkm/pvr_srv_public.h" #endif +/* The kernel charges CREATE_BO memory per process, across winsys instances. */ +static int64_t counted_bytes[2]; + +void pvr_winsys_count_bo(int kind, int64_t delta) +{ + assert(kind >= 0 && kind < ARRAY_SIZE(counted_bytes)); + p_atomic_add(&counted_bytes[kind], delta); +} + +uint64_t pvr_winsys_counted_bytes(int kind) +{ + assert(kind >= 0 && kind < ARRAY_SIZE(counted_bytes)); + return p_atomic_read(&counted_bytes[kind]); +} + void pvr_winsys_destroy(struct pvr_winsys *ws) { const int display_fd = ws->display_fd; diff --git a/src/imagination/vulkan/winsys/pvr_winsys.h b/src/imagination/vulkan/winsys/pvr_winsys.h index 00a9bdb48e7..5202e8d4630 100644 --- a/src/imagination/vulkan/winsys/pvr_winsys.h +++ b/src/imagination/vulkan/winsys/pvr_winsys.h @@ -292,7 +292,11 @@ struct pvr_winsys_transfer_submit_info { struct pvr_winsys_transfer_cmd cmds[PVR_TRANSFER_MAX_PREPARES_PER_SUBMIT]; }; +struct pvr_g5p_dispatch_log; + struct pvr_winsys_compute_submit_info { + const struct pvr_g5p_dispatch_log *g5p_log; + unsigned g5p_log_count; uint32_t frame_num; uint32_t job_num; @@ -356,8 +360,17 @@ struct pvr_winsys_render_submit_info { } fragment, fragment_pr; }; +static inline bool pvr_winsys_never_lost(struct pvr_winsys *ws) +{ + (void)ws; + return false; +} + struct pvr_winsys_ops { void (*destroy)(struct pvr_winsys *ws); + bool (*is_lost)(struct pvr_winsys *ws); + VkResult (*check_job_fences)(struct pvr_winsys *ws, uint32_t count, + const struct vk_sync_wait *waits); VkResult (*device_info_init)(struct pvr_winsys *ws, struct pvr_device_info *dev_info, struct pvr_device_runtime_info *runtime_info); @@ -459,6 +472,8 @@ struct pvr_winsys { uint32_t log2_page_size; const struct vk_sync_type *sync_types[3]; + const char *loss_source; + uint32_t job_sync_size; struct vk_sync_type syncobj_type; struct vk_sync_timeline_type timeline_syncobj_type; @@ -474,6 +489,9 @@ struct pvr_winsys { const struct pvr_winsys_ops *ops; }; +void pvr_winsys_count_bo(int kind, int64_t delta); +uint64_t pvr_winsys_counted_bytes(int kind); + void pvr_winsys_destroy(struct pvr_winsys *ws); VkResult pvr_winsys_create(const char *render_path, const char *display_path, diff --git a/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c b/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c index 341f07e05e8..68ab52d3222 100644 --- a/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c +++ b/src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv.c @@ -649,6 +649,7 @@ PER_ARCH_FUNCS(rogue); #define SRV_WINSYS(arch) \ static const struct pvr_winsys_ops pvr_##arch##_srv_winsys_ops = { \ + .is_lost = pvr_winsys_never_lost, \ .destroy = pvr_srv_winsys_destroy, \ .device_info_init = pvr_srv_winsys_device_info_init, \ .get_heaps_info = pvr_srv_winsys_get_heaps_info, \ @@ -823,7 +824,7 @@ static VkResult pvr_srv_create_presignaled_sync(struct pvr_device *device, goto err_close_sw_fence; result = vk_sync_create(&device->vk, - &device->pdevice->ws->syncobj_type, + &device->pdevice->sync_type, 0U, 0UL, &sync); diff --git a/src/mesa/glapi/glapi/gen/EXT_external_objects.xml b/src/mesa/glapi/glapi/gen/EXT_external_objects.xml index 297b5f4e7b4..c93bc5bb244 100644 --- a/src/mesa/glapi/glapi/gen/EXT_external_objects.xml +++ b/src/mesa/glapi/glapi/gen/EXT_external_objects.xml @@ -26,45 +26,45 @@ - + - + - + - + - + - + - + - + @@ -74,7 +74,7 @@ - + @@ -85,7 +85,7 @@ - + @@ -96,7 +96,7 @@ - + @@ -108,14 +108,14 @@ - + - + @@ -125,7 +125,7 @@ - + @@ -136,7 +136,7 @@ - + @@ -147,7 +147,7 @@ - + @@ -159,7 +159,7 @@ - + @@ -184,34 +184,34 @@ - + - + - + - + - + - + @@ -220,7 +220,7 @@ - + diff --git a/src/mesa/glapi/glapi/gen/EXT_external_objects_fd.xml b/src/mesa/glapi/glapi/gen/EXT_external_objects_fd.xml index 6eaa3f5b0cc..2bb1ed03fff 100644 --- a/src/mesa/glapi/glapi/gen/EXT_external_objects_fd.xml +++ b/src/mesa/glapi/glapi/gen/EXT_external_objects_fd.xml @@ -10,14 +10,14 @@ - + - + diff --git a/src/util/texcompress_astc.cpp b/src/util/texcompress_astc.cpp index a413f5743e3..2e31574cf58 100644 --- a/src/util/texcompress_astc.cpp +++ b/src/util/texcompress_astc.cpp @@ -54,6 +54,7 @@ class decode_error reserved_block_mode_2, dual_plane_and_too_many_partitions, invalid_range_in_void_extent, + reserved_void_extent_bits, weight_grid_exceeds_block_size, invalid_colour_endpoints_size, invalid_colour_endpoints_count, @@ -692,6 +693,13 @@ decode_error::type Block::decode_void_extent(InputBitVector block) return decode_error::unsupported_hdr_void_extent; } + /* The Khronos Data Format specification's ASTC void-extent block requires + * reserved bits [11:10] to be 11. Other values decode to the error colour, + * matching astcenc's error-block handling. */ + if (block.get_bits(10, 2) != 0x3) { + return decode_error::reserved_void_extent_bits; + } + if (void_extent_min_s == 0x1fff && void_extent_max_s == 0x1fff && void_extent_min_t == 0x1fff && void_extent_max_t == 0x1fff) { diff --git a/src/vulkan/runtime/vk_queue.c b/src/vulkan/runtime/vk_queue.c index c26ae4e5efd..f6916a14b7d 100644 --- a/src/vulkan/runtime/vk_queue.c +++ b/src/vulkan/runtime/vk_queue.c @@ -828,7 +828,11 @@ vk_queue_stop_submit_thread(struct vk_queue *queue) thrd_join(queue->submit.thread, NULL); - assert(list_is_empty(&queue->submit.submits)); + /* A failed submit may remain after queue loss. vk_queue_finish destroys + * pending submits after the worker has exited. + */ + assert(list_is_empty(&queue->submit.submits) || + vk_device_is_lost_no_report(queue->base.device)); queue->submit.mode = VK_QUEUE_SUBMIT_MODE_IMMEDIATE; } diff --git a/src/vulkan/wsi/wsi_common_wayland.c b/src/vulkan/wsi/wsi_common_wayland.c index 8a1e86f432e..a742c8474ba 100644 --- a/src/vulkan/wsi/wsi_common_wayland.c +++ b/src/vulkan/wsi/wsi_common_wayland.c @@ -38,6 +38,7 @@ #include "vk_instance.h" #include "vk_device.h" +#include "vk_format.h" #include "vk_physical_device.h" #include "vk_util.h" #include "wsi_common_entrypoints.h" @@ -99,12 +100,78 @@ struct dmabuf_feedback { struct dmabuf_feedback_tranche pending_tranche; }; +struct wsi_wl_output { + struct wl_list link; + struct wl_output *output; + uint32_t name; + enum wl_output_transform transform; +}; + +/* Vulkan VkSurfaceTransformFlagBitsKHR defines clockwise rotations, while + * wayland.xml wl_output.transform defines counter-clockwise rotations. Both + * VkSwapchainCreateInfoKHR.preTransform and wl_surface.set_buffer_transform + * describe the transform already applied to the image content. + * https://docs.vulkan.org/refpages/latest/refpages/source/VkSurfaceTransformFlagBitsKHR.html + * https://gitlab.freedesktop.org/wayland/wayland/-/blob/main/protocol/wayland.xml + */ +static VkSurfaceTransformFlagBitsKHR +wl_transform_to_vk(enum wl_output_transform transform) +{ + switch (transform) { + case WL_OUTPUT_TRANSFORM_NORMAL: + return VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; + case WL_OUTPUT_TRANSFORM_90: + return VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR; + case WL_OUTPUT_TRANSFORM_180: + return VK_SURFACE_TRANSFORM_ROTATE_180_BIT_KHR; + case WL_OUTPUT_TRANSFORM_270: + return VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR; + case WL_OUTPUT_TRANSFORM_FLIPPED: + return VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_BIT_KHR; + case WL_OUTPUT_TRANSFORM_FLIPPED_90: + return VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_270_BIT_KHR; + case WL_OUTPUT_TRANSFORM_FLIPPED_180: + return VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_180_BIT_KHR; + case WL_OUTPUT_TRANSFORM_FLIPPED_270: + return VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_90_BIT_KHR; + default: + UNREACHABLE("Invalid Wayland transform"); + } +} + +static enum wl_output_transform +vk_transform_to_wl(VkSurfaceTransformFlagBitsKHR transform) +{ + switch (transform) { + case VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR: + return WL_OUTPUT_TRANSFORM_NORMAL; + case VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR: + return WL_OUTPUT_TRANSFORM_90; + case VK_SURFACE_TRANSFORM_ROTATE_180_BIT_KHR: + return WL_OUTPUT_TRANSFORM_180; + case VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR: + return WL_OUTPUT_TRANSFORM_270; + case VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_BIT_KHR: + return WL_OUTPUT_TRANSFORM_FLIPPED; + case VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_270_BIT_KHR: + return WL_OUTPUT_TRANSFORM_FLIPPED_90; + case VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_180_BIT_KHR: + return WL_OUTPUT_TRANSFORM_FLIPPED_180; + case VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_90_BIT_KHR: + return WL_OUTPUT_TRANSFORM_FLIPPED_270; + default: + UNREACHABLE("Invalid Vulkan surface transform"); + } +} + struct wsi_wl_display { /* The real wl_display */ struct wl_display *wl_display; /* Actually a proxy wrapper around the event queue */ struct wl_display *wl_display_wrapper; struct wl_event_queue *queue; + struct wl_registry *registry; + struct wl_list outputs; struct wl_fixes *wl_fixes; struct wl_shm *wl_shm; @@ -179,6 +246,7 @@ struct wsi_wl_surface { VkIcdSurfaceWayland base; unsigned int chain_count; + enum wl_output_transform buffer_transform; struct wsi_wl_swapchain *chain; struct loader_wayland_surface wayland_surface; @@ -201,6 +269,7 @@ struct wsi_wl_swapchain { struct wl_callback *frame; VkExtent2D extent; + enum wl_output_transform buffer_transform; VkFormat vk_format; enum wsi_wl_buffer_type buffer_type; uint32_t drm_format; @@ -1497,12 +1566,80 @@ static const struct wp_presentation_listener presentation_listener = { #define MESA_WL_FIXES_VERSION 1 #endif +static void +output_handle_geometry(void *data, struct wl_output *output, + int32_t x, int32_t y, int32_t physical_width, + int32_t physical_height, int32_t subpixel, + const char *make, const char *model, int32_t transform) +{ + struct wsi_wl_output *wl_output = data; + wl_output->transform = transform; +} + +static void +output_handle_mode(void *data, struct wl_output *output, uint32_t flags, + int32_t width, int32_t height, int32_t refresh) +{ +} + +static void +output_handle_done(void *data, struct wl_output *output) +{ +} + +static void +output_handle_scale(void *data, struct wl_output *output, int32_t factor) +{ +} + +static void +output_handle_name(void *data, struct wl_output *output, const char *name) +{ +} + +static void +output_handle_description(void *data, struct wl_output *output, + const char *description) +{ +} + +static const struct wl_output_listener output_listener = { + .geometry = output_handle_geometry, + .mode = output_handle_mode, + .done = output_handle_done, + .scale = output_handle_scale, + .name = output_handle_name, + .description = output_handle_description, +}; + +static void +wsi_wl_output_destroy(struct wsi_wl_output *output) +{ + wl_list_remove(&output->link); + if (wl_output_get_version(output->output) >= WL_OUTPUT_RELEASE_SINCE_VERSION) + wl_output_release(output->output); + else + wl_output_destroy(output->output); + free(output); +} + static void registry_handle_global(void *data, struct wl_registry *registry, uint32_t name, const char *interface, uint32_t version) { struct wsi_wl_display *display = data; + if (strcmp(interface, wl_output_interface.name) == 0 && version >= 2) { + struct wsi_wl_output *output = calloc(1, sizeof(*output)); + if (!output) + return; + output->name = name; + output->output = wl_registry_bind(registry, name, &wl_output_interface, + MIN2(version, 4)); + wl_output_add_listener(output->output, &output_listener, output); + wl_list_insert(&display->outputs, &output->link); + } + if (display->sw) { if (strcmp(interface, wl_shm_interface.name) == 0) { display->wl_shm = wl_registry_bind(registry, name, &wl_shm_interface, 1); @@ -1564,9 +1701,14 @@ static void registry_handle_global_remove(void *data, struct wl_registry *registry, uint32_t name) { -#ifdef WL_FIXES_ACK_GLOBAL_REMOVE struct wsi_wl_display *display = data; + struct wsi_wl_output *output, *tmp; + wl_list_for_each_safe(output, tmp, &display->outputs, link) { + if (output->name == name) + wsi_wl_output_destroy(output); + } +#ifdef WL_FIXES_ACK_GLOBAL_REMOVE if (display->wl_fixes && wl_fixes_get_version(display->wl_fixes) >= WL_FIXES_ACK_GLOBAL_REMOVE_SINCE_VERSION) { wl_fixes_ack_global_remove(display->wl_fixes, registry, name); } @@ -1581,6 +1723,18 @@ static const struct wl_registry_listener registry_listener = { static void wsi_wl_display_finish(struct wsi_wl_display *display) { + struct wsi_wl_output *output, *tmp; + wl_list_for_each_safe(output, tmp, &display->outputs, link) + wsi_wl_output_destroy(output); + + if (display->registry) { +#ifdef WL_FIXES_INTERFACE + if (display->wl_fixes) + wl_fixes_destroy_registry(display->wl_fixes, display->registry); +#endif + wl_registry_destroy(display->registry); + } + struct wsi_wl_format *f; u_vector_foreach(f, &display->formats) u_vector_finish(&f->modifiers); @@ -1624,6 +1778,7 @@ wsi_wl_display_init(struct wsi_wayland *wsi_wl, { VkResult result = VK_SUCCESS; memset(display, 0, sizeof(*display)); + wl_list_init(&display->outputs); if (!u_vector_init(&display->formats, 8, sizeof(struct wsi_wl_format))) return VK_ERROR_OUT_OF_HOST_MEMORY; @@ -1657,6 +1812,7 @@ wsi_wl_display_init(struct wsi_wayland *wsi_wl, goto fail; } + display->registry = registry; wl_registry_add_listener(registry, ®istry_listener, display); /* Round-trip to get wl_shm and zwp_linux_dmabuf_v1 globals */ @@ -1739,27 +1895,8 @@ wsi_wl_display_init(struct wsi_wayland *wsi_wl, goto fail; } - if (wsi_wl->wsi->force_bgra8_unorm_first) { - /* Find BGRA8_UNORM in the list and swap it to the first position if we - * can find it. Some apps get confused if SRGB is first in the list. - */ - struct wsi_wl_format *first_fmt = u_vector_tail(&display->formats); - struct wsi_wl_format *f, tmp_fmt; - f = find_format(&display->formats, VK_FORMAT_B8G8R8A8_UNORM); - if (f) { - tmp_fmt = *f; - *f = *first_fmt; - *first_fmt = tmp_fmt; - } - } - out: - /* We don't need this anymore */ -#ifdef WL_FIXES_INTERFACE - if (display->wl_fixes) - wl_fixes_destroy_registry(display->wl_fixes, registry); -#endif - wl_registry_destroy(registry); + /* Keep the registry alive to track output additions and removals. */ /* Destroy default dma-buf feedback object and format table */ if (display->wl_dmabuf_feedback) { @@ -1771,14 +1908,6 @@ wsi_wl_display_init(struct wsi_wayland *wsi_wl, return VK_SUCCESS; fail_registry: - if (registry) { -#ifdef WL_FIXES_INTERFACE - if (display->wl_fixes) - wl_fixes_destroy_registry(display->wl_fixes, registry); -#endif - wl_registry_destroy(registry); - } - fail: wsi_wl_display_finish(display); return result; @@ -1921,6 +2050,30 @@ wsi_wl_surface_get_capabilities(VkIcdSurfaceBase *icd_surface, caps->surfaceCapabilities.minImageCount = wsi_wl_surface_get_min_image_count(display, present_mode); + caps->surfaceCapabilities.supportedTransforms = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; + caps->surfaceCapabilities.currentTransform = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; + if (wl_surface_get_version(surface->surface) >= + WL_SURFACE_SET_BUFFER_TRANSFORM_SINCE_VERSION) { + caps->surfaceCapabilities.supportedTransforms |= + VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR | + VK_SURFACE_TRANSFORM_ROTATE_180_BIT_KHR | + VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR | + VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_BIT_KHR | + VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_90_BIT_KHR | + VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_180_BIT_KHR | + VK_SURFACE_TRANSFORM_HORIZONTAL_MIRROR_ROTATE_270_BIT_KHR; + + /* Without the application's surface enter/leave events, only a single + * output tells us which transform applies. Otherwise prefer identity. + */ + if (wl_list_length(&display->outputs) == 1) { + struct wsi_wl_output *output = + wl_container_of(display->outputs.next, output, link); + caps->surfaceCapabilities.currentTransform = + wl_transform_to_vk(output->transform); + } + } + if (!wsi_wl_surface->display) wsi_wl_display_finish(&temp_display); @@ -1934,8 +2087,6 @@ wsi_wl_surface_get_capabilities(VkIcdSurfaceBase *icd_surface, wsi_device->maxImageDimension2D, }; - caps->surfaceCapabilities.supportedTransforms = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; - caps->surfaceCapabilities.currentTransform = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; caps->surfaceCapabilities.maxImageArrayLayers = 1; caps->surfaceCapabilities.supportedCompositeAlpha = @@ -2223,6 +2374,43 @@ wsi_wl_surface_get_capabilities2(VkIcdSurfaceBase *surface, return result; } +static void +wsi_wl_surface_sort_formats(struct wsi_wl_display *display, + const struct wsi_device *wsi_device, + struct wsi_wl_format **sorted_formats) +{ + unsigned count = 0; + + /* Prefer 8-bit color channels, preserving the compositor's order within + * each group. Higher precision formats can cost more memory and may not + * be supported by display planes. + */ + for (unsigned pass = 0; pass < 2; pass++) { + struct wsi_wl_format *disp_fmt; + u_vector_foreach(disp_fmt, &display->formats) { + VkFormat format = disp_fmt->vk_format; + bool is_8bit = + vk_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, 0) == 8 && + vk_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, 1) == 8 && + vk_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, 2) == 8; + if (is_8bit == (pass == 0)) + sorted_formats[count++] = disp_fmt; + } + } + + if (wsi_device->force_bgra8_unorm_first) { + for (unsigned i = 0; i < count; i++) { + if (sorted_formats[i]->vk_format == VK_FORMAT_B8G8R8A8_UNORM) { + struct wsi_wl_format *format = sorted_formats[i]; + memmove(&sorted_formats[1], &sorted_formats[0], + i * sizeof(*sorted_formats)); + sorted_formats[0] = format; + break; + } + } + } +} + static VkResult wsi_wl_surface_get_formats(VkIcdSurfaceBase *icd_surface, struct wsi_device *wsi_device, @@ -2240,13 +2428,21 @@ wsi_wl_surface_get_formats(VkIcdSurfaceBase *icd_surface, wsi_device->sw, "mesa formats query")) return VK_ERROR_SURFACE_LOST_KHR; + unsigned format_count = u_vector_length(&display.formats); + STACK_ARRAY(struct wsi_wl_format *, sorted_formats, format_count); + if (!sorted_formats) { + wsi_wl_display_finish(&display); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + wsi_wl_surface_sort_formats(&display, wsi_device, sorted_formats); + VK_OUTARRAY_MAKE_TYPED(VkSurfaceFormatKHR, out, pSurfaceFormats, pSurfaceFormatCount); VkColorSpaceKHR *cs; u_vector_foreach(cs, &display.colorspaces) { - struct wsi_wl_format *disp_fmt; - u_vector_foreach(disp_fmt, &display.formats) { + for (unsigned i = 0; i < format_count; i++) { + struct wsi_wl_format *disp_fmt = sorted_formats[i]; /* Skip formats for which we can't support both alpha & opaque * formats. */ @@ -2262,6 +2458,7 @@ wsi_wl_surface_get_formats(VkIcdSurfaceBase *icd_surface, } } + STACK_ARRAY_FINISH(sorted_formats); wsi_wl_display_finish(&display); return vk_outarray_status(&out); @@ -2285,13 +2482,21 @@ wsi_wl_surface_get_formats2(VkIcdSurfaceBase *icd_surface, wsi_device->sw, "mesa formats2 query")) return VK_ERROR_SURFACE_LOST_KHR; + unsigned format_count = u_vector_length(&display.formats); + STACK_ARRAY(struct wsi_wl_format *, sorted_formats, format_count); + if (!sorted_formats) { + wsi_wl_display_finish(&display); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + wsi_wl_surface_sort_formats(&display, wsi_device, sorted_formats); + VK_OUTARRAY_MAKE_TYPED(VkSurfaceFormat2KHR, out, pSurfaceFormats, pSurfaceFormatCount); VkColorSpaceKHR *cs; u_vector_foreach(cs, &display.colorspaces) { - struct wsi_wl_format *disp_fmt; - u_vector_foreach(disp_fmt, &display.formats) { + for (unsigned i = 0; i < format_count; i++) { + struct wsi_wl_format *disp_fmt = sorted_formats[i]; /* Skip formats for which we can't support both alpha & opaque * formats. */ @@ -2307,6 +2512,7 @@ wsi_wl_surface_get_formats2(VkIcdSurfaceBase *icd_surface, } } + STACK_ARRAY_FINISH(sorted_formats); wsi_wl_display_finish(&display); return vk_outarray_status(&out); @@ -3570,6 +3776,11 @@ wsi_wl_swapchain_queue_present(struct wsi_swapchain *wsi_chain, wp_fifo_v1_wait_barrier(chain->fifo); chain->next_present_force_wait_barrier = false; } + if (chain->buffer_transform != wsi_wl_surface->buffer_transform) { + wl_surface_set_buffer_transform(wsi_wl_surface->wayland_surface.wrapper, + chain->buffer_transform); + wsi_wl_surface->buffer_transform = chain->buffer_transform; + } wl_surface_commit(wsi_wl_surface->wayland_surface.wrapper); wl_display_flush(wsi_wl_surface->display->wl_display); @@ -4023,6 +4234,7 @@ wsi_wl_surface_create_swapchain(VkIcdSurfaceBase *icd_surface, chain->base.image_count = num_images; chain->base.set_hdr_metadata = wsi_wl_swapchain_set_hdr_metadata; chain->extent = pCreateInfo->imageExtent; + chain->buffer_transform = vk_transform_to_wl(pCreateInfo->preTransform); chain->vk_format = pCreateInfo->imageFormat; chain->buffer_type = buffer_type; if (buffer_type == WSI_WL_BUFFER_NATIVE) {