diff --git a/src/asahi/libagx/libagx_dgc.h b/src/asahi/libagx/libagx_dgc.h index 5fb80aab0085..0a3279f0b752 100644 --- a/src/asahi/libagx/libagx_dgc.h +++ b/src/asahi/libagx/libagx_dgc.h @@ -602,9 +602,29 @@ agx_cdm_barrier(GLOBAL uint32_t *out, enum agx_chip chip) // cfg.unk_20 = true; // cfg.unk_24 = true; if clustered? if (chip == AGX_CHIP_G13X) { + /* G13X (t600x, M1 Pro/Max, G13C cores): restore bits 0-2 + * (PBE/texture flush) alongside unk_4 + usc_cache_inval. + * Bits 0-2 are required for cross-launch coherency when + * control streams are not split and across dependency-skipped + * launches ({0,1,2,4,5,6,8}+usc_cache_inval = 0x17f). */ + cfg.unk_0 = true; + cfg.unk_1 = true; + cfg.unk_2 = true; cfg.unk_4 = true; + cfg.usc_cache_inval = true; // cfg.unk_26 = true; } + if (chip == AGX_CHIP_G13G) { + /* G13G (t8103, M1): the kitchen-sink bits below cost ~10 us per + * dispatch in real dependent compute chains (Qwen decode: ~2.1 ms + * of a ~9.8 ms token). Bits 4-8 hold the mlx-omarchy pinned + * generated-ID digests (48/48 interleaved runs, both legs) and + * the omarchy runtime suite (22 cases / 6189 assertions), and + * measure +3.05% ctx1053 decode on jwm1. Other chips keep the + * full sink until measured there. */ + cfg.unk_4 = true; + cfg.unk_7 = true; + } /* With multiple launches in the same CDM stream, we can get cache * coherency (? or sync?) issues. We hit this with blits, which need - in @@ -613,26 +633,71 @@ agx_cdm_barrier(GLOBAL uint32_t *out, enum agx_chip chip) * let's just set these after every launch to be safe. We can revisit in * the future when we figure out what the bits mean. */ + if (chip != AGX_CHIP_G13G && chip != AGX_CHIP_G13X) { + cfg.unk_0 = true; + cfg.unk_1 = true; + cfg.unk_2 = true; + cfg.usc_cache_inval = true; + cfg.unk_4 = true; + cfg.unk_5 = true; + cfg.unk_6 = true; + cfg.unk_7 = true; + cfg.unk_8 = true; + cfg.unk_9 = true; + cfg.unk_10 = true; + cfg.unk_11 = true; + cfg.unk_12 = true; + cfg.unk_13 = true; + cfg.unk_14 = true; + cfg.unk_15 = true; + cfg.unk_16 = true; + cfg.unk_17 = true; + cfg.unk_18 = true; + cfg.unk_19 = true; + } + } + + return out; +} + +/* + * Minimal variant of agx_cdm_barrier: invalidate the USC cache only. Used by + * the Vulkan driver for launches whose results are consumed by another compute + * launch through the same (L2-backed) storage path, where the PBE/texture + * cache maintenance of the full barrier is not required. + */ +static inline GLOBAL uint32_t * +agx_cdm_barrier_usc(GLOBAL uint32_t *out) +{ + agx_push(out, CDM_BARRIER, cfg) { + cfg.usc_cache_inval = true; + } + + return out; +} + +/* + * G13X candidate reduction: the designed set {unk_4, unk_5, unk_6, unk_8} + * (the bits that held mlx-omarchy pinned digests 48/48 in the termA + * battery) plus the USC cache invalidate (covers uniform/texture-state + * changes between launches; measured 4.6 us/launch of the 20.4 us + * sink+cache cost on real dependent chains). The full-sink unk_0..unk_19 + * block (PBE/texture cache maintenance, measured 15.8 us/launch) is + * dropped. Perftest arm designedusccdmbarrier; NOT a default-emission + * proposal until it holds both decode legs and the digest protocol. + */ +static inline GLOBAL uint32_t * +agx_cdm_barrier_designed_usc(GLOBAL uint32_t *out) +{ + agx_push(out, CDM_BARRIER, cfg) { cfg.unk_0 = true; cfg.unk_1 = true; cfg.unk_2 = true; - cfg.usc_cache_inval = true; cfg.unk_4 = true; cfg.unk_5 = true; cfg.unk_6 = true; - cfg.unk_7 = true; cfg.unk_8 = true; - cfg.unk_9 = true; - cfg.unk_10 = true; - cfg.unk_11 = true; - cfg.unk_12 = true; - cfg.unk_13 = true; - cfg.unk_14 = true; - cfg.unk_15 = true; - cfg.unk_16 = true; - cfg.unk_17 = true; - cfg.unk_18 = true; - cfg.unk_19 = true; + cfg.usc_cache_inval = true; } return out; diff --git a/src/asahi/vulkan/hk_cdm_flush_dirty.c b/src/asahi/vulkan/hk_cdm_flush_dirty.c new file mode 100644 index 000000000000..53e501607c8f --- /dev/null +++ b/src/asahi/vulkan/hk_cdm_flush_dirty.c @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Aurora Silicon + * SPDX-License-Identifier: MIT + */ + +#include "hk_cmd_buffer.h" +#include "hk_device.h" + +/* Flush unflushed dependency-tracked CDM work (cmd->state.cs.cdm_dirty) at a + * visibility boundary: a vkCmdPipelineBarrier2, the end of the stream's + * batch, or hk_EndCommandBuffer. No-op when the command buffer holds no + * unflushed launches or the stream is not the compute stream they went to. + */ +void +hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs) +{ + if (!cs || !cmd->state.cs.cdm_dirty || cs->type != HK_CS_CDM) + return; + + hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE); + hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cs); +} + +void +hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd) +{ + if (!cmd->state.cs.cdm_dirty) + return; + + if (cmd->current_cs.cs) + hk_cdm_flush_dirty(cmd, cmd->current_cs.cs); + + list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) { + if (cs != cmd->current_cs.cs) + hk_cdm_flush_dirty(cmd, cs); + } + + cmd->state.cs.cdm_dirty = false; +} diff --git a/src/asahi/vulkan/hk_cmd_buffer.c b/src/asahi/vulkan/hk_cmd_buffer.c index 49772d7f0d66..10abc0e0697f 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.c +++ b/src/asahi/vulkan/hk_cmd_buffer.c @@ -957,6 +957,7 @@ hk_EndCommandBuffer(VkCommandBuffer commandBuffer) perf_debug(cmd, "End command buffer"); hk_flush_deferred_clears(cmd); hk_sync_flush_pending_timestamp(cmd); + hk_cmd_buffer_end_compute(cmd); hk_cmd_buffer_end_compute_internal(cmd, &cmd->current_cs.post_gfx); hk_resolve_deferred_oq_available(cmd); @@ -1058,6 +1059,45 @@ hk_classify_pass_barrier(struct hk_cmd_buffer *cmd, const VkDependencyInfo *info return hk_barrier_classify(info, &state); } +/* xfer/compute→* barriers need a CDM visibility record in the stream that + * holds the source. The G14 dependency-tracked fast path used to skip this + * and only flush through cdm_dirty. */ +static void +hk_cdm_xfer_visibility_at_barrier(struct hk_cmd_buffer *cmd, + const VkDependencyInfo *dep) +{ + struct hk_device *dev = hk_cmd_buffer_device(cmd); + uint8_t src_kinds, dst_kinds; + + hk_sync_dependency_kinds(dep, hk_sync_image_class(cmd), &src_kinds, + &dst_kinds); + if (!(src_kinds & (HK_SYNC_COMPUTE | HK_SYNC_XFER)) || !dst_kinds) + return; + + const bool alias_scope = hk_cdm_alias_scope(dev); + struct hk_cs *list[16]; + unsigned n = 0; + + if (cmd->current_cs.cs && n < ARRAY_SIZE(list)) + list[n++] = cmd->current_cs.cs; + list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) { + if (cs == cmd->current_cs.cs || n >= ARRAY_SIZE(list)) + continue; + list[n++] = cs; + } + + for (unsigned i = 0; i < n; i++) { + struct hk_cs *cs = list[i]; + + if (cs->type != HK_CS_CDM || + !hk_cdm_visibility_flush_needed(dev, cs, alias_scope)) + continue; + + hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE); + hk_cdm_visibility_flush(dev, cs, alias_scope); + } +} + VKAPI_ATTR void VKAPI_CALL hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, const VkDependencyInfo *pDependencyInfo) @@ -1098,6 +1138,35 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, /* M5 records its barriers with the native dependency tracker. */ if (!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip)) { + if (!HK_PERF(dev, ALWAYSCDMBARRIER)) { + /* Outside a render pass only a CDM control stream can be open; the + * batches must end when a graphics batch is open: the dependency + * then spans batch types and the kernel orders the resulting + * commands (agx_cmd_header barriers). */ + if (!cmd->current_cs.gfx && !cmd->current_cs.pre_gfx && + !cmd->current_cs.post_gfx) { + /* The app declared a memory dependency: flush unflushed CDM work + * here, at the visibility point, in the stream that holds it. + * Deferring the record to the next CDM launch loses it when the + * consumer is graphics or was recorded in another command + * buffer (ExecuteSecondaryCommandBuffer); dEQP + * memory.pipeline_barrier.transfer_* fails that way. The batch + * stays open: merge_control_streams stitches it back together + * at hk_EndCommandBuffer, without a stream link per barrier. + */ + hk_cdm_flush_all_dirty(cmd); + hk_cdm_xfer_visibility_at_barrier(cmd, pDependencyInfo); + + /* Preserve the same conservative ordering in the shared submit + * graph, including kernels that advertise exact prior barriers + * on AGX2. */ + hk_sync_barrier(cmd, HK_SYNC_ALL, HK_SYNC_ALL); + return; + } + } + + hk_cdm_flush_all_dirty(cmd); + hk_cdm_xfer_visibility_at_barrier(cmd, pDependencyInfo); hk_cmd_buffer_end_compute(cmd); hk_cmd_buffer_end_graphics(cmd); /* Preserve the same conservative ordering in the shared submit graph, diff --git a/src/asahi/vulkan/hk_cmd_buffer.h b/src/asahi/vulkan/hk_cmd_buffer.h index 38941cb1eb5c..767222a79acb 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.h +++ b/src/asahi/vulkan/hk_cmd_buffer.h @@ -1000,6 +1000,16 @@ hk_m5_fragment_root_select(struct hk_graphics_state *gfx, uint8_t root_reads, struct hk_compute_state { struct hk_descriptor_state descriptors; struct hk_api_shader *shader; + + /* True while this command buffer holds CDM launches whose writes are + * still cached: the dependency-tracked barrier flushed neither at a + * vkCmdPipelineBarrier2 nor at a batch boundary. Set by each tracked + * launch, cleared wherever a CDM cache flush is emitted for the stream + * holding those launches (hk_cdm_flush_dirty). Lives on the command + * buffer, not the hk_cs, so it survives CS splits and + * merge_control_streams. + */ + bool cdm_dirty; }; /* @@ -2283,12 +2293,15 @@ bool hk_g17p_smooth_varyings_supported( void *hk_cmd_buffer_emit_vdm_terminate(struct hk_cmd_buffer *cmd, void *current); +void hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd); + static void hk_cmd_buffer_end_compute_internal(struct hk_cmd_buffer *cmd, struct hk_cs **ptr) { if (*ptr) { struct hk_cs *cs = *ptr; + hk_cdm_flush_all_dirty(cmd); if (unlikely(cs->query_reset_pending)) hk_flush_query_resets(cmd, cs); @@ -2549,6 +2562,9 @@ void hk_usc_upload_spilled_rt_descs(struct agx_usc_builder *b, void hk_cdm_cache_flush(struct hk_device *dev, struct hk_cs *cs); +void hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs); +void hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd); + /* As hk_cdm_cache_flush(), for a flush an application dependency asks for. * With `memory`, the flush also has memory scope on G17P (M5 flushes always * do): needed for host availability, and for device dependencies once the diff --git a/src/asahi/vulkan/hk_cmd_dispatch.c b/src/asahi/vulkan/hk_cmd_dispatch.c index e51e41fd222b..897d42f1b96e 100644 --- a/src/asahi/vulkan/hk_cmd_dispatch.c +++ b/src/asahi/vulkan/hk_cmd_dispatch.c @@ -573,7 +573,26 @@ hk_cdm_flush(struct hk_device *dev, struct hk_cs *cs, bool memory) switch (agx_chip_command_stream_generation(dev->dev.chip)) { case AGX_CS_GENERATION_AGX2: - cs->current = agx_cdm_barrier(cs->current, dev->dev.chip); + if (unlikely(dev->cdm_barrier_mask)) { + /* Perftest only: exact CDM_BARRIER flag word from the environment. */ + uint32_t *word = cs->current; + *word = dev->cdm_barrier_mask | (AGX_CDM_BLOCK_TYPE_BARRIER << 29); + cs->current = word + 1; + } else if (HK_PERF(dev, NOCDMBARRIER)) { + /* Perftest only: launch with no cache maintenance at all. */ + return; + } else if (HK_PERF(dev, DESIGNEDUSCCDMBARRIER)) { + /* Perftest only: designed set {4,5,6,8} + USC cache invalidate. */ + cs->current = agx_cdm_barrier_designed_usc(cs->current); + } else if (HK_PERF(dev, USCCDMBARRIER)) { + /* Perftest only: keep the USC cache invalidate, drop the rest. */ + cs->current = agx_cdm_barrier_usc(cs->current); + } else { + cs->current = agx_cdm_barrier(cs->current, dev->dev.chip); + } + /* Record the trailing flush so a back-to-back barrier request at a + * declared dependency can reuse it (G15+ already did this). */ + cs->strong_flush_end = cs->current; break; case AGX_CS_GENERATION_G15: @@ -701,10 +720,28 @@ hk_dispatch_with_usc_launch(struct hk_device *dev, struct hk_cs *cs, hk_ensure_cs_has_space(cs->cmd, cs, 0x2000 /* TODO */); cs->stats.cmds++; + + if (HK_PERF(dev, ALWAYSCDMBARRIER)) { + cs->current = + agx_cdm_launch(cs->current, dev->dev.chip, grid, wg, launch, usc); + hk_cdm_cache_flush(dev, cs); + return; + } + + /* Dependency-tracked CDM barrier: this launch's writes stay cached, and + * cdm_dirty remembers them. The flush lands where Vulkan makes them + * visible: the next vkCmdPipelineBarrier2 in this command buffer, the + * end of the stream's batch, or hk_EndCommandBuffer (queue submission + * order orders the next submission after this one). Launches the app + * left unordered inside one command buffer keep skipping the drain + * entirely. Driver-internal ordering that does not ride on + * vkCmdPipelineBarrier (query availability writes) calls + * hk_cdm_cache_flush explicitly and is unaffected. + */ + cs->cmd->state.cs.cdm_dirty = true; + cs->current = agx_cdm_launch(cs->current, dev->dev.chip, grid, wg, launch, usc); - - hk_cdm_cache_flush(dev, cs); } void diff --git a/src/asahi/vulkan/hk_device.c b/src/asahi/vulkan/hk_device.c index 670632153f93..673eab887908 100644 --- a/src/asahi/vulkan/hk_device.c +++ b/src/asahi/vulkan/hk_device.c @@ -84,6 +84,15 @@ static const struct debug_named_value hk_perf_options[] = { {"nobarrier", HK_PERF_NOBARRIER,"Ignore pipeline barriers"}, {"nobatch", HK_PERF_NOBATCH, "Disable submission batching"}, {"norobust", HK_PERF_NOROBUST, "Disable robustness"}, + {"nocdmbarrier", HK_PERF_NOCDMBARRIER, + "Omit the per-launch CDM barrier entirely"}, + {"usccdmbarrier", HK_PERF_USCCDMBARRIER, + "Reduce the per-launch CDM barrier to a USC cache invalidate"}, + {"designedusccdmbarrier", HK_PERF_DESIGNEDUSCCDMBARRIER, + "G13X designed set {0,1,2,4,5,6,8} plus USC cache invalidate (0x17f)"}, + {"alwayscdmbarrier", HK_PERF_ALWAYSCDMBARRIER, "Unconditional per-launch CDM barrier (disable dependency tracking)"}, + {"trackcdmbarrier", HK_PERF_TRACKCDMBARRIER, + "Dependency-track the CDM barrier on chips other than G13"}, DEBUG_NAMED_VALUE_END }; /* clang-format on */ @@ -867,6 +876,27 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice, } dev->perftest = debug_get_flags_option("HK_PERFTEST", hk_perf_options, 0); + dev->cdm_barrier_mask = + debug_get_num_option("HK_CDM_BARRIER_MASK", 0) & 0x07ffffff; + + /* The dependency-tracked CDM barrier holds token-digest parity with the + * reference driver on G13 (M1 family) and on G14X of generation 14 (M2 + * Pro/Max/Ultra class; verified on M2 Max: greedy token digests of three + * LLM decode runs of 2048 tokens, CTS compute groups and the + * floating-point gates equal to the per-launch barrier). Other chips, + * including multi-cluster generation 15 and later parts that also map to + * G14X, keep the per-launch barrier until they pass the same gate; + * HK_PERFTEST=trackcdmbarrier opts them in for that run and + * HK_PERFTEST=alwayscdmbarrier forces the per-launch barrier everywhere. + */ + bool tracked_by_default = + pdev->dev.chip == AGX_CHIP_G13G || pdev->dev.chip == AGX_CHIP_G13X || + (pdev->dev.chip == AGX_CHIP_G14X && + pdev->dev.params.gpu_generation == 14); + + if (!tracked_by_default && !HK_PERF(dev, TRACKCDMBARRIER)) + dev->perftest |= HK_PERF_ALWAYSCDMBARRIER; + if (instance->drirc.misc.disable_border_emulation) { dev->perftest |= HK_PERF_NOBORDER; diff --git a/src/asahi/vulkan/hk_device.h b/src/asahi/vulkan/hk_device.h index 725bd4ca8b27..7512899c6eaf 100644 --- a/src/asahi/vulkan/hk_device.h +++ b/src/asahi/vulkan/hk_device.h @@ -327,6 +327,9 @@ struct hk_device { } scratch; uint32_t perftest; + /* HK_CDM_BARRIER_MASK=: perftest-only override of the per-launch + * CDM_BARRIER flag word (bits 0..26). 0 = unset. */ + uint32_t cdm_barrier_mask; struct { struct u_rwlock lock; @@ -348,6 +351,16 @@ enum hk_perftest { HK_PERF_NOBARRIER = BITFIELD_BIT(2), HK_PERF_NOBATCH = BITFIELD_BIT(3), HK_PERF_NOROBUST = BITFIELD_BIT(4), + HK_PERF_NOCDMBARRIER = BITFIELD_BIT(5), + HK_PERF_USCCDMBARRIER = BITFIELD_BIT(6), + HK_PERF_DESIGNEDUSCCDMBARRIER = BITFIELD_BIT(7), + /* Restore the unconditional per-launch CDM barrier and the batch split at + * every vkCmdPipelineBarrier2. The default on G13 is the + * dependency-tracked barrier; other chips default to this. + */ + HK_PERF_ALWAYSCDMBARRIER = BITFIELD_BIT(8), + /* Use the dependency-tracked barrier on chips other than G13. */ + HK_PERF_TRACKCDMBARRIER = BITFIELD_BIT(9), }; #define HK_PERF(dev, flag) unlikely((dev)->perftest &HK_PERF_##flag) diff --git a/src/asahi/vulkan/meson.build b/src/asahi/vulkan/meson.build index 7519b6e2716f..ec95e2800f94 100644 --- a/src/asahi/vulkan/meson.build +++ b/src/asahi/vulkan/meson.build @@ -15,6 +15,7 @@ hk_files = files( 'hk_buffer_view.h', 'hk_bvh.h', 'hk_cmd_buffer.c', + 'hk_cdm_flush_dirty.c', 'hk_cmd_upload.c', 'hk_cmd_buffer.h', 'hk_conditional_render_core.h', @@ -186,7 +187,8 @@ if with_tests 'hk_query_reset', executable( 'hk_query_reset_test', - files('tests/test-hk-query-reset.c', 'hk_query_reset.c'), + files('tests/test-hk-query-reset.c', 'hk_query_reset.c', + 'hk_cdm_flush_dirty.c'), include_directories : [ include_directories('.'), inc_gallium, inc_gallium_aux, inc_include, inc_src, inc_asahi, diff --git a/src/asahi/vulkan/tests/test-hk-query-oq.c b/src/asahi/vulkan/tests/test-hk-query-oq.c index b6d914c9b5d1..23567773828e 100644 --- a/src/asahi/vulkan/tests/test-hk-query-oq.c +++ b/src/asahi/vulkan/tests/test-hk-query-oq.c @@ -54,6 +54,12 @@ hk_flush_query_resets(struct hk_cmd_buffer *cmd, struct hk_cs *cs) abort(); } +void +hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd) +{ + (void)cmd; +} + void hk_g17p_meta_fill_addr(struct hk_cmd_buffer *cmd, struct hk_cs **target, uint64_t address, uint64_t size, uint32_t value) diff --git a/src/asahi/vulkan/tests/test-hk-sync-prior.c b/src/asahi/vulkan/tests/test-hk-sync-prior.c index fdc5e2c6e499..6a333998a971 100644 --- a/src/asahi/vulkan/tests/test-hk-sync-prior.c +++ b/src/asahi/vulkan/tests/test-hk-sync-prior.c @@ -249,6 +249,12 @@ hk_flush_query_resets(struct hk_cmd_buffer *cmd, struct hk_cs *cs) abort(); } +void +hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd) +{ + (void)cmd; +} + /* Timestamp recording references payload/allocation entry points for other * branches. These tests attach end stamps to already-created closed streams; * reaching a payload emission branch would invalidate that fixture contract.