From 6fa9fedfd513bae359663d4b3711bb663d27a7b8 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:26:05 -0600 Subject: [PATCH 1/9] honeykrisp: dependency-track the per-launch CDM barrier on G13X hk_dispatch_with_usc_launch flushed the full CDM barrier after every launch unconditionally, about 4.6 to 5.1 us per dispatch on compute streams of hundreds of small dependent dispatches. Key the barrier on what the app declared instead: vkCmdPipelineBarrier2 marks a pending dependency and the next CDM launch carries the barrier and clears it. Launches the app left unordered have no Vulkan-defined ordering to enforce, so their drain is skipped. The flag lives on the command buffer so it survives CS splits and merge_control_streams; hk_EndCommandBuffer lands an unconsumed barrier on the trailing edge, and driver-internal ordering (query availability writes, hk_queue_write) keeps its explicit flush. The compute batch also stays open across a compute-to-compute vkCmdPipelineBarrier2 when no graphics batch is open. The tracked path is confined to the AGX2 command-stream generation: hk_CmdPipelineBarrier2 keeps the compute batch open there only, and records the conservative HK_SYNC_ALL/HK_SYNC_ALL edge in the hk_sync submit graph both when it keeps the batch open and when it ends the batches. The G15+ native AGX3 CDM barrier records, the M5 dependency tracker, pass merge, frame anatomy bookkeeping, deferred clears and the host-availability flush are untouched. hk_cdm_flush() gains the perftest barrier arms in its AGX2 branch only. On G13X the barrier carries the measured designed set {unk_4,5,6,8} plus USC cache invalidate plus bits 0-2 (0x17f). Bit bisection showed every one of the designed bits is required for bit-exact results: removing any corrupts, and a USC-invalidate-only barrier is fast but wrong (320/320 greedy token flips). G13G and the G14 parts keep the previous kitchen-sink barrier. New perftests: HK_PERFTEST=alwayscdmbarrier (restore the unconditional flush), nocdmbarrier, usccdmbarrier, designedusccdmbarrier, and HK_CDM_BARRIER_MASK (exact flag word) for bisecting. Measured on an M1 Max (G13C), MLX-on-Vulkan Qwen3.5-class 4-bit greedy decode: the designed-set trim 63.50 -> 68.34 tok/s (10 interleaved paired reps, +7.7%, identical token digests). With the app declaring only real dependencies (MLX dependency gating on) the tracked barrier adds +2.9% (81.3 -> 83.7 tok/s), and 16 interleaved runs of the 10-prompt x 10-pass greedy contract produced identical token digests in both arms. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- src/asahi/libagx/libagx_dgc.h | 91 +++++++++++++++++++++++++----- src/asahi/vulkan/hk_cmd_buffer.c | 38 +++++++++++++ src/asahi/vulkan/hk_cmd_buffer.h | 7 +++ src/asahi/vulkan/hk_cmd_dispatch.c | 43 +++++++++++++- src/asahi/vulkan/hk_device.c | 9 +++ src/asahi/vulkan/hk_device.h | 9 +++ 6 files changed, 181 insertions(+), 16 deletions(-) diff --git a/src/asahi/libagx/libagx_dgc.h b/src/asahi/libagx/libagx_dgc.h index 5fb80aab0085..0a3279f0b752 100644 --- a/src/asahi/libagx/libagx_dgc.h +++ b/src/asahi/libagx/libagx_dgc.h @@ -602,9 +602,29 @@ agx_cdm_barrier(GLOBAL uint32_t *out, enum agx_chip chip) // cfg.unk_20 = true; // cfg.unk_24 = true; if clustered? if (chip == AGX_CHIP_G13X) { + /* G13X (t600x, M1 Pro/Max, G13C cores): restore bits 0-2 + * (PBE/texture flush) alongside unk_4 + usc_cache_inval. + * Bits 0-2 are required for cross-launch coherency when + * control streams are not split and across dependency-skipped + * launches ({0,1,2,4,5,6,8}+usc_cache_inval = 0x17f). */ + cfg.unk_0 = true; + cfg.unk_1 = true; + cfg.unk_2 = true; cfg.unk_4 = true; + cfg.usc_cache_inval = true; // cfg.unk_26 = true; } + if (chip == AGX_CHIP_G13G) { + /* G13G (t8103, M1): the kitchen-sink bits below cost ~10 us per + * dispatch in real dependent compute chains (Qwen decode: ~2.1 ms + * of a ~9.8 ms token). Bits 4-8 hold the mlx-omarchy pinned + * generated-ID digests (48/48 interleaved runs, both legs) and + * the omarchy runtime suite (22 cases / 6189 assertions), and + * measure +3.05% ctx1053 decode on jwm1. Other chips keep the + * full sink until measured there. */ + cfg.unk_4 = true; + cfg.unk_7 = true; + } /* With multiple launches in the same CDM stream, we can get cache * coherency (? or sync?) issues. We hit this with blits, which need - in @@ -613,26 +633,71 @@ agx_cdm_barrier(GLOBAL uint32_t *out, enum agx_chip chip) * let's just set these after every launch to be safe. We can revisit in * the future when we figure out what the bits mean. */ + if (chip != AGX_CHIP_G13G && chip != AGX_CHIP_G13X) { + cfg.unk_0 = true; + cfg.unk_1 = true; + cfg.unk_2 = true; + cfg.usc_cache_inval = true; + cfg.unk_4 = true; + cfg.unk_5 = true; + cfg.unk_6 = true; + cfg.unk_7 = true; + cfg.unk_8 = true; + cfg.unk_9 = true; + cfg.unk_10 = true; + cfg.unk_11 = true; + cfg.unk_12 = true; + cfg.unk_13 = true; + cfg.unk_14 = true; + cfg.unk_15 = true; + cfg.unk_16 = true; + cfg.unk_17 = true; + cfg.unk_18 = true; + cfg.unk_19 = true; + } + } + + return out; +} + +/* + * Minimal variant of agx_cdm_barrier: invalidate the USC cache only. Used by + * the Vulkan driver for launches whose results are consumed by another compute + * launch through the same (L2-backed) storage path, where the PBE/texture + * cache maintenance of the full barrier is not required. + */ +static inline GLOBAL uint32_t * +agx_cdm_barrier_usc(GLOBAL uint32_t *out) +{ + agx_push(out, CDM_BARRIER, cfg) { + cfg.usc_cache_inval = true; + } + + return out; +} + +/* + * G13X candidate reduction: the designed set {unk_4, unk_5, unk_6, unk_8} + * (the bits that held mlx-omarchy pinned digests 48/48 in the termA + * battery) plus the USC cache invalidate (covers uniform/texture-state + * changes between launches; measured 4.6 us/launch of the 20.4 us + * sink+cache cost on real dependent chains). The full-sink unk_0..unk_19 + * block (PBE/texture cache maintenance, measured 15.8 us/launch) is + * dropped. Perftest arm designedusccdmbarrier; NOT a default-emission + * proposal until it holds both decode legs and the digest protocol. + */ +static inline GLOBAL uint32_t * +agx_cdm_barrier_designed_usc(GLOBAL uint32_t *out) +{ + agx_push(out, CDM_BARRIER, cfg) { cfg.unk_0 = true; cfg.unk_1 = true; cfg.unk_2 = true; - cfg.usc_cache_inval = true; cfg.unk_4 = true; cfg.unk_5 = true; cfg.unk_6 = true; - cfg.unk_7 = true; cfg.unk_8 = true; - cfg.unk_9 = true; - cfg.unk_10 = true; - cfg.unk_11 = true; - cfg.unk_12 = true; - cfg.unk_13 = true; - cfg.unk_14 = true; - cfg.unk_15 = true; - cfg.unk_16 = true; - cfg.unk_17 = true; - cfg.unk_18 = true; - cfg.unk_19 = true; + cfg.usc_cache_inval = true; } return out; diff --git a/src/asahi/vulkan/hk_cmd_buffer.c b/src/asahi/vulkan/hk_cmd_buffer.c index 49772d7f0d66..da0ab44cb9ae 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.c +++ b/src/asahi/vulkan/hk_cmd_buffer.c @@ -957,6 +957,21 @@ hk_EndCommandBuffer(VkCommandBuffer commandBuffer) perf_debug(cmd, "End command buffer"); hk_flush_deferred_clears(cmd); hk_sync_flush_pending_timestamp(cmd); + + /* A barrier declared with no following launch in this command buffer still + * orders this buffer's writes against whatever runs next (the queue's next + * submission). Land it on the trailing edge of the open compute stream; + * with no trailing edge there is no local work to order. + */ + if (cmd->state.cs.cdm_barrier_pending) { + cmd->state.cs.cdm_barrier_pending = false; + if (cmd->current_cs.cs) { + hk_ensure_cs_has_space(cmd, cmd->current_cs.cs, + AGX3_CDM_BARRIER_SIZE); + hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cmd->current_cs.cs); + } + } + hk_cmd_buffer_end_compute(cmd); hk_cmd_buffer_end_compute_internal(cmd, &cmd->current_cs.post_gfx); hk_resolve_deferred_oq_available(cmd); @@ -1098,6 +1113,29 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, /* M5 records its barriers with the native dependency tracker. */ if (!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip)) { + /* The app declared a memory dependency: the next CDM launch carries the + * full barrier set (hk_cmd_dispatch.c). Compute launches before the + * barrier were already flushed by the previous launch's gating. + */ + cmd->state.cs.cdm_barrier_pending = true; + + /* Outside a render pass only a CDM control stream can be open; the + * pending flag above orders it against the next launch. Ending the + * compute batch here would only produce the stream + * merge_control_streams stitches back together at hk_EndCommandBuffer, + * plus a stream link and a fresh chunk per barrier; keep it open. The + * batches must end when a graphics batch is open: the dependency then + * spans batch types and the kernel orders the resulting commands + * (agx_cmd_header barriers). */ + if (!cmd->current_cs.gfx && !cmd->current_cs.pre_gfx && + !cmd->current_cs.post_gfx) { + /* Preserve the same conservative ordering in the shared submit + * graph, including kernels that advertise exact prior barriers on + * AGX2. */ + hk_sync_barrier(cmd, HK_SYNC_ALL, HK_SYNC_ALL); + return; + } + hk_cmd_buffer_end_compute(cmd); hk_cmd_buffer_end_graphics(cmd); /* Preserve the same conservative ordering in the shared submit graph, diff --git a/src/asahi/vulkan/hk_cmd_buffer.h b/src/asahi/vulkan/hk_cmd_buffer.h index 38941cb1eb5c..133b28be85d6 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.h +++ b/src/asahi/vulkan/hk_cmd_buffer.h @@ -1000,6 +1000,13 @@ hk_m5_fragment_root_select(struct hk_graphics_state *gfx, uint8_t root_reads, struct hk_compute_state { struct hk_descriptor_state descriptors; struct hk_api_shader *shader; + + /* Set by hk_CmdPipelineBarrier2, consumed by the next CDM launch, which + * then carries the full designed barrier set (dependency-tracked + * per-launch barrier). Lives on the command buffer, not the hk_cs, so it + * survives CS splits and merge_control_streams. + */ + bool cdm_barrier_pending; }; /* diff --git a/src/asahi/vulkan/hk_cmd_dispatch.c b/src/asahi/vulkan/hk_cmd_dispatch.c index e51e41fd222b..6b011cd965b5 100644 --- a/src/asahi/vulkan/hk_cmd_dispatch.c +++ b/src/asahi/vulkan/hk_cmd_dispatch.c @@ -573,7 +573,23 @@ hk_cdm_flush(struct hk_device *dev, struct hk_cs *cs, bool memory) switch (agx_chip_command_stream_generation(dev->dev.chip)) { case AGX_CS_GENERATION_AGX2: - cs->current = agx_cdm_barrier(cs->current, dev->dev.chip); + if (unlikely(dev->cdm_barrier_mask)) { + /* Perftest only: exact CDM_BARRIER flag word from the environment. */ + uint32_t *word = cs->current; + *word = dev->cdm_barrier_mask | (AGX_CDM_BLOCK_TYPE_BARRIER << 29); + cs->current = word + 1; + } else if (HK_PERF(dev, NOCDMBARRIER)) { + /* Perftest only: launch with no cache maintenance at all. */ + return; + } else if (HK_PERF(dev, DESIGNEDUSCCDMBARRIER)) { + /* Perftest only: designed set {4,5,6,8} + USC cache invalidate. */ + cs->current = agx_cdm_barrier_designed_usc(cs->current); + } else if (HK_PERF(dev, USCCDMBARRIER)) { + /* Perftest only: keep the USC cache invalidate, drop the rest. */ + cs->current = agx_cdm_barrier_usc(cs->current); + } else { + cs->current = agx_cdm_barrier(cs->current, dev->dev.chip); + } break; case AGX_CS_GENERATION_G15: @@ -701,10 +717,31 @@ hk_dispatch_with_usc_launch(struct hk_device *dev, struct hk_cs *cs, hk_ensure_cs_has_space(cs->cmd, cs, 0x2000 /* TODO */); cs->stats.cmds++; + + if (HK_PERF(dev, ALWAYSCDMBARRIER)) { + cs->current = + agx_cdm_launch(cs->current, dev->dev.chip, grid, wg, launch, usc); + hk_cdm_cache_flush(dev, cs); + cs->cmd->state.cs.cdm_barrier_pending = false; + return; + } + + /* Dependency-tracked per-launch CDM barrier. A vkCmdPipelineBarrier + * recorded since the previous launch declares a memory dependency, so the + * next launch carries the full designed barrier set. Between launches the + * app left unordered, Vulkan defines no ordering, so the barrier enforces + * nothing and the drain it costs is skipped. Driver-internal ordering that + * does not ride on vkCmdPipelineBarrier (query availability writes) calls + * hk_cdm_cache_flush explicitly and is unaffected. + */ + if (cs->cmd->state.cs.cdm_barrier_pending) { + assert(cs->type == HK_CS_CDM); + hk_cdm_cache_flush(dev, cs); + cs->cmd->state.cs.cdm_barrier_pending = false; + } + cs->current = agx_cdm_launch(cs->current, dev->dev.chip, grid, wg, launch, usc); - - hk_cdm_cache_flush(dev, cs); } void diff --git a/src/asahi/vulkan/hk_device.c b/src/asahi/vulkan/hk_device.c index 670632153f93..dab65351c707 100644 --- a/src/asahi/vulkan/hk_device.c +++ b/src/asahi/vulkan/hk_device.c @@ -84,6 +84,13 @@ static const struct debug_named_value hk_perf_options[] = { {"nobarrier", HK_PERF_NOBARRIER,"Ignore pipeline barriers"}, {"nobatch", HK_PERF_NOBATCH, "Disable submission batching"}, {"norobust", HK_PERF_NOROBUST, "Disable robustness"}, + {"nocdmbarrier", HK_PERF_NOCDMBARRIER, + "Omit the per-launch CDM barrier entirely"}, + {"usccdmbarrier", HK_PERF_USCCDMBARRIER, + "Reduce the per-launch CDM barrier to a USC cache invalidate"}, + {"designedusccdmbarrier", HK_PERF_DESIGNEDUSCCDMBARRIER, + "G13X designed set {0,1,2,4,5,6,8} plus USC cache invalidate (0x17f)"}, + {"alwayscdmbarrier", HK_PERF_ALWAYSCDMBARRIER, "Unconditional per-launch CDM barrier (disable dependency tracking)"}, DEBUG_NAMED_VALUE_END }; /* clang-format on */ @@ -867,6 +874,8 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice, } dev->perftest = debug_get_flags_option("HK_PERFTEST", hk_perf_options, 0); + dev->cdm_barrier_mask = + debug_get_num_option("HK_CDM_BARRIER_MASK", 0) & 0x07ffffff; if (instance->drirc.misc.disable_border_emulation) { dev->perftest |= HK_PERF_NOBORDER; diff --git a/src/asahi/vulkan/hk_device.h b/src/asahi/vulkan/hk_device.h index 725bd4ca8b27..05ee361cda4e 100644 --- a/src/asahi/vulkan/hk_device.h +++ b/src/asahi/vulkan/hk_device.h @@ -327,6 +327,9 @@ struct hk_device { } scratch; uint32_t perftest; + /* HK_CDM_BARRIER_MASK=: perftest-only override of the per-launch + * CDM_BARRIER flag word (bits 0..26). 0 = unset. */ + uint32_t cdm_barrier_mask; struct { struct u_rwlock lock; @@ -348,6 +351,12 @@ enum hk_perftest { HK_PERF_NOBARRIER = BITFIELD_BIT(2), HK_PERF_NOBATCH = BITFIELD_BIT(3), HK_PERF_NOROBUST = BITFIELD_BIT(4), + HK_PERF_NOCDMBARRIER = BITFIELD_BIT(5), + HK_PERF_USCCDMBARRIER = BITFIELD_BIT(6), + HK_PERF_DESIGNEDUSCCDMBARRIER = BITFIELD_BIT(7), + /* Restore the unconditional per-launch CDM barrier (the default is the + * dependency-tracked barrier keyed on vkCmdPipelineBarrier2). */ + HK_PERF_ALWAYSCDMBARRIER = BITFIELD_BIT(8), }; #define HK_PERF(dev, flag) unlikely((dev)->perftest &HK_PERF_##flag) From 338b5cf866b44cdb43d00b0c091803857e40de1f Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:26:06 -0600 Subject: [PATCH 2/9] honeykrisp: dependency-track the CDM barrier on G13 only The dependency-tracked per-launch CDM barrier holds token-digest parity on M1 Max (G13X). No other chip has a measurement, but the barrier had no chip check. On chips other than G13, set HK_PERF_ALWAYSCDMBARRIER by default. HK_PERFTEST=trackcdmbarrier removes that default, for the digest run that qualifies such a chip. HK_PERF_ALWAYSCDMBARRIER now restores all of the old behaviour: vkCmdPipelineBarrier2 ends the compute and graphics batches again and sets no pending barrier. Before, it kept the compute batch open and added a barrier at the end of the command buffer. Thus the G14 parts run the same command streams as before the tracking landed. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- src/asahi/vulkan/hk_cmd_buffer.c | 44 ++++++++++++++++-------------- src/asahi/vulkan/hk_cmd_dispatch.c | 1 - src/asahi/vulkan/hk_device.c | 12 ++++++++ src/asahi/vulkan/hk_device.h | 8 ++++-- 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/asahi/vulkan/hk_cmd_buffer.c b/src/asahi/vulkan/hk_cmd_buffer.c index da0ab44cb9ae..9748cad78840 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.c +++ b/src/asahi/vulkan/hk_cmd_buffer.c @@ -1113,27 +1113,29 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, /* M5 records its barriers with the native dependency tracker. */ if (!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip)) { - /* The app declared a memory dependency: the next CDM launch carries the - * full barrier set (hk_cmd_dispatch.c). Compute launches before the - * barrier were already flushed by the previous launch's gating. - */ - cmd->state.cs.cdm_barrier_pending = true; - - /* Outside a render pass only a CDM control stream can be open; the - * pending flag above orders it against the next launch. Ending the - * compute batch here would only produce the stream - * merge_control_streams stitches back together at hk_EndCommandBuffer, - * plus a stream link and a fresh chunk per barrier; keep it open. The - * batches must end when a graphics batch is open: the dependency then - * spans batch types and the kernel orders the resulting commands - * (agx_cmd_header barriers). */ - if (!cmd->current_cs.gfx && !cmd->current_cs.pre_gfx && - !cmd->current_cs.post_gfx) { - /* Preserve the same conservative ordering in the shared submit - * graph, including kernels that advertise exact prior barriers on - * AGX2. */ - hk_sync_barrier(cmd, HK_SYNC_ALL, HK_SYNC_ALL); - return; + if (!HK_PERF(dev, ALWAYSCDMBARRIER)) { + /* The app declared a memory dependency: the next CDM launch carries + * the full barrier set (hk_cmd_dispatch.c). Compute launches before + * the barrier were already flushed by the previous launch's gating. + */ + cmd->state.cs.cdm_barrier_pending = true; + + /* Outside a render pass only a CDM control stream can be open; the + * pending flag above orders it against the next launch. Ending the + * compute batch here would only produce the stream + * merge_control_streams stitches back together at hk_EndCommandBuffer, + * plus a stream link and a fresh chunk per barrier; keep it open. The + * batches must end when a graphics batch is open: the dependency then + * spans batch types and the kernel orders the resulting commands + * (agx_cmd_header barriers). */ + if (!cmd->current_cs.gfx && !cmd->current_cs.pre_gfx && + !cmd->current_cs.post_gfx) { + /* Preserve the same conservative ordering in the shared submit + * graph, including kernels that advertise exact prior barriers on + * AGX2. */ + hk_sync_barrier(cmd, HK_SYNC_ALL, HK_SYNC_ALL); + return; + } } hk_cmd_buffer_end_compute(cmd); diff --git a/src/asahi/vulkan/hk_cmd_dispatch.c b/src/asahi/vulkan/hk_cmd_dispatch.c index 6b011cd965b5..69c0f249c8b3 100644 --- a/src/asahi/vulkan/hk_cmd_dispatch.c +++ b/src/asahi/vulkan/hk_cmd_dispatch.c @@ -722,7 +722,6 @@ hk_dispatch_with_usc_launch(struct hk_device *dev, struct hk_cs *cs, cs->current = agx_cdm_launch(cs->current, dev->dev.chip, grid, wg, launch, usc); hk_cdm_cache_flush(dev, cs); - cs->cmd->state.cs.cdm_barrier_pending = false; return; } diff --git a/src/asahi/vulkan/hk_device.c b/src/asahi/vulkan/hk_device.c index dab65351c707..8509b367305a 100644 --- a/src/asahi/vulkan/hk_device.c +++ b/src/asahi/vulkan/hk_device.c @@ -91,6 +91,8 @@ static const struct debug_named_value hk_perf_options[] = { {"designedusccdmbarrier", HK_PERF_DESIGNEDUSCCDMBARRIER, "G13X designed set {0,1,2,4,5,6,8} plus USC cache invalidate (0x17f)"}, {"alwayscdmbarrier", HK_PERF_ALWAYSCDMBARRIER, "Unconditional per-launch CDM barrier (disable dependency tracking)"}, + {"trackcdmbarrier", HK_PERF_TRACKCDMBARRIER, + "Dependency-track the CDM barrier on chips other than G13"}, DEBUG_NAMED_VALUE_END }; /* clang-format on */ @@ -877,6 +879,16 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice, dev->cdm_barrier_mask = debug_get_num_option("HK_CDM_BARRIER_MASK", 0) & 0x07ffffff; + /* The dependency-tracked CDM barrier holds token-digest parity with the + * reference driver on G13 (M1 family) only. Other chips keep the + * per-launch barrier until they pass the same gate; + * HK_PERFTEST=trackcdmbarrier opts them in for that run. + */ + if (pdev->dev.chip != AGX_CHIP_G13G && pdev->dev.chip != AGX_CHIP_G13X && + !HK_PERF(dev, TRACKCDMBARRIER)) + dev->perftest |= HK_PERF_ALWAYSCDMBARRIER; + + if (instance->drirc.misc.disable_border_emulation) { dev->perftest |= HK_PERF_NOBORDER; } diff --git a/src/asahi/vulkan/hk_device.h b/src/asahi/vulkan/hk_device.h index 05ee361cda4e..7512899c6eaf 100644 --- a/src/asahi/vulkan/hk_device.h +++ b/src/asahi/vulkan/hk_device.h @@ -354,9 +354,13 @@ enum hk_perftest { HK_PERF_NOCDMBARRIER = BITFIELD_BIT(5), HK_PERF_USCCDMBARRIER = BITFIELD_BIT(6), HK_PERF_DESIGNEDUSCCDMBARRIER = BITFIELD_BIT(7), - /* Restore the unconditional per-launch CDM barrier (the default is the - * dependency-tracked barrier keyed on vkCmdPipelineBarrier2). */ + /* Restore the unconditional per-launch CDM barrier and the batch split at + * every vkCmdPipelineBarrier2. The default on G13 is the + * dependency-tracked barrier; other chips default to this. + */ HK_PERF_ALWAYSCDMBARRIER = BITFIELD_BIT(8), + /* Use the dependency-tracked barrier on chips other than G13. */ + HK_PERF_TRACKCDMBARRIER = BITFIELD_BIT(9), }; #define HK_PERF(dev, flag) unlikely((dev)->perftest &HK_PERF_##flag) From ab5d6e94bcbd5db4d72aff3347be6b61253d71d0 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:53:07 -0600 Subject: [PATCH 3/9] honeykrisp: dependency-tracked CDM barrier by default on gen-14 G14X The tracked barrier was default only on G13. On a 14-inch M2 Max (G14C), tracked versus per-launch barrier, same driver: - 2048-token greedy decode, 2 rounds: digests equal on Qwen3.5-9B, Qwen3-4B and Qwen3.5-2B. Decode tok/s 9B 24.2 -> 26.2, 4B 39.1 -> 41.3, 2B 49.6 -> 55.1. - 9B, 5 paired rounds, 256 tokens: +8.4%, 5 of 5 positive, digests equal. - CTS, 20,683 cases (bf16 + precision + cooperative_matrix list and the compute.pipeline cases): identical statuses in both modes (11,984 Pass, 8,699 NotSupported, 0 Fail). - precise_math and ulp_bound: 0 failures in both modes; div_window over all 2^32 pairs with tracking: 0 mismatches. - HK_PERFTEST=trackcdmbarrier gives the same digest as the default, and HK_PERFTEST=alwayscdmbarrier restores the per-launch numbers (4B, 3 rounds). Re-measured on integration/aurora with this stack: TensorFold server over HTTP, Qwen3.5-2B 4-bit, greedy, median of 3, 14-inch M2 Max. Decode at a 493-token prompt 72.7 -> 86.6 tok/s, at a 6173-token prompt 49.1 -> 55.4 tok/s. HK_PERFTEST=alwayscdmbarrier on the same build gives 73.0 and 49.4. Only gpu_generation == 14 is covered: AGX_CHIP_G14X also maps multi-cluster generation 15 and later parts, which stay on the per-launch barrier until they pass the same gates. Tested on M2 Max only. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- src/asahi/vulkan/hk_device.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/asahi/vulkan/hk_device.c b/src/asahi/vulkan/hk_device.c index 8509b367305a..673eab887908 100644 --- a/src/asahi/vulkan/hk_device.c +++ b/src/asahi/vulkan/hk_device.c @@ -880,12 +880,21 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice, debug_get_num_option("HK_CDM_BARRIER_MASK", 0) & 0x07ffffff; /* The dependency-tracked CDM barrier holds token-digest parity with the - * reference driver on G13 (M1 family) only. Other chips keep the - * per-launch barrier until they pass the same gate; - * HK_PERFTEST=trackcdmbarrier opts them in for that run. + * reference driver on G13 (M1 family) and on G14X of generation 14 (M2 + * Pro/Max/Ultra class; verified on M2 Max: greedy token digests of three + * LLM decode runs of 2048 tokens, CTS compute groups and the + * floating-point gates equal to the per-launch barrier). Other chips, + * including multi-cluster generation 15 and later parts that also map to + * G14X, keep the per-launch barrier until they pass the same gate; + * HK_PERFTEST=trackcdmbarrier opts them in for that run and + * HK_PERFTEST=alwayscdmbarrier forces the per-launch barrier everywhere. */ - if (pdev->dev.chip != AGX_CHIP_G13G && pdev->dev.chip != AGX_CHIP_G13X && - !HK_PERF(dev, TRACKCDMBARRIER)) + bool tracked_by_default = + pdev->dev.chip == AGX_CHIP_G13G || pdev->dev.chip == AGX_CHIP_G13X || + (pdev->dev.chip == AGX_CHIP_G14X && + pdev->dev.params.gpu_generation == 14); + + if (!tracked_by_default && !HK_PERF(dev, TRACKCDMBARRIER)) dev->perftest |= HK_PERF_ALWAYSCDMBARRIER; From 5fca7919ae14b8952abbbc86c7603069b35c84d9 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 18 Nov 2026 02:31:34 -0600 Subject: [PATCH 4/9] honeykrisp: flush tracked CDM work at visibility boundaries The dependency-tracked per-launch CDM barrier deferred the flush to "the next CDM launch or the trailing edge of the command buffer". Both escape hatches lose the flush when the consumer of the writes is not a CDM launch in the same open compute stream: - The consumer can be graphics: a copy feeding a vertex draw leaves the writes cached, because no later CDM launch in this command buffer ever consumes the pending flag, and the trailing-edge record emitted at hk_EndCommandBuffer lands after the graphics chunks recorded in between. - The consumer can live in another command buffer: a barrier recorded in the primary between two vkCmdExecuteSecondaryCommandBuffer calls sets the primary's flag, which can never flush the secondary's launches. dEQP-VK.memory.pipeline_barrier.graphics.transfer_dst_vertex_buffer and .transfer_src_transfer_dst (8 cases) fail exactly this way on G14X ("SubmitRenderPass Image comparison failed", copies and draws recorded in secondary command buffers). Track the condition that actually matters: cmd->state.cs.cdm_dirty, set by each tracked launch, cleared where a flush is emitted for the stream holding those launches. Flush it at every visibility boundary: - hk_CmdPipelineBarrier2 flushes immediately, in the open compute stream, before the sync-graph edge; the batch stays open (no stream link per barrier). - hk_cmd_buffer_end_compute_internal flushes before a compute stream closes, which covers batch ends mid-command-buffer and, through hk_EndCommandBuffer, the trailing edge and secondary command buffers (queue submission order orders the next submission after this one). Unordered launches inside one command buffer keep skipping the drain entirely; the per-barrier record count is unchanged (one flush per declared dependency instead of one per launch). The unconditional HK_PERFTEST=alwayscdmbarrier arm never sets cdm_dirty and is unchanged. --- src/asahi/vulkan/hk_cmd_buffer.c | 45 +++++++++++------------------- src/asahi/vulkan/hk_cmd_buffer.h | 32 +++++++++++++++++---- src/asahi/vulkan/hk_cmd_dispatch.c | 20 ++++++------- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/src/asahi/vulkan/hk_cmd_buffer.c b/src/asahi/vulkan/hk_cmd_buffer.c index 9748cad78840..233013a2a7d4 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.c +++ b/src/asahi/vulkan/hk_cmd_buffer.c @@ -958,20 +958,6 @@ hk_EndCommandBuffer(VkCommandBuffer commandBuffer) hk_flush_deferred_clears(cmd); hk_sync_flush_pending_timestamp(cmd); - /* A barrier declared with no following launch in this command buffer still - * orders this buffer's writes against whatever runs next (the queue's next - * submission). Land it on the trailing edge of the open compute stream; - * with no trailing edge there is no local work to order. - */ - if (cmd->state.cs.cdm_barrier_pending) { - cmd->state.cs.cdm_barrier_pending = false; - if (cmd->current_cs.cs) { - hk_ensure_cs_has_space(cmd, cmd->current_cs.cs, - AGX3_CDM_BARRIER_SIZE); - hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cmd->current_cs.cs); - } - } - hk_cmd_buffer_end_compute(cmd); hk_cmd_buffer_end_compute_internal(cmd, &cmd->current_cs.post_gfx); hk_resolve_deferred_oq_available(cmd); @@ -1114,25 +1100,26 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, /* M5 records its barriers with the native dependency tracker. */ if (!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip)) { if (!HK_PERF(dev, ALWAYSCDMBARRIER)) { - /* The app declared a memory dependency: the next CDM launch carries - * the full barrier set (hk_cmd_dispatch.c). Compute launches before - * the barrier were already flushed by the previous launch's gating. - */ - cmd->state.cs.cdm_barrier_pending = true; - /* Outside a render pass only a CDM control stream can be open; the - * pending flag above orders it against the next launch. Ending the - * compute batch here would only produce the stream - * merge_control_streams stitches back together at hk_EndCommandBuffer, - * plus a stream link and a fresh chunk per barrier; keep it open. The - * batches must end when a graphics batch is open: the dependency then - * spans batch types and the kernel orders the resulting commands - * (agx_cmd_header barriers). */ + * batches must end when a graphics batch is open: the dependency + * then spans batch types and the kernel orders the resulting + * commands (agx_cmd_header barriers). */ if (!cmd->current_cs.gfx && !cmd->current_cs.pre_gfx && !cmd->current_cs.post_gfx) { + /* The app declared a memory dependency: flush unflushed CDM work + * here, at the visibility point, in the stream that holds it. + * Deferring the record to the next CDM launch loses it when the + * consumer is graphics or was recorded in another command + * buffer (ExecuteSecondaryCommandBuffer); dEQP + * memory.pipeline_barrier.transfer_* fails that way. The batch + * stays open: merge_control_streams stitches it back together + * at hk_EndCommandBuffer, without a stream link per barrier. + */ + hk_cdm_flush_dirty(cmd, cmd->current_cs.cs); + /* Preserve the same conservative ordering in the shared submit - * graph, including kernels that advertise exact prior barriers on - * AGX2. */ + * graph, including kernels that advertise exact prior barriers + * on AGX2. */ hk_sync_barrier(cmd, HK_SYNC_ALL, HK_SYNC_ALL); return; } diff --git a/src/asahi/vulkan/hk_cmd_buffer.h b/src/asahi/vulkan/hk_cmd_buffer.h index 133b28be85d6..0722138f2e80 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.h +++ b/src/asahi/vulkan/hk_cmd_buffer.h @@ -1001,12 +1001,15 @@ struct hk_compute_state { struct hk_descriptor_state descriptors; struct hk_api_shader *shader; - /* Set by hk_CmdPipelineBarrier2, consumed by the next CDM launch, which - * then carries the full designed barrier set (dependency-tracked - * per-launch barrier). Lives on the command buffer, not the hk_cs, so it - * survives CS splits and merge_control_streams. + /* True while this command buffer holds CDM launches whose writes are + * still cached: the dependency-tracked barrier flushed neither at a + * vkCmdPipelineBarrier2 nor at a batch boundary. Set by each tracked + * launch, cleared wherever a CDM cache flush is emitted for the stream + * holding those launches (hk_cdm_flush_dirty). Lives on the command + * buffer, not the hk_cs, so it survives CS splits and + * merge_control_streams. */ - bool cdm_barrier_pending; + bool cdm_dirty; }; /* @@ -2290,12 +2293,16 @@ bool hk_g17p_smooth_varyings_supported( void *hk_cmd_buffer_emit_vdm_terminate(struct hk_cmd_buffer *cmd, void *current); +static inline void hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, + struct hk_cs *cs); + static void hk_cmd_buffer_end_compute_internal(struct hk_cmd_buffer *cmd, struct hk_cs **ptr) { if (*ptr) { struct hk_cs *cs = *ptr; + hk_cdm_flush_dirty(cmd, cs); if (unlikely(cs->query_reset_pending)) hk_flush_query_resets(cmd, cs); @@ -2556,6 +2563,21 @@ void hk_usc_upload_spilled_rt_descs(struct agx_usc_builder *b, void hk_cdm_cache_flush(struct hk_device *dev, struct hk_cs *cs); +/* Flush unflushed dependency-tracked CDM work (cmd->state.cs.cdm_dirty) at a + * visibility boundary: a vkCmdPipelineBarrier2, the end of the stream's + * batch, or hk_EndCommandBuffer. No-op when the command buffer holds no + * unflushed launches or the stream is not the compute stream they went to. + */ +static inline void +hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs) +{ + if (cs && cmd->state.cs.cdm_dirty && cs->type == HK_CS_CDM) { + hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE); + hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cs); + cmd->state.cs.cdm_dirty = false; + } +} + /* As hk_cdm_cache_flush(), for a flush an application dependency asks for. * With `memory`, the flush also has memory scope on G17P (M5 flushes always * do): needed for host availability, and for device dependencies once the diff --git a/src/asahi/vulkan/hk_cmd_dispatch.c b/src/asahi/vulkan/hk_cmd_dispatch.c index 69c0f249c8b3..4277521a6579 100644 --- a/src/asahi/vulkan/hk_cmd_dispatch.c +++ b/src/asahi/vulkan/hk_cmd_dispatch.c @@ -725,19 +725,17 @@ hk_dispatch_with_usc_launch(struct hk_device *dev, struct hk_cs *cs, return; } - /* Dependency-tracked per-launch CDM barrier. A vkCmdPipelineBarrier - * recorded since the previous launch declares a memory dependency, so the - * next launch carries the full designed barrier set. Between launches the - * app left unordered, Vulkan defines no ordering, so the barrier enforces - * nothing and the drain it costs is skipped. Driver-internal ordering that - * does not ride on vkCmdPipelineBarrier (query availability writes) calls + /* Dependency-tracked CDM barrier: this launch's writes stay cached, and + * cdm_dirty remembers them. The flush lands where Vulkan makes them + * visible: the next vkCmdPipelineBarrier2 in this command buffer, the + * end of the stream's batch, or hk_EndCommandBuffer (queue submission + * order orders the next submission after this one). Launches the app + * left unordered inside one command buffer keep skipping the drain + * entirely. Driver-internal ordering that does not ride on + * vkCmdPipelineBarrier (query availability writes) calls * hk_cdm_cache_flush explicitly and is unaffected. */ - if (cs->cmd->state.cs.cdm_barrier_pending) { - assert(cs->type == HK_CS_CDM); - hk_cdm_cache_flush(dev, cs); - cs->cmd->state.cs.cdm_barrier_pending = false; - } + cs->cmd->state.cs.cdm_dirty = true; cs->current = agx_cdm_launch(cs->current, dev->dev.chip, grid, wg, launch, usc); From b1d826b835b42217a2a1232099148ce5168a3d49 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:17:52 -0600 Subject: [PATCH 5/9] honeykrisp: flush every CDM stream when cdm_dirty transfer_src_transfer_dst still failed 3/8 after boundary flush: stale bytes on BufferCopyToImage when unflushed work sat on a non-current CDM chunk in control_streams. Flush all CDM streams (current + listed) before clearing cdm_dirty; also flush at gfx-open pipeline barriers before end_compute. --- src/asahi/vulkan/hk_cmd_buffer.c | 3 ++- src/asahi/vulkan/hk_cmd_buffer.h | 28 +++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/asahi/vulkan/hk_cmd_buffer.c b/src/asahi/vulkan/hk_cmd_buffer.c index 233013a2a7d4..28a0c306372f 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.c +++ b/src/asahi/vulkan/hk_cmd_buffer.c @@ -1115,7 +1115,7 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, * stays open: merge_control_streams stitches it back together * at hk_EndCommandBuffer, without a stream link per barrier. */ - hk_cdm_flush_dirty(cmd, cmd->current_cs.cs); + hk_cdm_flush_all_dirty(cmd); /* Preserve the same conservative ordering in the shared submit * graph, including kernels that advertise exact prior barriers @@ -1125,6 +1125,7 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, } } + hk_cdm_flush_all_dirty(cmd); hk_cmd_buffer_end_compute(cmd); hk_cmd_buffer_end_graphics(cmd); /* Preserve the same conservative ordering in the shared submit graph, diff --git a/src/asahi/vulkan/hk_cmd_buffer.h b/src/asahi/vulkan/hk_cmd_buffer.h index 0722138f2e80..795fea4ec723 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.h +++ b/src/asahi/vulkan/hk_cmd_buffer.h @@ -2295,6 +2295,7 @@ void *hk_cmd_buffer_emit_vdm_terminate(struct hk_cmd_buffer *cmd, static inline void hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs); +static inline void hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd); static void hk_cmd_buffer_end_compute_internal(struct hk_cmd_buffer *cmd, @@ -2302,7 +2303,7 @@ hk_cmd_buffer_end_compute_internal(struct hk_cmd_buffer *cmd, { if (*ptr) { struct hk_cs *cs = *ptr; - hk_cdm_flush_dirty(cmd, cs); + hk_cdm_flush_all_dirty(cmd); if (unlikely(cs->query_reset_pending)) hk_flush_query_resets(cmd, cs); @@ -2571,11 +2572,28 @@ void hk_cdm_cache_flush(struct hk_device *dev, struct hk_cs *cs); static inline void hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs) { - if (cs && cmd->state.cs.cdm_dirty && cs->type == HK_CS_CDM) { - hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE); - hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cs); - cmd->state.cs.cdm_dirty = false; + if (!cs || !cmd->state.cs.cdm_dirty || cs->type != HK_CS_CDM) + return; + + hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE); + hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cs); +} + +static inline void +hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd) +{ + if (!cmd->state.cs.cdm_dirty) + return; + + if (cmd->current_cs.cs) + hk_cdm_flush_dirty(cmd, cmd->current_cs.cs); + + list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) { + if (cs != cmd->current_cs.cs) + hk_cdm_flush_dirty(cmd, cs); } + + cmd->state.cs.cdm_dirty = false; } /* As hk_cdm_cache_flush(), for a flush an application dependency asks for. From 2e0906b7c5bfc65ba1de41a7033670620f42fe66 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:40:01 -0600 Subject: [PATCH 6/9] honeykrisp: CDM xfer visibility on G14 dependency-tracked barriers --- src/asahi/vulkan/hk_cmd_buffer.c | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/asahi/vulkan/hk_cmd_buffer.c b/src/asahi/vulkan/hk_cmd_buffer.c index 28a0c306372f..10abc0e0697f 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.c +++ b/src/asahi/vulkan/hk_cmd_buffer.c @@ -1059,6 +1059,45 @@ hk_classify_pass_barrier(struct hk_cmd_buffer *cmd, const VkDependencyInfo *info return hk_barrier_classify(info, &state); } +/* xfer/compute→* barriers need a CDM visibility record in the stream that + * holds the source. The G14 dependency-tracked fast path used to skip this + * and only flush through cdm_dirty. */ +static void +hk_cdm_xfer_visibility_at_barrier(struct hk_cmd_buffer *cmd, + const VkDependencyInfo *dep) +{ + struct hk_device *dev = hk_cmd_buffer_device(cmd); + uint8_t src_kinds, dst_kinds; + + hk_sync_dependency_kinds(dep, hk_sync_image_class(cmd), &src_kinds, + &dst_kinds); + if (!(src_kinds & (HK_SYNC_COMPUTE | HK_SYNC_XFER)) || !dst_kinds) + return; + + const bool alias_scope = hk_cdm_alias_scope(dev); + struct hk_cs *list[16]; + unsigned n = 0; + + if (cmd->current_cs.cs && n < ARRAY_SIZE(list)) + list[n++] = cmd->current_cs.cs; + list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) { + if (cs == cmd->current_cs.cs || n >= ARRAY_SIZE(list)) + continue; + list[n++] = cs; + } + + for (unsigned i = 0; i < n; i++) { + struct hk_cs *cs = list[i]; + + if (cs->type != HK_CS_CDM || + !hk_cdm_visibility_flush_needed(dev, cs, alias_scope)) + continue; + + hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE); + hk_cdm_visibility_flush(dev, cs, alias_scope); + } +} + VKAPI_ATTR void VKAPI_CALL hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, const VkDependencyInfo *pDependencyInfo) @@ -1116,6 +1155,7 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, * at hk_EndCommandBuffer, without a stream link per barrier. */ hk_cdm_flush_all_dirty(cmd); + hk_cdm_xfer_visibility_at_barrier(cmd, pDependencyInfo); /* Preserve the same conservative ordering in the shared submit * graph, including kernels that advertise exact prior barriers @@ -1126,6 +1166,7 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, } hk_cdm_flush_all_dirty(cmd); + hk_cdm_xfer_visibility_at_barrier(cmd, pDependencyInfo); hk_cmd_buffer_end_compute(cmd); hk_cmd_buffer_end_graphics(cmd); /* Preserve the same conservative ordering in the shared submit graph, From a576a4e984d4a7972f7541b74ef9c691ea0afc3b Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:27:46 -0600 Subject: [PATCH 7/9] honeykrisp: track AGX2 CDM strong flush end like AGX3 After agx_cdm_barrier on AGX2 command streams set strong_flush_end so hk_cdm_visibility_flush_needed() skips a redundant second barrier when hk_cdm_flush_all_dirty() and xfer visibility run at the same pipeline barrier. --- src/asahi/vulkan/hk_cmd_dispatch.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/asahi/vulkan/hk_cmd_dispatch.c b/src/asahi/vulkan/hk_cmd_dispatch.c index 4277521a6579..897d42f1b96e 100644 --- a/src/asahi/vulkan/hk_cmd_dispatch.c +++ b/src/asahi/vulkan/hk_cmd_dispatch.c @@ -590,6 +590,9 @@ hk_cdm_flush(struct hk_device *dev, struct hk_cs *cs, bool memory) } else { cs->current = agx_cdm_barrier(cs->current, dev->dev.chip); } + /* Record the trailing flush so a back-to-back barrier request at a + * declared dependency can reuse it (G15+ already did this). */ + cs->strong_flush_end = cs->current; break; case AGX_CS_GENERATION_G15: From 3d5db9af624cf560c4863b43c15be11c6c2ca60b Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:50:52 -0600 Subject: [PATCH 8/9] honeykrisp: move hk_cdm_flush_dirty out of header for unit tests Inline hk_cdm_flush_dirty in hk_cmd_buffer.h pulled hk_ensure_cs_has_space and hk_cdm_cache_flush into hk_sync_prior_test and hk_query_oq_test, which only link hk_sync.c / hk_query_oq.c. Provide hk_cdm_flush_all_dirty in hk_cdm_flush_dirty.c and no-op stubs in those CPU-only fixtures; link the flush helper from hk_query_reset_test. --- src/asahi/vulkan/hk_cdm_flush_dirty.c | 39 +++++++++++++++++++++ src/asahi/vulkan/hk_cmd_buffer.h | 37 ++----------------- src/asahi/vulkan/meson.build | 4 ++- src/asahi/vulkan/tests/test-hk-query-oq.c | 6 ++++ src/asahi/vulkan/tests/test-hk-sync-prior.c | 6 ++++ 5 files changed, 56 insertions(+), 36 deletions(-) create mode 100644 src/asahi/vulkan/hk_cdm_flush_dirty.c diff --git a/src/asahi/vulkan/hk_cdm_flush_dirty.c b/src/asahi/vulkan/hk_cdm_flush_dirty.c new file mode 100644 index 000000000000..53e501607c8f --- /dev/null +++ b/src/asahi/vulkan/hk_cdm_flush_dirty.c @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Aurora Silicon + * SPDX-License-Identifier: MIT + */ + +#include "hk_cmd_buffer.h" +#include "hk_device.h" + +/* Flush unflushed dependency-tracked CDM work (cmd->state.cs.cdm_dirty) at a + * visibility boundary: a vkCmdPipelineBarrier2, the end of the stream's + * batch, or hk_EndCommandBuffer. No-op when the command buffer holds no + * unflushed launches or the stream is not the compute stream they went to. + */ +void +hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs) +{ + if (!cs || !cmd->state.cs.cdm_dirty || cs->type != HK_CS_CDM) + return; + + hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE); + hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cs); +} + +void +hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd) +{ + if (!cmd->state.cs.cdm_dirty) + return; + + if (cmd->current_cs.cs) + hk_cdm_flush_dirty(cmd, cmd->current_cs.cs); + + list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) { + if (cs != cmd->current_cs.cs) + hk_cdm_flush_dirty(cmd, cs); + } + + cmd->state.cs.cdm_dirty = false; +} diff --git a/src/asahi/vulkan/hk_cmd_buffer.h b/src/asahi/vulkan/hk_cmd_buffer.h index 795fea4ec723..8ea6e6aeee0b 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.h +++ b/src/asahi/vulkan/hk_cmd_buffer.h @@ -2293,10 +2293,6 @@ bool hk_g17p_smooth_varyings_supported( void *hk_cmd_buffer_emit_vdm_terminate(struct hk_cmd_buffer *cmd, void *current); -static inline void hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, - struct hk_cs *cs); -static inline void hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd); - static void hk_cmd_buffer_end_compute_internal(struct hk_cmd_buffer *cmd, struct hk_cs **ptr) @@ -2564,37 +2560,8 @@ void hk_usc_upload_spilled_rt_descs(struct agx_usc_builder *b, void hk_cdm_cache_flush(struct hk_device *dev, struct hk_cs *cs); -/* Flush unflushed dependency-tracked CDM work (cmd->state.cs.cdm_dirty) at a - * visibility boundary: a vkCmdPipelineBarrier2, the end of the stream's - * batch, or hk_EndCommandBuffer. No-op when the command buffer holds no - * unflushed launches or the stream is not the compute stream they went to. - */ -static inline void -hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs) -{ - if (!cs || !cmd->state.cs.cdm_dirty || cs->type != HK_CS_CDM) - return; - - hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE); - hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cs); -} - -static inline void -hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd) -{ - if (!cmd->state.cs.cdm_dirty) - return; - - if (cmd->current_cs.cs) - hk_cdm_flush_dirty(cmd, cmd->current_cs.cs); - - list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) { - if (cs != cmd->current_cs.cs) - hk_cdm_flush_dirty(cmd, cs); - } - - cmd->state.cs.cdm_dirty = false; -} +void hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs); +void hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd); /* As hk_cdm_cache_flush(), for a flush an application dependency asks for. * With `memory`, the flush also has memory scope on G17P (M5 flushes always diff --git a/src/asahi/vulkan/meson.build b/src/asahi/vulkan/meson.build index 7519b6e2716f..ec95e2800f94 100644 --- a/src/asahi/vulkan/meson.build +++ b/src/asahi/vulkan/meson.build @@ -15,6 +15,7 @@ hk_files = files( 'hk_buffer_view.h', 'hk_bvh.h', 'hk_cmd_buffer.c', + 'hk_cdm_flush_dirty.c', 'hk_cmd_upload.c', 'hk_cmd_buffer.h', 'hk_conditional_render_core.h', @@ -186,7 +187,8 @@ if with_tests 'hk_query_reset', executable( 'hk_query_reset_test', - files('tests/test-hk-query-reset.c', 'hk_query_reset.c'), + files('tests/test-hk-query-reset.c', 'hk_query_reset.c', + 'hk_cdm_flush_dirty.c'), include_directories : [ include_directories('.'), inc_gallium, inc_gallium_aux, inc_include, inc_src, inc_asahi, diff --git a/src/asahi/vulkan/tests/test-hk-query-oq.c b/src/asahi/vulkan/tests/test-hk-query-oq.c index b6d914c9b5d1..23567773828e 100644 --- a/src/asahi/vulkan/tests/test-hk-query-oq.c +++ b/src/asahi/vulkan/tests/test-hk-query-oq.c @@ -54,6 +54,12 @@ hk_flush_query_resets(struct hk_cmd_buffer *cmd, struct hk_cs *cs) abort(); } +void +hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd) +{ + (void)cmd; +} + void hk_g17p_meta_fill_addr(struct hk_cmd_buffer *cmd, struct hk_cs **target, uint64_t address, uint64_t size, uint32_t value) diff --git a/src/asahi/vulkan/tests/test-hk-sync-prior.c b/src/asahi/vulkan/tests/test-hk-sync-prior.c index fdc5e2c6e499..6a333998a971 100644 --- a/src/asahi/vulkan/tests/test-hk-sync-prior.c +++ b/src/asahi/vulkan/tests/test-hk-sync-prior.c @@ -249,6 +249,12 @@ hk_flush_query_resets(struct hk_cmd_buffer *cmd, struct hk_cs *cs) abort(); } +void +hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd) +{ + (void)cmd; +} + /* Timestamp recording references payload/allocation entry points for other * branches. These tests attach end stamps to already-created closed streams; * reaching a payload emission branch would invalidate that fixture contract. From 8b744aaf814746ec5d3948cf6f540b02428d028b Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:52:35 -0600 Subject: [PATCH 9/9] honeykrisp: declare hk_cdm_flush_all_dirty before inline consumer --- src/asahi/vulkan/hk_cmd_buffer.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/asahi/vulkan/hk_cmd_buffer.h b/src/asahi/vulkan/hk_cmd_buffer.h index 8ea6e6aeee0b..767222a79acb 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.h +++ b/src/asahi/vulkan/hk_cmd_buffer.h @@ -2293,6 +2293,8 @@ bool hk_g17p_smooth_varyings_supported( void *hk_cmd_buffer_emit_vdm_terminate(struct hk_cmd_buffer *cmd, void *current); +void hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd); + static void hk_cmd_buffer_end_compute_internal(struct hk_cmd_buffer *cmd, struct hk_cs **ptr)