Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 78 additions & 13 deletions src/asahi/libagx/libagx_dgc.h
Original file line number Diff line number Diff line change
Expand Up @@ -602,9 +602,29 @@ agx_cdm_barrier(GLOBAL uint32_t *out, enum agx_chip chip)
// cfg.unk_20 = true;
// cfg.unk_24 = true; if clustered?
if (chip == AGX_CHIP_G13X) {
/* G13X (t600x, M1 Pro/Max, G13C cores): restore bits 0-2
* (PBE/texture flush) alongside unk_4 + usc_cache_inval.
* Bits 0-2 are required for cross-launch coherency when
* control streams are not split and across dependency-skipped
* launches ({0,1,2,4,5,6,8}+usc_cache_inval = 0x17f). */
cfg.unk_0 = true;
cfg.unk_1 = true;
cfg.unk_2 = true;
cfg.unk_4 = true;
cfg.usc_cache_inval = true;
// cfg.unk_26 = true;
}
if (chip == AGX_CHIP_G13G) {
/* G13G (t8103, M1): the kitchen-sink bits below cost ~10 us per
* dispatch in real dependent compute chains (Qwen decode: ~2.1 ms
* of a ~9.8 ms token). Bits 4-8 hold the mlx-omarchy pinned
* generated-ID digests (48/48 interleaved runs, both legs) and
* the omarchy runtime suite (22 cases / 6189 assertions), and
* measure +3.05% ctx1053 decode on jwm1. Other chips keep the
* full sink until measured there. */
cfg.unk_4 = true;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This changes the G13G barrier for Vulkan and GL (agx_state.c:4832), and alwayscdmbarrier doesn't restore it. Commit 1 says G13G keeps the full barrier. Could this move to its own PR? #6 looks like it covers it.

— Marcelo's agent

cfg.unk_7 = true;
}

/* With multiple launches in the same CDM stream, we can get cache
* coherency (? or sync?) issues. We hit this with blits, which need - in
Expand All @@ -613,26 +633,71 @@ agx_cdm_barrier(GLOBAL uint32_t *out, enum agx_chip chip)
* let's just set these after every launch to be safe. We can revisit in
* the future when we figure out what the bits mean.
*/
if (chip != AGX_CHIP_G13G && chip != AGX_CHIP_G13X) {
cfg.unk_0 = true;
cfg.unk_1 = true;
cfg.unk_2 = true;
cfg.usc_cache_inval = true;
cfg.unk_4 = true;
cfg.unk_5 = true;
cfg.unk_6 = true;
cfg.unk_7 = true;
cfg.unk_8 = true;
cfg.unk_9 = true;
cfg.unk_10 = true;
cfg.unk_11 = true;
cfg.unk_12 = true;
cfg.unk_13 = true;
cfg.unk_14 = true;
cfg.unk_15 = true;
cfg.unk_16 = true;
cfg.unk_17 = true;
cfg.unk_18 = true;
cfg.unk_19 = true;
}
}

return out;
}

/*
* Minimal variant of agx_cdm_barrier: invalidate the USC cache only. Used by
* the Vulkan driver for launches whose results are consumed by another compute
* launch through the same (L2-backed) storage path, where the PBE/texture
* cache maintenance of the full barrier is not required.
*/
static inline GLOBAL uint32_t *
agx_cdm_barrier_usc(GLOBAL uint32_t *out)
{
agx_push(out, CDM_BARRIER, cfg) {
cfg.usc_cache_inval = true;
}

return out;
}

/*
* G13X candidate reduction: the designed set {unk_4, unk_5, unk_6, unk_8}
* (the bits that held mlx-omarchy pinned digests 48/48 in the termA
* battery) plus the USC cache invalidate (covers uniform/texture-state
* changes between launches; measured 4.6 us/launch of the 20.4 us
* sink+cache cost on real dependent chains). The full-sink unk_0..unk_19
* block (PBE/texture cache maintenance, measured 15.8 us/launch) is
* dropped. Perftest arm designedusccdmbarrier; NOT a default-emission
* proposal until it holds both decode legs and the digest protocol.
*/
static inline GLOBAL uint32_t *
agx_cdm_barrier_designed_usc(GLOBAL uint32_t *out)
{
agx_push(out, CDM_BARRIER, cfg) {
cfg.unk_0 = true;
cfg.unk_1 = true;
cfg.unk_2 = true;
cfg.usc_cache_inval = true;
cfg.unk_4 = true;
cfg.unk_5 = true;
cfg.unk_6 = true;
cfg.unk_7 = true;
cfg.unk_8 = true;
cfg.unk_9 = true;
cfg.unk_10 = true;
cfg.unk_11 = true;
cfg.unk_12 = true;
cfg.unk_13 = true;
cfg.unk_14 = true;
cfg.unk_15 = true;
cfg.unk_16 = true;
cfg.unk_17 = true;
cfg.unk_18 = true;
cfg.unk_19 = true;
cfg.usc_cache_inval = true;
}

return out;
Expand Down
39 changes: 39 additions & 0 deletions src/asahi/vulkan/hk_cdm_flush_dirty.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright 2026 Aurora Silicon
* SPDX-License-Identifier: MIT
*/

#include "hk_cmd_buffer.h"
#include "hk_device.h"

/* Flush unflushed dependency-tracked CDM work (cmd->state.cs.cdm_dirty) at a
* visibility boundary: a vkCmdPipelineBarrier2, the end of the stream's
* batch, or hk_EndCommandBuffer. No-op when the command buffer holds no
* unflushed launches or the stream is not the compute stream they went to.
*/
void
hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
{
if (!cs || !cmd->state.cs.cdm_dirty || cs->type != HK_CS_CDM)
return;

hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE);
hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cs);
}

void
hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd)
{
if (!cmd->state.cs.cdm_dirty)
return;

if (cmd->current_cs.cs)
hk_cdm_flush_dirty(cmd, cmd->current_cs.cs);

list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) {
if (cs != cmd->current_cs.cs)
hk_cdm_flush_dirty(cmd, cs);
}

cmd->state.cs.cdm_dirty = false;
}
69 changes: 69 additions & 0 deletions src/asahi/vulkan/hk_cmd_buffer.c
Original file line number Diff line number Diff line change
Expand Up @@ -957,6 +957,7 @@ hk_EndCommandBuffer(VkCommandBuffer commandBuffer)
perf_debug(cmd, "End command buffer");
hk_flush_deferred_clears(cmd);
hk_sync_flush_pending_timestamp(cmd);

hk_cmd_buffer_end_compute(cmd);
hk_cmd_buffer_end_compute_internal(cmd, &cmd->current_cs.post_gfx);
hk_resolve_deferred_oq_available(cmd);
Expand Down Expand Up @@ -1058,6 +1059,45 @@ hk_classify_pass_barrier(struct hk_cmd_buffer *cmd, const VkDependencyInfo *info
return hk_barrier_classify(info, &state);
}

/* xfer/compute→* barriers need a CDM visibility record in the stream that
* holds the source. The G14 dependency-tracked fast path used to skip this
* and only flush through cdm_dirty. */
static void
hk_cdm_xfer_visibility_at_barrier(struct hk_cmd_buffer *cmd,
const VkDependencyInfo *dep)
{
struct hk_device *dev = hk_cmd_buffer_device(cmd);
uint8_t src_kinds, dst_kinds;

hk_sync_dependency_kinds(dep, hk_sync_image_class(cmd), &src_kinds,
&dst_kinds);
if (!(src_kinds & (HK_SYNC_COMPUTE | HK_SYNC_XFER)) || !dst_kinds)
return;

const bool alias_scope = hk_cdm_alias_scope(dev);
struct hk_cs *list[16];
unsigned n = 0;

if (cmd->current_cs.cs && n < ARRAY_SIZE(list))
list[n++] = cmd->current_cs.cs;
list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) {
if (cs == cmd->current_cs.cs || n >= ARRAY_SIZE(list))
continue;
list[n++] = cs;
}

for (unsigned i = 0; i < n; i++) {
struct hk_cs *cs = list[i];

if (cs->type != HK_CS_CDM ||
!hk_cdm_visibility_flush_needed(dev, cs, alias_scope))
continue;

hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE);
hk_cdm_visibility_flush(dev, cs, alias_scope);
}
}

VKAPI_ATTR void VKAPI_CALL
hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer,
const VkDependencyInfo *pDependencyInfo)
Expand Down Expand Up @@ -1098,6 +1138,35 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer,

/* M5 records its barriers with the native dependency tracker. */
if (!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip)) {
if (!HK_PERF(dev, ALWAYSCDMBARRIER)) {
/* Outside a render pass only a CDM control stream can be open; the
* batches must end when a graphics batch is open: the dependency
* then spans batch types and the kernel orders the resulting
* commands (agx_cmd_header barriers). */
if (!cmd->current_cs.gfx && !cmd->current_cs.pre_gfx &&

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Keeping the stream open defers the flush to the next CDM launch, which comes after any render that consumes this stream. Could the pending barrier land on the trailing edge when the stream ends?

— Marcelo's agent

!cmd->current_cs.post_gfx) {
/* The app declared a memory dependency: flush unflushed CDM work
* here, at the visibility point, in the stream that holds it.
* Deferring the record to the next CDM launch loses it when the
* consumer is graphics or was recorded in another command
* buffer (ExecuteSecondaryCommandBuffer); dEQP
* memory.pipeline_barrier.transfer_* fails that way. The batch
* stays open: merge_control_streams stitches it back together
* at hk_EndCommandBuffer, without a stream link per barrier.
*/
hk_cdm_flush_all_dirty(cmd);
hk_cdm_xfer_visibility_at_barrier(cmd, pDependencyInfo);

/* Preserve the same conservative ordering in the shared submit
* graph, including kernels that advertise exact prior barriers
* on AGX2. */
hk_sync_barrier(cmd, HK_SYNC_ALL, HK_SYNC_ALL);
return;
}
}

hk_cdm_flush_all_dirty(cmd);
hk_cdm_xfer_visibility_at_barrier(cmd, pDependencyInfo);
hk_cmd_buffer_end_compute(cmd);
hk_cmd_buffer_end_graphics(cmd);
/* Preserve the same conservative ordering in the shared submit graph,
Expand Down
16 changes: 16 additions & 0 deletions src/asahi/vulkan/hk_cmd_buffer.h
Original file line number Diff line number Diff line change
Expand Up @@ -1000,6 +1000,16 @@ hk_m5_fragment_root_select(struct hk_graphics_state *gfx, uint8_t root_reads,
struct hk_compute_state {
struct hk_descriptor_state descriptors;
struct hk_api_shader *shader;

/* True while this command buffer holds CDM launches whose writes are
* still cached: the dependency-tracked barrier flushed neither at a
* vkCmdPipelineBarrier2 nor at a batch boundary. Set by each tracked
* launch, cleared wherever a CDM cache flush is emitted for the stream
* holding those launches (hk_cdm_flush_dirty). Lives on the command
* buffer, not the hk_cs, so it survives CS splits and
* merge_control_streams.
*/
bool cdm_dirty;
};

/*
Expand Down Expand Up @@ -2283,12 +2293,15 @@ bool hk_g17p_smooth_varyings_supported(
void *hk_cmd_buffer_emit_vdm_terminate(struct hk_cmd_buffer *cmd,
void *current);

void hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd);

static void
hk_cmd_buffer_end_compute_internal(struct hk_cmd_buffer *cmd,
struct hk_cs **ptr)
{
if (*ptr) {
struct hk_cs *cs = *ptr;
hk_cdm_flush_all_dirty(cmd);
if (unlikely(cs->query_reset_pending))
hk_flush_query_resets(cmd, cs);

Expand Down Expand Up @@ -2549,6 +2562,9 @@ void hk_usc_upload_spilled_rt_descs(struct agx_usc_builder *b,

void hk_cdm_cache_flush(struct hk_device *dev, struct hk_cs *cs);

void hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs);
void hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd);

/* As hk_cdm_cache_flush(), for a flush an application dependency asks for.
* With `memory`, the flush also has memory scope on G17P (M5 flushes always
* do): needed for host availability, and for device dependencies once the
Expand Down
43 changes: 40 additions & 3 deletions src/asahi/vulkan/hk_cmd_dispatch.c
Original file line number Diff line number Diff line change
Expand Up @@ -573,7 +573,26 @@ hk_cdm_flush(struct hk_device *dev, struct hk_cs *cs, bool memory)

switch (agx_chip_command_stream_generation(dev->dev.chip)) {
case AGX_CS_GENERATION_AGX2:
cs->current = agx_cdm_barrier(cs->current, dev->dev.chip);
if (unlikely(dev->cdm_barrier_mask)) {
/* Perftest only: exact CDM_BARRIER flag word from the environment. */
uint32_t *word = cs->current;
*word = dev->cdm_barrier_mask | (AGX_CDM_BLOCK_TYPE_BARRIER << 29);
cs->current = word + 1;
} else if (HK_PERF(dev, NOCDMBARRIER)) {
/* Perftest only: launch with no cache maintenance at all. */
return;
} else if (HK_PERF(dev, DESIGNEDUSCCDMBARRIER)) {
/* Perftest only: designed set {4,5,6,8} + USC cache invalidate. */
cs->current = agx_cdm_barrier_designed_usc(cs->current);
} else if (HK_PERF(dev, USCCDMBARRIER)) {
/* Perftest only: keep the USC cache invalidate, drop the rest. */
cs->current = agx_cdm_barrier_usc(cs->current);
} else {
cs->current = agx_cdm_barrier(cs->current, dev->dev.chip);
}
/* Record the trailing flush so a back-to-back barrier request at a
* declared dependency can reuse it (G15+ already did this). */
cs->strong_flush_end = cs->current;
break;

case AGX_CS_GENERATION_G15:
Expand Down Expand Up @@ -701,10 +720,28 @@ hk_dispatch_with_usc_launch(struct hk_device *dev, struct hk_cs *cs,

hk_ensure_cs_has_space(cs->cmd, cs, 0x2000 /* TODO */);
cs->stats.cmds++;

if (HK_PERF(dev, ALWAYSCDMBARRIER)) {
cs->current =
agx_cdm_launch(cs->current, dev->dev.chip, grid, wg, launch, usc);
hk_cdm_cache_flush(dev, cs);
return;
}

/* Dependency-tracked CDM barrier: this launch's writes stay cached, and
* cdm_dirty remembers them. The flush lands where Vulkan makes them
* visible: the next vkCmdPipelineBarrier2 in this command buffer, the
* end of the stream's batch, or hk_EndCommandBuffer (queue submission
* order orders the next submission after this one). Launches the app
* left unordered inside one command buffer keep skipping the drain
* entirely. Driver-internal ordering that does not ride on
* vkCmdPipelineBarrier (query availability writes) calls
* hk_cdm_cache_flush explicitly and is unaffected.
*/
cs->cmd->state.cs.cdm_dirty = true;

cs->current =
agx_cdm_launch(cs->current, dev->dev.chip, grid, wg, launch, usc);

hk_cdm_cache_flush(dev, cs);
}

void
Expand Down
30 changes: 30 additions & 0 deletions src/asahi/vulkan/hk_device.c
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,15 @@ static const struct debug_named_value hk_perf_options[] = {
{"nobarrier", HK_PERF_NOBARRIER,"Ignore pipeline barriers"},
{"nobatch", HK_PERF_NOBATCH, "Disable submission batching"},
{"norobust", HK_PERF_NOROBUST, "Disable robustness"},
{"nocdmbarrier", HK_PERF_NOCDMBARRIER,
"Omit the per-launch CDM barrier entirely"},
{"usccdmbarrier", HK_PERF_USCCDMBARRIER,
"Reduce the per-launch CDM barrier to a USC cache invalidate"},
{"designedusccdmbarrier", HK_PERF_DESIGNEDUSCCDMBARRIER,
"G13X designed set {0,1,2,4,5,6,8} plus USC cache invalidate (0x17f)"},
{"alwayscdmbarrier", HK_PERF_ALWAYSCDMBARRIER, "Unconditional per-launch CDM barrier (disable dependency tracking)"},
{"trackcdmbarrier", HK_PERF_TRACKCDMBARRIER,
"Dependency-track the CDM barrier on chips other than G13"},
DEBUG_NAMED_VALUE_END
};
/* clang-format on */
Expand Down Expand Up @@ -867,6 +876,27 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
}

dev->perftest = debug_get_flags_option("HK_PERFTEST", hk_perf_options, 0);
dev->cdm_barrier_mask =
debug_get_num_option("HK_CDM_BARRIER_MASK", 0) & 0x07ffffff;

/* The dependency-tracked CDM barrier holds token-digest parity with the
* reference driver on G13 (M1 family) and on G14X of generation 14 (M2
* Pro/Max/Ultra class; verified on M2 Max: greedy token digests of three
* LLM decode runs of 2048 tokens, CTS compute groups and the
* floating-point gates equal to the per-launch barrier). Other chips,
* including multi-cluster generation 15 and later parts that also map to
* G14X, keep the per-launch barrier until they pass the same gate;
* HK_PERFTEST=trackcdmbarrier opts them in for that run and
* HK_PERFTEST=alwayscdmbarrier forces the per-launch barrier everywhere.
*/
bool tracked_by_default =

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: G13G and gen-14 parts other than M2 Max haven't been measured on this branch. Worth limiting the default to what passed the digest gate.

— Marcelo's agent

pdev->dev.chip == AGX_CHIP_G13G || pdev->dev.chip == AGX_CHIP_G13X ||
(pdev->dev.chip == AGX_CHIP_G14X &&
pdev->dev.params.gpu_generation == 14);

if (!tracked_by_default && !HK_PERF(dev, TRACKCDMBARRIER))
dev->perftest |= HK_PERF_ALWAYSCDMBARRIER;


if (instance->drirc.misc.disable_border_emulation) {
dev->perftest |= HK_PERF_NOBORDER;
Expand Down
Loading
Loading