Repository navigation
honeykrisp: dependency-tracked CDM barrier on G13 and G14X (M2 Max decode +19% on the shipped driver) #47
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: integration/aurora
Are you sure you want to change the base?
Changes from all commits
6fa9fed
338b5cf
ab5d6e9
5fca791
b1d826b
2e0906b
a576a4e
3d5db9a
8b744aa
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| /* | ||
| * Copyright 2026 Aurora Silicon | ||
| * SPDX-License-Identifier: MIT | ||
| */ | ||
|
|
||
| #include "hk_cmd_buffer.h" | ||
| #include "hk_device.h" | ||
|
|
||
| /* Flush unflushed dependency-tracked CDM work (cmd->state.cs.cdm_dirty) at a | ||
| * visibility boundary: a vkCmdPipelineBarrier2, the end of the stream's | ||
| * batch, or hk_EndCommandBuffer. No-op when the command buffer holds no | ||
| * unflushed launches or the stream is not the compute stream they went to. | ||
| */ | ||
| void | ||
| hk_cdm_flush_dirty(struct hk_cmd_buffer *cmd, struct hk_cs *cs) | ||
| { | ||
| if (!cs || !cmd->state.cs.cdm_dirty || cs->type != HK_CS_CDM) | ||
| return; | ||
|
|
||
| hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE); | ||
| hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cs); | ||
| } | ||
|
|
||
| void | ||
| hk_cdm_flush_all_dirty(struct hk_cmd_buffer *cmd) | ||
| { | ||
| if (!cmd->state.cs.cdm_dirty) | ||
| return; | ||
|
|
||
| if (cmd->current_cs.cs) | ||
| hk_cdm_flush_dirty(cmd, cmd->current_cs.cs); | ||
|
|
||
| list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) { | ||
| if (cs != cmd->current_cs.cs) | ||
| hk_cdm_flush_dirty(cmd, cs); | ||
| } | ||
|
|
||
| cmd->state.cs.cdm_dirty = false; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -957,6 +957,7 @@ hk_EndCommandBuffer(VkCommandBuffer commandBuffer) | |
| perf_debug(cmd, "End command buffer"); | ||
| hk_flush_deferred_clears(cmd); | ||
| hk_sync_flush_pending_timestamp(cmd); | ||
|
|
||
| hk_cmd_buffer_end_compute(cmd); | ||
| hk_cmd_buffer_end_compute_internal(cmd, &cmd->current_cs.post_gfx); | ||
| hk_resolve_deferred_oq_available(cmd); | ||
|
|
@@ -1058,6 +1059,45 @@ hk_classify_pass_barrier(struct hk_cmd_buffer *cmd, const VkDependencyInfo *info | |
| return hk_barrier_classify(info, &state); | ||
| } | ||
|
|
||
| /* xfer/compute→* barriers need a CDM visibility record in the stream that | ||
| * holds the source. The G14 dependency-tracked fast path used to skip this | ||
| * and only flush through cdm_dirty. */ | ||
| static void | ||
| hk_cdm_xfer_visibility_at_barrier(struct hk_cmd_buffer *cmd, | ||
| const VkDependencyInfo *dep) | ||
| { | ||
| struct hk_device *dev = hk_cmd_buffer_device(cmd); | ||
| uint8_t src_kinds, dst_kinds; | ||
|
|
||
| hk_sync_dependency_kinds(dep, hk_sync_image_class(cmd), &src_kinds, | ||
| &dst_kinds); | ||
| if (!(src_kinds & (HK_SYNC_COMPUTE | HK_SYNC_XFER)) || !dst_kinds) | ||
| return; | ||
|
|
||
| const bool alias_scope = hk_cdm_alias_scope(dev); | ||
| struct hk_cs *list[16]; | ||
| unsigned n = 0; | ||
|
|
||
| if (cmd->current_cs.cs && n < ARRAY_SIZE(list)) | ||
| list[n++] = cmd->current_cs.cs; | ||
| list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) { | ||
| if (cs == cmd->current_cs.cs || n >= ARRAY_SIZE(list)) | ||
| continue; | ||
| list[n++] = cs; | ||
| } | ||
|
|
||
| for (unsigned i = 0; i < n; i++) { | ||
| struct hk_cs *cs = list[i]; | ||
|
|
||
| if (cs->type != HK_CS_CDM || | ||
| !hk_cdm_visibility_flush_needed(dev, cs, alias_scope)) | ||
| continue; | ||
|
|
||
| hk_ensure_cs_has_space(cmd, cs, AGX3_CDM_BARRIER_SIZE); | ||
| hk_cdm_visibility_flush(dev, cs, alias_scope); | ||
| } | ||
| } | ||
|
|
||
| VKAPI_ATTR void VKAPI_CALL | ||
| hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, | ||
| const VkDependencyInfo *pDependencyInfo) | ||
|
|
@@ -1098,6 +1138,35 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer, | |
|
|
||
| /* M5 records its barriers with the native dependency tracker. */ | ||
| if (!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip)) { | ||
| if (!HK_PERF(dev, ALWAYSCDMBARRIER)) { | ||
| /* Outside a render pass only a CDM control stream can be open; the | ||
| * batches must end when a graphics batch is open: the dependency | ||
| * then spans batch types and the kernel orders the resulting | ||
| * commands (agx_cmd_header barriers). */ | ||
| if (!cmd->current_cs.gfx && !cmd->current_cs.pre_gfx && | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Keeping the stream open defers the flush to the next CDM launch, which comes after any render that consumes this stream. Could the pending barrier land on the trailing edge when the stream ends? — Marcelo's agent |
||
| !cmd->current_cs.post_gfx) { | ||
| /* The app declared a memory dependency: flush unflushed CDM work | ||
| * here, at the visibility point, in the stream that holds it. | ||
| * Deferring the record to the next CDM launch loses it when the | ||
| * consumer is graphics or was recorded in another command | ||
| * buffer (ExecuteSecondaryCommandBuffer); dEQP | ||
| * memory.pipeline_barrier.transfer_* fails that way. The batch | ||
| * stays open: merge_control_streams stitches it back together | ||
| * at hk_EndCommandBuffer, without a stream link per barrier. | ||
| */ | ||
| hk_cdm_flush_all_dirty(cmd); | ||
| hk_cdm_xfer_visibility_at_barrier(cmd, pDependencyInfo); | ||
|
|
||
| /* Preserve the same conservative ordering in the shared submit | ||
| * graph, including kernels that advertise exact prior barriers | ||
| * on AGX2. */ | ||
| hk_sync_barrier(cmd, HK_SYNC_ALL, HK_SYNC_ALL); | ||
| return; | ||
| } | ||
| } | ||
|
|
||
| hk_cdm_flush_all_dirty(cmd); | ||
| hk_cdm_xfer_visibility_at_barrier(cmd, pDependencyInfo); | ||
| hk_cmd_buffer_end_compute(cmd); | ||
| hk_cmd_buffer_end_graphics(cmd); | ||
| /* Preserve the same conservative ordering in the shared submit graph, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -84,6 +84,15 @@ static const struct debug_named_value hk_perf_options[] = { | |
| {"nobarrier", HK_PERF_NOBARRIER,"Ignore pipeline barriers"}, | ||
| {"nobatch", HK_PERF_NOBATCH, "Disable submission batching"}, | ||
| {"norobust", HK_PERF_NOROBUST, "Disable robustness"}, | ||
| {"nocdmbarrier", HK_PERF_NOCDMBARRIER, | ||
| "Omit the per-launch CDM barrier entirely"}, | ||
| {"usccdmbarrier", HK_PERF_USCCDMBARRIER, | ||
| "Reduce the per-launch CDM barrier to a USC cache invalidate"}, | ||
| {"designedusccdmbarrier", HK_PERF_DESIGNEDUSCCDMBARRIER, | ||
| "G13X designed set {0,1,2,4,5,6,8} plus USC cache invalidate (0x17f)"}, | ||
| {"alwayscdmbarrier", HK_PERF_ALWAYSCDMBARRIER, "Unconditional per-launch CDM barrier (disable dependency tracking)"}, | ||
| {"trackcdmbarrier", HK_PERF_TRACKCDMBARRIER, | ||
| "Dependency-track the CDM barrier on chips other than G13"}, | ||
| DEBUG_NAMED_VALUE_END | ||
| }; | ||
| /* clang-format on */ | ||
|
|
@@ -867,6 +876,27 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice, | |
| } | ||
|
|
||
| dev->perftest = debug_get_flags_option("HK_PERFTEST", hk_perf_options, 0); | ||
| dev->cdm_barrier_mask = | ||
| debug_get_num_option("HK_CDM_BARRIER_MASK", 0) & 0x07ffffff; | ||
|
|
||
| /* The dependency-tracked CDM barrier holds token-digest parity with the | ||
| * reference driver on G13 (M1 family) and on G14X of generation 14 (M2 | ||
| * Pro/Max/Ultra class; verified on M2 Max: greedy token digests of three | ||
| * LLM decode runs of 2048 tokens, CTS compute groups and the | ||
| * floating-point gates equal to the per-launch barrier). Other chips, | ||
| * including multi-cluster generation 15 and later parts that also map to | ||
| * G14X, keep the per-launch barrier until they pass the same gate; | ||
| * HK_PERFTEST=trackcdmbarrier opts them in for that run and | ||
| * HK_PERFTEST=alwayscdmbarrier forces the per-launch barrier everywhere. | ||
| */ | ||
| bool tracked_by_default = | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: G13G and gen-14 parts other than M2 Max haven't been measured on this branch. Worth limiting the default to what passed the digest gate. — Marcelo's agent |
||
| pdev->dev.chip == AGX_CHIP_G13G || pdev->dev.chip == AGX_CHIP_G13X || | ||
| (pdev->dev.chip == AGX_CHIP_G14X && | ||
| pdev->dev.params.gpu_generation == 14); | ||
|
|
||
| if (!tracked_by_default && !HK_PERF(dev, TRACKCDMBARRIER)) | ||
| dev->perftest |= HK_PERF_ALWAYSCDMBARRIER; | ||
|
|
||
|
|
||
| if (instance->drirc.misc.disable_border_emulation) { | ||
| dev->perftest |= HK_PERF_NOBORDER; | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This changes the G13G barrier for Vulkan and GL (
agx_state.c:4832), andalwayscdmbarrierdoesn't restore it. Commit 1 says G13G keeps the full barrier. Could this move to its own PR? #6 looks like it covers it.— Marcelo's agent