From 8aee10dd4c3fd8aa0f0fa6b38769ab8d95b544cf Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 23:04:51 -0500 Subject: [PATCH] drm/asahi: g17: M5 Pro: wait for other queues' compute in firmware when it ends visible On M5 Pro a compute command waiting for another queue's compute command waits on the host: the producer's completion interrupt, the event worker and a new publication sit between the two kicks, and a cross-queue timeline semaphore hop takes 155 us. macOS on the same firmware keeps the wait in firmware and hops in 17.4 us. Traces of macOS's hop show how its kicks are encoded: the consumer names the producer's kick and its own parent in the completion class (1), each descriptor counts its queue's completion ordinal and carries it as the submission sequence, the kick waits on its own context's pool event, and the dispatch base is the qualified 0x10c08860 with the auxiliary control set only for a kick whose parent is pipelined (class 2). The previous M5 Pro form (ordinal 0, auxiliary control 1) hangs as soon as a parent takes class 1. M5 Pro compute kicks now take macOS's form. A firmware wait is satisfied when the producer completes, which a stream ending in Honeykrisp's weak barrier reports before its writes are visible: with the waits in firmware, dEQP-VK.synchronization2.op.multi_queue. timeline_semaphore read stale data in 31 of 214 cases, every one with a compute or transfer producer. A stream that ends with the strong memory-scoped flush is visible at completion. Add DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION, by which userspace declares such a stream, and report DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION on M5 Pro; there only a submission whose compute command sets it is waited for in firmware, and the others keep the host wait. Measured on J714s (pinned to CPU 12, two boots of each kernel): the hop is 155.3 us before, 155.9 us with this change and current Mesa, and 21.5 us with Mesa that sets the flag. Light compute and render submits are unchanged within run-to-run noise. dEQP s0, compute, glsl, draw, texture, synchronization, synchronization2, the multi-queue samples and occlusion queries have no failures with either Mesa. Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/file.rs | 8 +++ drivers/gpu/drm/asahi/g17/command.rs | 7 ++- drivers/gpu/drm/asahi/g17/fw/m5/compute.rs | 64 +++++++++++++++----- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 14 +++++ drivers/gpu/drm/asahi/g17/job.rs | 3 + drivers/gpu/drm/asahi/g17/mod.rs | 5 ++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 13 ++-- drivers/gpu/drm/asahi/g17/queue/mod.rs | 6 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 8 ++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 5 +- include/uapi/drm/asahi_drm.h | 16 +++++ 11 files changed, 123 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index f11f53e8a978d..11e68066d424b 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -94,6 +94,14 @@ pub(crate) const DRM_ASAHI_RENDER_SHADER_WRITES: u32 = /// (DRM_ASAHI_RENDER_STAGE_PREFIXES), as the complete struct drm_asahi_cmd_render_m5. pub(crate) const DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2: u64 = uapi::DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 as u64; +/// Compute commands may declare a stream that ends with a memory-scoped flush +/// (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION). +pub(crate) const DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION: u64 = + uapi::DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION as u64; +/// The compute command's writes are visible when it completes, so another queue's compute +/// command may wait for it in firmware. +pub(crate) const DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION as u32; /// The tiling half waits only on the render and compute prefixes of the command's stage-prefix /// fields, its fragment half on the whole ones. #[allow(dead_code)] diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 140d0ba9d3bac..796d072364fd2 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -441,6 +441,8 @@ pub(crate) enum Validated { /// The command may copy or reset queries, so it must follow the host's /// timestamp publication for earlier commands. query_access: bool, + /// The command's writes are visible when it completes (M5). + visible_completion: bool, resources: Option>, plan: Option, scratch: ScratchRequest, @@ -651,7 +653,8 @@ impl Payload { if (m5 && (cmd.flags & !(crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX - | crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS) + | crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS + | crate::file::DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION) != 0 || cmd.reserved != 0)) || (!m5 && resources.is_some()) @@ -710,6 +713,8 @@ impl Payload { intersection: *intersection, matrix_ops: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX != 0, query_access: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS == 0, + visible_completion: m5 + && cmd.flags & crate::file::DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION != 0, resources: resources.clone(), plan: None, scratch, m5_scratch, diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs index d44b8b06c2bf9..c6da35ab3d648 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -29,21 +29,37 @@ pub(crate) const fn topology_dispatch_control(base: u64, clusters: u32) -> u64 { /// for compute work (tiling's is also 2, fragment's 3). pub(crate) const COMPUTE_PIPELINED_DEPENDENCY: u8 = 2; +/// The completion class, which holds a kick until the kick it names has completed. +pub(crate) const COMPLETION_DEPENDENCY: u8 = 1; + +/// Dependency class of an explicit dependency of a compute kick on `board`. +pub(crate) fn explicit_dependency(board: &Board) -> u8 { + if board.completion_inputs() { COMPLETION_DEPENDENCY } else { COMPUTE_PIPELINED_DEPENDENCY } +} + /// Dependency class of a compute kick's implicit parent (the previous kick of its queue) on -/// `board`. With the G17D forms it is derived as the kick-queue encoder derives it: the compute -/// class only for a command whose pipeline dependency type is 1, which only a late-evaluated wait -/// sets, and class 1 otherwise, which holds the kick until its parent has completed. No DRM -/// compute kick is a late-evaluated wait. Otherwise it is the pipelined class, with which a kick -/// may start while its parent still runs. +/// `board`. With the counted forms ([`counted_forms`]) it is derived as the kick-queue encoder +/// derives it: the compute class only for a command whose pipeline dependency type is 1, which +/// only a late-evaluated wait sets, and class 1 otherwise, which holds the kick until its parent +/// has completed. No DRM compute kick is a late-evaluated wait. Otherwise it is the pipelined +/// class, with which a kick may start while its parent still runs. pub(crate) fn parent_dependency(board: &Board, pipelined: bool) -> u8 { - if pipelined || !board.g17d_forms() { COMPUTE_PIPELINED_DEPENDENCY } else { 1 } + if pipelined || !counted_forms(board) { COMPUTE_PIPELINED_DEPENDENCY } else { 1 } } -/// The completion ordinal of a compute command on `board`. With the G17D forms it is the count of -/// compute commands its hardware queue published before it (zero for a new queue), wrapping as a -/// u32; render commands count separately. Otherwise it is 0 on every kick. +/// Whether compute kicks on `board` count their completion ordinal and hold for their parent: +/// with the G17D forms, and on boards whose explicit waits take the completion class +/// ([`Board::completion_inputs`]), as macOS encodes M5 Pro compute. +pub(crate) fn counted_forms(board: &Board) -> bool { + board.g17d_forms() || board.completion_inputs() +} + +/// The completion ordinal of a compute command on `board`. With the counted forms +/// ([`counted_forms`]) it is the count of compute commands its hardware queue published before it +/// (zero for a new queue), wrapping as a u32; render commands count separately. Otherwise it is 0 +/// on every kick. pub(crate) fn completion_ordinal(board: &Board, published_before: u32) -> u32 { - if board.g17d_forms() { published_before } else { 0 } + if counted_forms(board) { published_before } else { 0 } } /// Contexts per engine in the event space: a pool's event is 64 x its engine + its context. @@ -58,17 +74,20 @@ const ENTRY_GATE_EVENT: u32 = 254; /// The event mask of a compute kick of context `context_id` on `board`, published with its /// current-kick notification. With the G17D forms ([`Board::g17d_forms`]) the kick waits on its /// own context's pool event (64 x 2 + the context), which the firmware holds while it processes -/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL. -/// Otherwise it is the fixed `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate). +/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL. With +/// the other counted forms ([`counted_forms`]) it is the pool event alone. Otherwise it is the +/// fixed `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate). pub(crate) fn kick_event_mask(board: &Board, context_id: u32) -> Result<[u64; 4]> { - if !board.g17d_forms() { + if !counted_forms(board) { return Ok([0, 0, 8, 0]); } if context_id >= POOL_EVENT_CONTEXTS { return Err(EINVAL); } let mut mask = [0u64; 4]; - for event in [POOL_EVENT_ENGINE * POOL_EVENT_CONTEXTS + context_id, ENTRY_GATE_EVENT] { + let pool = POOL_EVENT_ENGINE * POOL_EVENT_CONTEXTS + context_id; + let events = [Some(pool), board.g17d_forms().then_some(ENTRY_GATE_EVENT)]; + for event in events.into_iter().flatten() { mask[(event / 64) as usize] |= 1 << (event % 64); } Ok(mask) @@ -92,9 +111,22 @@ pub(crate) const NORMAL_TUPLE: KickTuple = pub(crate) const QUALIFIED_TUPLE: KickTuple = KickTuple { preempt_state_offset: 0x1480, dispatch_base: 0x10c0_8860, auxiliary_control: 0 }; -/// The DRM compute tuple of `board` ([`Board::g17d_forms`]). +/// The DRM compute tuple of `board` ([`Board::g17d_forms`]). Where explicit waits take the +/// completion class ([`Board::completion_inputs`]) it is macOS's M5 Pro form: the J714 +/// preemption offset with the qualified dispatch base, and the auxiliary control set only when +/// the kick's parent is pipelined. pub(crate) fn kick_tuple(board: &Board) -> KickTuple { - if board.g17d_forms() { QUALIFIED_TUPLE } else { NORMAL_TUPLE } + if board.g17d_forms() { + QUALIFIED_TUPLE + } else if board.completion_inputs() { + KickTuple { + preempt_state_offset: NORMAL_TUPLE.preempt_state_offset, + dispatch_base: QUALIFIED_TUPLE.dispatch_base, + auxiliary_control: u64::from(parent_dependency(board, false) == COMPUTE_PIPELINED_DEPENDENCY), + } + } else { + NORMAL_TUPLE + } } const _: () = { diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index a7b56f5bab724..ed397b27917cb 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1796,6 +1796,9 @@ pub(crate) struct Board { /// Whether the host encodings are the G17D ones measured on the M5 Ultra /// ([`Board::g17d_forms`]). g17d_forms: bool, + /// Whether explicit waits on other kicks take the completion class + /// ([`Board::completion_inputs`]). + completion_inputs: bool, } /// M5 Pro: two clusters of ten cores. @@ -1818,6 +1821,7 @@ pub(crate) const J714: Board = Board { topology_words: None, fan_curve_start_mc: 84_000, g17d_forms: false, + completion_inputs: true, }; /// M5 Max: four clusters of ten cores. @@ -1842,6 +1846,7 @@ pub(crate) const J716C: Board = Board { topology_words: Some((16, 40)), fan_curve_start_mc: 84_000, g17d_forms: false, + completion_inputs: false, }; /// J775D's performance controllers run from P3. @@ -1876,6 +1881,7 @@ pub(crate) const J775D: Board = Board { // above the SMC's own start rather than below it. fan_curve_start_mc: 97_000, g17d_forms: true, + completion_inputs: false, }; const _: () = { @@ -2162,6 +2168,14 @@ impl Board { self.g17d_forms } + /// Whether a kick's explicit waits on other kicks (inputs from other queues, other-engine + /// producers) take the completion class, which holds them in firmware until those kicks + /// complete. A wait on another queue's compute stays in firmware only for work that declares + /// a visible completion (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION); others wait on the host. + pub(crate) const fn completion_inputs(&self) -> bool { + self.completion_inputs + } + /// The board's `index`th register window. pub(crate) fn register_window_spec(&self, index: usize) -> Option<&'static M5RegisterWindowSpec> { self.register_windows.get(index) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 9b85de020d24d..d5c07c1c2220e 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -319,6 +319,9 @@ pub(crate) trait Backend: Send + Sync { /// prefix producers: without the G17D forms an M5 kick may start while its ring /// predecessor still runs. fn host_producer_waits(&self) -> bool { false } + /// Whether a compute command may be waited for in firmware only when it declares that its + /// writes are visible at completion (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION). + fn firmware_sources_need_visible_completion(&self) -> bool { false } /// The reclaim-safe queue for the scheduler's timeout work, whose handler signals the /// fences of timed-out jobs. fn fence_queue(&self) -> Arc; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 8c748844d117d..63e04695c6c39 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -2241,6 +2241,11 @@ impl gpu::Gpu for Gpu { if crate::module_parameters::m5_mesh_storage_chunks.value() == 0 { features &= !crate::file::DRM_ASAHI_FEATURE_TASK_MESH_STORAGE; } + // Only where firmware waits need it (runtime::backend's + // firmware_sources_need_visible_completion), so userspace adds the flush only there. + if self.shared.m5_board.is_some_and(|board| board.completion_inputs()) { + features |= crate::file::DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION; + } params.features |= features; } params.gpu_generation = self.cfg.gpu_generation as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b0f2d795bcc3b..ae3a02f139c19 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2637,7 +2637,10 @@ impl Queue { dispatch_control: crate::g17::fw::m5::compute::dispatch_control(board, tuple.dispatch_base), auxiliary_control: tuple.auxiliary_control, }, - descriptor_va: descriptor_low, submit_sequence: 0, work_state_va: work_node, + descriptor_va: descriptor_low, + // M5 Pro's descriptors carry the ordinal as the submission sequence too, as macOS's do. + submit_sequence: if board.completion_inputs() { u64::from(self.ordinal) } else { 0 }, + work_state_va: work_node, cdm_end_va: *cdm_end_va, stamp_va: [args.stamp_va, args.aux_stamp_va], stamp_value: crate::g17::fw::stamp_value(kick_timestamp.get() as u32), qid: u32::from(qid), kick_timestamp: kick_timestamp.get(), @@ -2646,9 +2649,11 @@ impl Queue { shared_control_va: self.pool.control_va(), event_metadata: [0;16], usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1, }}); - // Explicit dependencies keep the pipelined class; the implicit parent, last, takes its - // own ([`crate::g17::fw::m5::compute::parent_dependency`]). - let mut m5_dependency_types = [crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY; + // Explicit dependencies take the board's explicit class; the implicit parent, last, + // takes its own ([`crate::g17::fw::m5::compute::parent_dependency`]). + let mut m5_dependency_types = [m5_board.map_or( + crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY, + crate::g17::fw::m5::compute::explicit_dependency); crate::g17::fw::kick::QID_MAX as usize + 1]; if let (Some(board), Some(parent)) = (m5_board, m5_dependency_types.get_mut(dependencies.len())) { *parent = crate::g17::fw::m5::compute::parent_dependency(board, false); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index b750313faa76a..e3be25d0ceb20 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -528,8 +528,12 @@ impl crate::queue::Queue for Queue { } } let mut guards = guards.into_iter(); + // Where a firmware wait reports completion before a weak stream tail's writes are + // visible, only a command that declares a visible completion is waited for there. + let visible_required = self.backend.firmware_sources_need_visible_completion(); let firmware_source = pending.len() == 1 - && matches!(&pending[0].0, Validated::Compute { .. }) + && matches!(&pending[0].0, Validated::Compute { visible_completion, .. } + if *visible_completion || !visible_required) && !pending[0].2.has_outputs(); let aggregate = outputs.publish(self.context.clone(), firmware_source)?; for (command, order, destinations) in pending { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 5958c48e228eb..ca504ed87db91 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -619,6 +619,7 @@ fn render_barriers( stage_prefixes: bool, dependencies: &RenderDependencies, own_tiling: KickDependency, + compute_class: u8, ) -> RenderBarriers { let fragment_stage = fragment_stage || stage_prefixes; let mut tiling_barriers = [KickDependency::ZERO; 5]; @@ -658,12 +659,12 @@ fn render_barriers( } if let Some(compute) = dependencies.compute { tiling_barriers[tiling_count] = compute; - tiling_types[tiling_count] = 2; + tiling_types[tiling_count] = compute_class; tiling_count += 1; } if let Some(compute) = dependencies.fragment_compute { fragment_barriers[fragment_count] = compute; - fragment_types[fragment_count] = 2; + fragment_types[fragment_count] = compute_class; fragment_count += 1; } if stage_prefixes { @@ -897,7 +898,8 @@ impl Pair { }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; let barriers = - render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling); + render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling, + self.board.map_or(2, crate::g17::fw::m5::compute::explicit_dependency)); let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; // With the G17D forms a fragment kick with history also names its parent's // predecessor, in class 1, after the self-parent (classes [1, 3, 1] for a lone pass). diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index af0e4b801ba99..705325bf6d153 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -122,7 +122,10 @@ impl job::Backend for Backend { self.shared.follows_submitter() } fn host_producer_waits(&self) -> bool { - self.shared.m5_board.is_some_and(|board| !board.g17d_forms()) + self.shared.m5_board.is_some_and(|board| !board.g17d_forms() && !board.completion_inputs()) + } + fn firmware_sources_need_visible_completion(&self) -> bool { + self.shared.m5_board.is_some_and(|board| board.completion_inputs()) } fn fence_queue(&self) -> Arc { self.shared.fence_queue() diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 4519da91ac0ca..ea7bc61ae60a8 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -483,6 +483,13 @@ enum drm_asahi_feature { */ #define DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 ((1ULL) << 33) +/* + * DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION: M5: compute commands accept + * DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION. A feature bit like those of enum + * drm_asahi_feature, defined here like bit 32. + */ +#define DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION ((1ULL) << 34) + #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 #define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 @@ -2023,6 +2030,15 @@ enum drm_asahi_compute_flags { * Any bytes after it are zero. */ DRM_ASAHI_COMPUTE_G18_RESOURCES = (1L << 19), + + /** + * @DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION: M5: the command's control + * stream ends with a strong, memory-scoped cache flush, so its writes + * are visible once it completes. A compute command of another queue may + * then wait for it in firmware rather than on the host. Requires + * DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION. + */ + DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION = (1L << 20), }; /**