diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index f11f53e8a978d..11e68066d424b 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -94,6 +94,14 @@ pub(crate) const DRM_ASAHI_RENDER_SHADER_WRITES: u32 = /// (DRM_ASAHI_RENDER_STAGE_PREFIXES), as the complete struct drm_asahi_cmd_render_m5. pub(crate) const DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2: u64 = uapi::DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 as u64; +/// Compute commands may declare a stream that ends with a memory-scoped flush +/// (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION). +pub(crate) const DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION: u64 = + uapi::DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION as u64; +/// The compute command's writes are visible when it completes, so another queue's compute +/// command may wait for it in firmware. +pub(crate) const DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION as u32; /// The tiling half waits only on the render and compute prefixes of the command's stage-prefix /// fields, its fragment half on the whole ones. #[allow(dead_code)] diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 140d0ba9d3bac..796d072364fd2 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -441,6 +441,8 @@ pub(crate) enum Validated { /// The command may copy or reset queries, so it must follow the host's /// timestamp publication for earlier commands. query_access: bool, + /// The command's writes are visible when it completes (M5). + visible_completion: bool, resources: Option>, plan: Option, scratch: ScratchRequest, @@ -651,7 +653,8 @@ impl Payload { if (m5 && (cmd.flags & !(crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX - | crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS) + | crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS + | crate::file::DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION) != 0 || cmd.reserved != 0)) || (!m5 && resources.is_some()) @@ -710,6 +713,8 @@ impl Payload { intersection: *intersection, matrix_ops: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX != 0, query_access: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS == 0, + visible_completion: m5 + && cmd.flags & crate::file::DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION != 0, resources: resources.clone(), plan: None, scratch, m5_scratch, diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs index d44b8b06c2bf9..c6da35ab3d648 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -29,21 +29,37 @@ pub(crate) const fn topology_dispatch_control(base: u64, clusters: u32) -> u64 { /// for compute work (tiling's is also 2, fragment's 3). pub(crate) const COMPUTE_PIPELINED_DEPENDENCY: u8 = 2; +/// The completion class, which holds a kick until the kick it names has completed. +pub(crate) const COMPLETION_DEPENDENCY: u8 = 1; + +/// Dependency class of an explicit dependency of a compute kick on `board`. +pub(crate) fn explicit_dependency(board: &Board) -> u8 { + if board.completion_inputs() { COMPLETION_DEPENDENCY } else { COMPUTE_PIPELINED_DEPENDENCY } +} + /// Dependency class of a compute kick's implicit parent (the previous kick of its queue) on -/// `board`. With the G17D forms it is derived as the kick-queue encoder derives it: the compute -/// class only for a command whose pipeline dependency type is 1, which only a late-evaluated wait -/// sets, and class 1 otherwise, which holds the kick until its parent has completed. No DRM -/// compute kick is a late-evaluated wait. Otherwise it is the pipelined class, with which a kick -/// may start while its parent still runs. +/// `board`. With the counted forms ([`counted_forms`]) it is derived as the kick-queue encoder +/// derives it: the compute class only for a command whose pipeline dependency type is 1, which +/// only a late-evaluated wait sets, and class 1 otherwise, which holds the kick until its parent +/// has completed. No DRM compute kick is a late-evaluated wait. Otherwise it is the pipelined +/// class, with which a kick may start while its parent still runs. pub(crate) fn parent_dependency(board: &Board, pipelined: bool) -> u8 { - if pipelined || !board.g17d_forms() { COMPUTE_PIPELINED_DEPENDENCY } else { 1 } + if pipelined || !counted_forms(board) { COMPUTE_PIPELINED_DEPENDENCY } else { 1 } } -/// The completion ordinal of a compute command on `board`. With the G17D forms it is the count of -/// compute commands its hardware queue published before it (zero for a new queue), wrapping as a -/// u32; render commands count separately. Otherwise it is 0 on every kick. +/// Whether compute kicks on `board` count their completion ordinal and hold for their parent: +/// with the G17D forms, and on boards whose explicit waits take the completion class +/// ([`Board::completion_inputs`]), as macOS encodes M5 Pro compute. +pub(crate) fn counted_forms(board: &Board) -> bool { + board.g17d_forms() || board.completion_inputs() +} + +/// The completion ordinal of a compute command on `board`. With the counted forms +/// ([`counted_forms`]) it is the count of compute commands its hardware queue published before it +/// (zero for a new queue), wrapping as a u32; render commands count separately. Otherwise it is 0 +/// on every kick. pub(crate) fn completion_ordinal(board: &Board, published_before: u32) -> u32 { - if board.g17d_forms() { published_before } else { 0 } + if counted_forms(board) { published_before } else { 0 } } /// Contexts per engine in the event space: a pool's event is 64 x its engine + its context. @@ -58,17 +74,20 @@ const ENTRY_GATE_EVENT: u32 = 254; /// The event mask of a compute kick of context `context_id` on `board`, published with its /// current-kick notification. With the G17D forms ([`Board::g17d_forms`]) the kick waits on its /// own context's pool event (64 x 2 + the context), which the firmware holds while it processes -/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL. -/// Otherwise it is the fixed `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate). +/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL. With +/// the other counted forms ([`counted_forms`]) it is the pool event alone. Otherwise it is the +/// fixed `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate). pub(crate) fn kick_event_mask(board: &Board, context_id: u32) -> Result<[u64; 4]> { - if !board.g17d_forms() { + if !counted_forms(board) { return Ok([0, 0, 8, 0]); } if context_id >= POOL_EVENT_CONTEXTS { return Err(EINVAL); } let mut mask = [0u64; 4]; - for event in [POOL_EVENT_ENGINE * POOL_EVENT_CONTEXTS + context_id, ENTRY_GATE_EVENT] { + let pool = POOL_EVENT_ENGINE * POOL_EVENT_CONTEXTS + context_id; + let events = [Some(pool), board.g17d_forms().then_some(ENTRY_GATE_EVENT)]; + for event in events.into_iter().flatten() { mask[(event / 64) as usize] |= 1 << (event % 64); } Ok(mask) @@ -92,9 +111,22 @@ pub(crate) const NORMAL_TUPLE: KickTuple = pub(crate) const QUALIFIED_TUPLE: KickTuple = KickTuple { preempt_state_offset: 0x1480, dispatch_base: 0x10c0_8860, auxiliary_control: 0 }; -/// The DRM compute tuple of `board` ([`Board::g17d_forms`]). +/// The DRM compute tuple of `board` ([`Board::g17d_forms`]). Where explicit waits take the +/// completion class ([`Board::completion_inputs`]) it is macOS's M5 Pro form: the J714 +/// preemption offset with the qualified dispatch base, and the auxiliary control set only when +/// the kick's parent is pipelined. pub(crate) fn kick_tuple(board: &Board) -> KickTuple { - if board.g17d_forms() { QUALIFIED_TUPLE } else { NORMAL_TUPLE } + if board.g17d_forms() { + QUALIFIED_TUPLE + } else if board.completion_inputs() { + KickTuple { + preempt_state_offset: NORMAL_TUPLE.preempt_state_offset, + dispatch_base: QUALIFIED_TUPLE.dispatch_base, + auxiliary_control: u64::from(parent_dependency(board, false) == COMPUTE_PIPELINED_DEPENDENCY), + } + } else { + NORMAL_TUPLE + } } const _: () = { diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index a7b56f5bab724..ed397b27917cb 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1796,6 +1796,9 @@ pub(crate) struct Board { /// Whether the host encodings are the G17D ones measured on the M5 Ultra /// ([`Board::g17d_forms`]). g17d_forms: bool, + /// Whether explicit waits on other kicks take the completion class + /// ([`Board::completion_inputs`]). + completion_inputs: bool, } /// M5 Pro: two clusters of ten cores. @@ -1818,6 +1821,7 @@ pub(crate) const J714: Board = Board { topology_words: None, fan_curve_start_mc: 84_000, g17d_forms: false, + completion_inputs: true, }; /// M5 Max: four clusters of ten cores. @@ -1842,6 +1846,7 @@ pub(crate) const J716C: Board = Board { topology_words: Some((16, 40)), fan_curve_start_mc: 84_000, g17d_forms: false, + completion_inputs: false, }; /// J775D's performance controllers run from P3. @@ -1876,6 +1881,7 @@ pub(crate) const J775D: Board = Board { // above the SMC's own start rather than below it. fan_curve_start_mc: 97_000, g17d_forms: true, + completion_inputs: false, }; const _: () = { @@ -2162,6 +2168,14 @@ impl Board { self.g17d_forms } + /// Whether a kick's explicit waits on other kicks (inputs from other queues, other-engine + /// producers) take the completion class, which holds them in firmware until those kicks + /// complete. A wait on another queue's compute stays in firmware only for work that declares + /// a visible completion (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION); others wait on the host. + pub(crate) const fn completion_inputs(&self) -> bool { + self.completion_inputs + } + /// The board's `index`th register window. pub(crate) fn register_window_spec(&self, index: usize) -> Option<&'static M5RegisterWindowSpec> { self.register_windows.get(index) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 9b85de020d24d..d5c07c1c2220e 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -319,6 +319,9 @@ pub(crate) trait Backend: Send + Sync { /// prefix producers: without the G17D forms an M5 kick may start while its ring /// predecessor still runs. fn host_producer_waits(&self) -> bool { false } + /// Whether a compute command may be waited for in firmware only when it declares that its + /// writes are visible at completion (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION). + fn firmware_sources_need_visible_completion(&self) -> bool { false } /// The reclaim-safe queue for the scheduler's timeout work, whose handler signals the /// fences of timed-out jobs. fn fence_queue(&self) -> Arc; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 8c748844d117d..63e04695c6c39 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -2241,6 +2241,11 @@ impl gpu::Gpu for Gpu { if crate::module_parameters::m5_mesh_storage_chunks.value() == 0 { features &= !crate::file::DRM_ASAHI_FEATURE_TASK_MESH_STORAGE; } + // Only where firmware waits need it (runtime::backend's + // firmware_sources_need_visible_completion), so userspace adds the flush only there. + if self.shared.m5_board.is_some_and(|board| board.completion_inputs()) { + features |= crate::file::DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION; + } params.features |= features; } params.gpu_generation = self.cfg.gpu_generation as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b0f2d795bcc3b..ae3a02f139c19 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2637,7 +2637,10 @@ impl Queue { dispatch_control: crate::g17::fw::m5::compute::dispatch_control(board, tuple.dispatch_base), auxiliary_control: tuple.auxiliary_control, }, - descriptor_va: descriptor_low, submit_sequence: 0, work_state_va: work_node, + descriptor_va: descriptor_low, + // M5 Pro's descriptors carry the ordinal as the submission sequence too, as macOS's do. + submit_sequence: if board.completion_inputs() { u64::from(self.ordinal) } else { 0 }, + work_state_va: work_node, cdm_end_va: *cdm_end_va, stamp_va: [args.stamp_va, args.aux_stamp_va], stamp_value: crate::g17::fw::stamp_value(kick_timestamp.get() as u32), qid: u32::from(qid), kick_timestamp: kick_timestamp.get(), @@ -2646,9 +2649,11 @@ impl Queue { shared_control_va: self.pool.control_va(), event_metadata: [0;16], usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1, }}); - // Explicit dependencies keep the pipelined class; the implicit parent, last, takes its - // own ([`crate::g17::fw::m5::compute::parent_dependency`]). - let mut m5_dependency_types = [crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY; + // Explicit dependencies take the board's explicit class; the implicit parent, last, + // takes its own ([`crate::g17::fw::m5::compute::parent_dependency`]). + let mut m5_dependency_types = [m5_board.map_or( + crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY, + crate::g17::fw::m5::compute::explicit_dependency); crate::g17::fw::kick::QID_MAX as usize + 1]; if let (Some(board), Some(parent)) = (m5_board, m5_dependency_types.get_mut(dependencies.len())) { *parent = crate::g17::fw::m5::compute::parent_dependency(board, false); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index b750313faa76a..e3be25d0ceb20 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -528,8 +528,12 @@ impl crate::queue::Queue for Queue { } } let mut guards = guards.into_iter(); + // Where a firmware wait reports completion before a weak stream tail's writes are + // visible, only a command that declares a visible completion is waited for there. + let visible_required = self.backend.firmware_sources_need_visible_completion(); let firmware_source = pending.len() == 1 - && matches!(&pending[0].0, Validated::Compute { .. }) + && matches!(&pending[0].0, Validated::Compute { visible_completion, .. } + if *visible_completion || !visible_required) && !pending[0].2.has_outputs(); let aggregate = outputs.publish(self.context.clone(), firmware_source)?; for (command, order, destinations) in pending { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 5958c48e228eb..ca504ed87db91 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -619,6 +619,7 @@ fn render_barriers( stage_prefixes: bool, dependencies: &RenderDependencies, own_tiling: KickDependency, + compute_class: u8, ) -> RenderBarriers { let fragment_stage = fragment_stage || stage_prefixes; let mut tiling_barriers = [KickDependency::ZERO; 5]; @@ -658,12 +659,12 @@ fn render_barriers( } if let Some(compute) = dependencies.compute { tiling_barriers[tiling_count] = compute; - tiling_types[tiling_count] = 2; + tiling_types[tiling_count] = compute_class; tiling_count += 1; } if let Some(compute) = dependencies.fragment_compute { fragment_barriers[fragment_count] = compute; - fragment_types[fragment_count] = 2; + fragment_types[fragment_count] = compute_class; fragment_count += 1; } if stage_prefixes { @@ -897,7 +898,8 @@ impl Pair { }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; let barriers = - render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling); + render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling, + self.board.map_or(2, crate::g17::fw::m5::compute::explicit_dependency)); let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; // With the G17D forms a fragment kick with history also names its parent's // predecessor, in class 1, after the self-parent (classes [1, 3, 1] for a lone pass). diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index af0e4b801ba99..705325bf6d153 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -122,7 +122,10 @@ impl job::Backend for Backend { self.shared.follows_submitter() } fn host_producer_waits(&self) -> bool { - self.shared.m5_board.is_some_and(|board| !board.g17d_forms()) + self.shared.m5_board.is_some_and(|board| !board.g17d_forms() && !board.completion_inputs()) + } + fn firmware_sources_need_visible_completion(&self) -> bool { + self.shared.m5_board.is_some_and(|board| board.completion_inputs()) } fn fence_queue(&self) -> Arc { self.shared.fence_queue() diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 4519da91ac0ca..ea7bc61ae60a8 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -483,6 +483,13 @@ enum drm_asahi_feature { */ #define DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 ((1ULL) << 33) +/* + * DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION: M5: compute commands accept + * DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION. A feature bit like those of enum + * drm_asahi_feature, defined here like bit 32. + */ +#define DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION ((1ULL) << 34) + #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 #define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 @@ -2023,6 +2030,15 @@ enum drm_asahi_compute_flags { * Any bytes after it are zero. */ DRM_ASAHI_COMPUTE_G18_RESOURCES = (1L << 19), + + /** + * @DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION: M5: the command's control + * stream ends with a strong, memory-scoped cache flush, so its writes + * are visible once it completes. A compute command of another queue may + * then wait for it in firmware rather than on the host. Requires + * DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION. + */ + DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION = (1L << 20), }; /**