Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions drivers/gpu/drm/asahi/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,14 @@ pub(crate) const DRM_ASAHI_RENDER_SHADER_WRITES: u32 =
/// (DRM_ASAHI_RENDER_STAGE_PREFIXES), as the complete struct drm_asahi_cmd_render_m5.
pub(crate) const DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2: u64 =
uapi::DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 as u64;
/// Compute commands may declare a stream that ends with a memory-scoped flush
/// (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION).
pub(crate) const DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION: u64 =
uapi::DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION as u64;
/// The compute command's writes are visible when it completes, so another queue's compute
/// command may wait for it in firmware.
pub(crate) const DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION: u32 =
uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION as u32;
/// The tiling half waits only on the render and compute prefixes of the command's stage-prefix
/// fields, its fragment half on the whole ones.
#[allow(dead_code)]
Expand Down
7 changes: 6 additions & 1 deletion drivers/gpu/drm/asahi/g17/command.rs
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,8 @@ pub(crate) enum Validated {
/// The command may copy or reset queries, so it must follow the host's
/// timestamp publication for earlier commands.
query_access: bool,
/// The command's writes are visible when it completes (M5).
visible_completion: bool,
resources: Option<Arc<ComputePrograms>>,
plan: Option<super::render_resources::Plan>,
scratch: ScratchRequest,
Expand Down Expand Up @@ -651,7 +653,8 @@ impl Payload {
if (m5
&& (cmd.flags
& !(crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX
| crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS)
| crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS
| crate::file::DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION)
!= 0
|| cmd.reserved != 0))
|| (!m5 && resources.is_some())
Expand Down Expand Up @@ -710,6 +713,8 @@ impl Payload {
intersection: *intersection,
matrix_ops: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX != 0,
query_access: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS == 0,
visible_completion: m5
&& cmd.flags & crate::file::DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION != 0,
resources: resources.clone(), plan: None,
scratch,
m5_scratch,
Expand Down
64 changes: 48 additions & 16 deletions drivers/gpu/drm/asahi/g17/fw/m5/compute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,21 +29,37 @@ pub(crate) const fn topology_dispatch_control(base: u64, clusters: u32) -> u64 {
/// for compute work (tiling's is also 2, fragment's 3).
pub(crate) const COMPUTE_PIPELINED_DEPENDENCY: u8 = 2;

/// The completion class, which holds a kick until the kick it names has completed.
pub(crate) const COMPLETION_DEPENDENCY: u8 = 1;

/// Dependency class of an explicit dependency of a compute kick on `board`.
pub(crate) fn explicit_dependency(board: &Board) -> u8 {
if board.completion_inputs() { COMPLETION_DEPENDENCY } else { COMPUTE_PIPELINED_DEPENDENCY }
}

/// Dependency class of a compute kick's implicit parent (the previous kick of its queue) on
/// `board`. With the G17D forms it is derived as the kick-queue encoder derives it: the compute
/// class only for a command whose pipeline dependency type is 1, which only a late-evaluated wait
/// sets, and class 1 otherwise, which holds the kick until its parent has completed. No DRM
/// compute kick is a late-evaluated wait. Otherwise it is the pipelined class, with which a kick
/// may start while its parent still runs.
/// `board`. With the counted forms ([`counted_forms`]) it is derived as the kick-queue encoder
/// derives it: the compute class only for a command whose pipeline dependency type is 1, which
/// only a late-evaluated wait sets, and class 1 otherwise, which holds the kick until its parent
/// has completed. No DRM compute kick is a late-evaluated wait. Otherwise it is the pipelined
/// class, with which a kick may start while its parent still runs.
pub(crate) fn parent_dependency(board: &Board, pipelined: bool) -> u8 {
if pipelined || !board.g17d_forms() { COMPUTE_PIPELINED_DEPENDENCY } else { 1 }
if pipelined || !counted_forms(board) { COMPUTE_PIPELINED_DEPENDENCY } else { 1 }
}

/// The completion ordinal of a compute command on `board`. With the G17D forms it is the count of
/// compute commands its hardware queue published before it (zero for a new queue), wrapping as a
/// u32; render commands count separately. Otherwise it is 0 on every kick.
/// Whether compute kicks on `board` count their completion ordinal and hold for their parent:
/// with the G17D forms, and on boards whose explicit waits take the completion class
/// ([`Board::completion_inputs`]), as macOS encodes M5 Pro compute.
pub(crate) fn counted_forms(board: &Board) -> bool {
board.g17d_forms() || board.completion_inputs()
}

/// The completion ordinal of a compute command on `board`. With the counted forms
/// ([`counted_forms`]) it is the count of compute commands its hardware queue published before it
/// (zero for a new queue), wrapping as a u32; render commands count separately. Otherwise it is 0
/// on every kick.
pub(crate) fn completion_ordinal(board: &Board, published_before: u32) -> u32 {
if board.g17d_forms() { published_before } else { 0 }
if counted_forms(board) { published_before } else { 0 }
}

/// Contexts per engine in the event space: a pool's event is 64 x its engine + its context.
Expand All @@ -58,17 +74,20 @@ const ENTRY_GATE_EVENT: u32 = 254;
/// The event mask of a compute kick of context `context_id` on `board`, published with its
/// current-kick notification. With the G17D forms ([`Board::g17d_forms`]) the kick waits on its
/// own context's pool event (64 x 2 + the context), which the firmware holds while it processes
/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL.
/// Otherwise it is the fixed `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate).
/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL. With
/// the other counted forms ([`counted_forms`]) it is the pool event alone. Otherwise it is the
/// fixed `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate).
pub(crate) fn kick_event_mask(board: &Board, context_id: u32) -> Result<[u64; 4]> {
if !board.g17d_forms() {
if !counted_forms(board) {
return Ok([0, 0, 8, 0]);
}
if context_id >= POOL_EVENT_CONTEXTS {
return Err(EINVAL);
}
let mut mask = [0u64; 4];
for event in [POOL_EVENT_ENGINE * POOL_EVENT_CONTEXTS + context_id, ENTRY_GATE_EVENT] {
let pool = POOL_EVENT_ENGINE * POOL_EVENT_CONTEXTS + context_id;
let events = [Some(pool), board.g17d_forms().then_some(ENTRY_GATE_EVENT)];
for event in events.into_iter().flatten() {
mask[(event / 64) as usize] |= 1 << (event % 64);
}
Ok(mask)
Expand All @@ -92,9 +111,22 @@ pub(crate) const NORMAL_TUPLE: KickTuple =
pub(crate) const QUALIFIED_TUPLE: KickTuple =
KickTuple { preempt_state_offset: 0x1480, dispatch_base: 0x10c0_8860, auxiliary_control: 0 };

/// The DRM compute tuple of `board` ([`Board::g17d_forms`]).
/// The DRM compute tuple of `board` ([`Board::g17d_forms`]). Where explicit waits take the
/// completion class ([`Board::completion_inputs`]) it is macOS's M5 Pro form: the J714
/// preemption offset with the qualified dispatch base, and the auxiliary control set only when
/// the kick's parent is pipelined.
pub(crate) fn kick_tuple(board: &Board) -> KickTuple {
if board.g17d_forms() { QUALIFIED_TUPLE } else { NORMAL_TUPLE }
if board.g17d_forms() {
QUALIFIED_TUPLE
} else if board.completion_inputs() {
KickTuple {
preempt_state_offset: NORMAL_TUPLE.preempt_state_offset,
dispatch_base: QUALIFIED_TUPLE.dispatch_base,
auxiliary_control: u64::from(parent_dependency(board, false) == COMPUTE_PIPELINED_DEPENDENCY),
}
} else {
NORMAL_TUPLE
}
}

const _: () = {
Expand Down
14 changes: 14 additions & 0 deletions drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1796,6 +1796,9 @@ pub(crate) struct Board {
/// Whether the host encodings are the G17D ones measured on the M5 Ultra
/// ([`Board::g17d_forms`]).
g17d_forms: bool,
/// Whether explicit waits on other kicks take the completion class
/// ([`Board::completion_inputs`]).
completion_inputs: bool,
}

/// M5 Pro: two clusters of ten cores.
Expand All @@ -1818,6 +1821,7 @@ pub(crate) const J714: Board = Board {
topology_words: None,
fan_curve_start_mc: 84_000,
g17d_forms: false,
completion_inputs: true,
};

/// M5 Max: four clusters of ten cores.
Expand All @@ -1842,6 +1846,7 @@ pub(crate) const J716C: Board = Board {
topology_words: Some((16, 40)),
fan_curve_start_mc: 84_000,
g17d_forms: false,
completion_inputs: false,
};

/// J775D's performance controllers run from P3.
Expand Down Expand Up @@ -1876,6 +1881,7 @@ pub(crate) const J775D: Board = Board {
// above the SMC's own start rather than below it.
fan_curve_start_mc: 97_000,
g17d_forms: true,
completion_inputs: false,
};

const _: () = {
Expand Down Expand Up @@ -2162,6 +2168,14 @@ impl Board {
self.g17d_forms
}

/// Whether a kick's explicit waits on other kicks (inputs from other queues, other-engine
/// producers) take the completion class, which holds them in firmware until those kicks
/// complete. A wait on another queue's compute stays in firmware only for work that declares
/// a visible completion (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION); others wait on the host.
pub(crate) const fn completion_inputs(&self) -> bool {
self.completion_inputs
}

/// The board's `index`th register window.
pub(crate) fn register_window_spec(&self, index: usize) -> Option<&'static M5RegisterWindowSpec> {
self.register_windows.get(index)
Expand Down
3 changes: 3 additions & 0 deletions drivers/gpu/drm/asahi/g17/job.rs
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,9 @@ pub(crate) trait Backend: Send + Sync {
/// prefix producers: without the G17D forms an M5 kick may start while its ring
/// predecessor still runs.
fn host_producer_waits(&self) -> bool { false }
/// Whether a compute command may be waited for in firmware only when it declares that its
/// writes are visible at completion (DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION).
fn firmware_sources_need_visible_completion(&self) -> bool { false }
/// The reclaim-safe queue for the scheduler's timeout work, whose handler signals the
/// fences of timed-out jobs.
fn fence_queue(&self) -> Arc<workqueue::OwnedQueue>;
Expand Down
5 changes: 5 additions & 0 deletions drivers/gpu/drm/asahi/g17/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2241,6 +2241,11 @@ impl gpu::Gpu for Gpu {
if crate::module_parameters::m5_mesh_storage_chunks.value() == 0 {
features &= !crate::file::DRM_ASAHI_FEATURE_TASK_MESH_STORAGE;
}
// Only where firmware waits need it (runtime::backend's
// firmware_sources_need_visible_completion), so userspace adds the flush only there.
if self.shared.m5_board.is_some_and(|board| board.completion_inputs()) {
features |= crate::file::DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION;
}
params.features |= features;
}
params.gpu_generation = self.cfg.gpu_generation as u32;
Expand Down
13 changes: 9 additions & 4 deletions drivers/gpu/drm/asahi/g17/queue/compute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2637,7 +2637,10 @@ impl Queue {
dispatch_control: crate::g17::fw::m5::compute::dispatch_control(board, tuple.dispatch_base),
auxiliary_control: tuple.auxiliary_control,
},
descriptor_va: descriptor_low, submit_sequence: 0, work_state_va: work_node,
descriptor_va: descriptor_low,
// M5 Pro's descriptors carry the ordinal as the submission sequence too, as macOS's do.
submit_sequence: if board.completion_inputs() { u64::from(self.ordinal) } else { 0 },
work_state_va: work_node,
cdm_end_va: *cdm_end_va, stamp_va: [args.stamp_va, args.aux_stamp_va],
stamp_value: crate::g17::fw::stamp_value(kick_timestamp.get() as u32), qid: u32::from(qid),
kick_timestamp: kick_timestamp.get(),
Expand All @@ -2646,9 +2649,11 @@ impl Queue {
shared_control_va: self.pool.control_va(), event_metadata: [0;16],
usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1,
}});
// Explicit dependencies keep the pipelined class; the implicit parent, last, takes its
// own ([`crate::g17::fw::m5::compute::parent_dependency`]).
let mut m5_dependency_types = [crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY;
// Explicit dependencies take the board's explicit class; the implicit parent, last,
// takes its own ([`crate::g17::fw::m5::compute::parent_dependency`]).
let mut m5_dependency_types = [m5_board.map_or(
crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY,
crate::g17::fw::m5::compute::explicit_dependency);
crate::g17::fw::kick::QID_MAX as usize + 1];
if let (Some(board), Some(parent)) = (m5_board, m5_dependency_types.get_mut(dependencies.len())) {
*parent = crate::g17::fw::m5::compute::parent_dependency(board, false);
Expand Down
6 changes: 5 additions & 1 deletion drivers/gpu/drm/asahi/g17/queue/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -528,8 +528,12 @@ impl<B: Backend> crate::queue::Queue for Queue<B> {
}
}
let mut guards = guards.into_iter();
// Where a firmware wait reports completion before a weak stream tail's writes are
// visible, only a command that declares a visible completion is waited for there.
let visible_required = self.backend.firmware_sources_need_visible_completion();
let firmware_source = pending.len() == 1
&& matches!(&pending[0].0, Validated::Compute { .. })
&& matches!(&pending[0].0, Validated::Compute { visible_completion, .. }
if *visible_completion || !visible_required)
&& !pending[0].2.has_outputs();
let aggregate = outputs.publish(self.context.clone(), firmware_source)?;
for (command, order, destinations) in pending {
Expand Down
8 changes: 5 additions & 3 deletions drivers/gpu/drm/asahi/g17/queue/render.rs
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,7 @@ fn render_barriers(
stage_prefixes: bool,
dependencies: &RenderDependencies,
own_tiling: KickDependency,
compute_class: u8,
) -> RenderBarriers {
let fragment_stage = fragment_stage || stage_prefixes;
let mut tiling_barriers = [KickDependency::ZERO; 5];
Expand Down Expand Up @@ -658,12 +659,12 @@ fn render_barriers(
}
if let Some(compute) = dependencies.compute {
tiling_barriers[tiling_count] = compute;
tiling_types[tiling_count] = 2;
tiling_types[tiling_count] = compute_class;
tiling_count += 1;
}
if let Some(compute) = dependencies.fragment_compute {
fragment_barriers[fragment_count] = compute;
fragment_types[fragment_count] = 2;
fragment_types[fragment_count] = compute_class;
fragment_count += 1;
}
if stage_prefixes {
Expand Down Expand Up @@ -897,7 +898,8 @@ impl Pair {
};
let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?;
let barriers =
render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling);
render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling,
self.board.map_or(2, crate::g17::fw::m5::compute::explicit_dependency));
let tiling_barriers = &barriers.tiling[..barriers.tiling_count];
// With the G17D forms a fragment kick with history also names its parent's
// predecessor, in class 1, after the self-parent (classes [1, 3, 1] for a lone pass).
Expand Down
5 changes: 4 additions & 1 deletion drivers/gpu/drm/asahi/g17/runtime/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,10 @@ impl job::Backend for Backend {
self.shared.follows_submitter()
}
fn host_producer_waits(&self) -> bool {
self.shared.m5_board.is_some_and(|board| !board.g17d_forms())
self.shared.m5_board.is_some_and(|board| !board.g17d_forms() && !board.completion_inputs())
}
fn firmware_sources_need_visible_completion(&self) -> bool {
self.shared.m5_board.is_some_and(|board| board.completion_inputs())
}
fn fence_queue(&self) -> Arc<kernel::workqueue::OwnedQueue> {
self.shared.fence_queue()
Expand Down
16 changes: 16 additions & 0 deletions include/uapi/drm/asahi_drm.h
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,13 @@ enum drm_asahi_feature {
*/
#define DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 ((1ULL) << 33)

/*
* DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION: M5: compute commands accept
* DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION. A feature bit like those of enum
* drm_asahi_feature, defined here like bit 32.
*/
#define DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION ((1ULL) << 34)

#define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1
#define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2

Expand Down Expand Up @@ -2023,6 +2030,15 @@ enum drm_asahi_compute_flags {
* Any bytes after it are zero.
*/
DRM_ASAHI_COMPUTE_G18_RESOURCES = (1L << 19),

/**
* @DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION: M5: the command's control
* stream ends with a strong, memory-scoped cache flush, so its writes
* are visible once it completes. A compute command of another queue may
* then wait for it in firmware rather than on the host. Requires
* DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION.
*/
DRM_ASAHI_COMPUTE_VISIBLE_COMPLETION = (1L << 20),
};

/**
Expand Down