Skip to content
4 changes: 4 additions & 0 deletions drivers/gpu/drm/asahi/asahi_core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,10 @@ module! {
// permissions: 0o644,
description: "Initial TVB size in blocks",
},
g16_pb_reuse: u32 {
default: 1,
description: "G16 parameter buffer: 1 = reuse a pass slot's last retired epoch for the same VM while the slot is still bound to it (default), 0 = a fresh epoch and InitBM per render",
},
robust_isolation: u32 {
default: 0,
// permissions: 0o644,
Expand Down
13 changes: 10 additions & 3 deletions drivers/gpu/drm/asahi/g16/command.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,19 @@ impl Order {
/// A same-engine command cannot jump a hole in its queue's completed frontier.
/// Cross-engine historical dependencies end at the named prefix, even while later
/// commands in that older submission remain pending.
pub(crate) fn ready(&self, completed: [u64; 2]) -> bool {
completed[self.engine].checked_add(1) == Some(self.sequence)
pub(crate) fn ready(&self, completed: [u64; 2], published: [u64; 2]) -> bool {
// Same-engine compute order and barriers are kept by the firmware compute queue,
// which finishes each command before it starts the next: its published frontier
// suffices. Renders, and every cross-engine dependency, need the completed one.
let mut frontier = completed;
if self.engine == COMPUTE {
frontier[COMPUTE] = published[COMPUTE].max(completed[COMPUTE]);
}
frontier[self.engine].checked_add(1) == Some(self.sequence)
&& self
.wait_through
.iter()
.zip(completed)
.zip(frontier)
.all(|(prefix, observed)| prefix.is_none_or(|last| observed >= last))
}
}
Expand Down
277 changes: 218 additions & 59 deletions drivers/gpu/drm/asahi/g16/engine.rs

Large diffs are not rendered by default.

30 changes: 22 additions & 8 deletions drivers/gpu/drm/asahi/g16/hw.rs
Original file line number Diff line number Diff line change
Expand Up @@ -182,10 +182,17 @@ pub(crate) mod client {
pub(crate) const CLIENT_APERTURE: Range<u64> = 0x7000000000..0x7100000000;
pub(crate) const COMPACT_WINDOW: Range<u64> = 0x1000000000..0x9000000000;

// Two persistent compute units, each in its own 64 MiB interval and reused across
// application queues. Every alias is inside Mesa's reserved driver hole.
// Persistent compute units, each in its own 64 MiB interval and reused across
// application queues. Units 0 and 1 keep their original placement below the render
// allocators; further units (up to [`super::sizes::COMPUTE_UNITS`]) live between the
// unused CT window and the PB heap, clear of the dynamic PB page-list range
// 0x7040000000..0x7080000000. Every alias is inside Mesa's reserved driver hole.
pub(crate) const fn compute_base(k: u64) -> u64 {
0x7001000000 + k * 0x04000000
if k < 2 {
0x7001000000 + k * 0x04000000
} else {
0x7090000000 + (k - 2) * 0x04000000
}
}
pub(crate) const fn compute_pool(k: u64) -> u64 {
compute_base(k)
Expand Down Expand Up @@ -280,13 +287,20 @@ pub(crate) mod sizes {
pub(crate) const PB_BLOCKS: u32 = 2048;
pub(crate) const PB_LIST_RECORDS: u32 = 4096;
pub(crate) const PB_GROW_STEP: u32 = 10;
/// The qualified M4 protocol permits only one published command at a time.
pub(crate) const FLIGHTS: usize = 1;
/// Published prefixes at most. A render is published only with nothing else in flight;
/// compute commands may queue behind each other on the one firmware compute queue, each
/// in its own retained unit ([`COMPUTE_UNITS`]).
pub(crate) const FLIGHTS: usize = 16;
/// Retained compute units per VM at most, all in hardware UMA slot 1. A unit is reused
/// only after a later publication retired (it displaced the unit), so `n` units keep up
/// to `n - 1` compute commands in flight.
pub(crate) const COMPUTE_UNITS: usize = 8;
const _: () = assert!(super::client::compute_base(COMPUTE_UNITS as u64) <= super::client::PB_HEAP);
/// Inherited diagnostic histogram width; M4 does not fold jobs into a compute horizon.
pub(crate) const HORIZON_JOBS: u32 = 1;
/// Two jobs may be admitted per logical queue so exact prior-command dependencies can
/// advance independently of a preceding job's aggregate fence. Firmware stays serialized.
pub(crate) const SCHEDULER_CREDITS: u32 = 2;
/// Jobs admitted to the engine per logical queue: enough to keep a queue's compute
/// commands in flight back to back; exact prior-command dependencies still gate each one.
pub(crate) const SCHEDULER_CREDITS: u32 = 16;
/// Unconsumed records per 256-entry channel ring a prefix may be posted behind.
pub(crate) const AHEAD_RECORDS: u32 = 64;
/// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit).
Expand Down
15 changes: 14 additions & 1 deletion drivers/gpu/drm/asahi/g16/job.rs
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,12 @@ impl Job {
.completed
.each_ref()
.map(|value| value.load(Ordering::Acquire));
if !order.ready(completed) {
let published = self
.queue
.published
.each_ref()
.map(|value| value.load(Ordering::Acquire));
if !order.ready(completed, published) {
return Err(EAGAIN);
}
Ok(())
Expand All @@ -299,6 +304,14 @@ impl Job {
/// Advances exactly one queue-local engine ordinal after a full successful wire receipt.
/// Called under the device lock before another command is considered. No allocation,
/// fence callback or aggregate-job completion occurs here.
/// Advances the queue's published frontier past command `index` (under the device
/// lock, when its first producer became visible).
pub(crate) fn command_published(&self, index: u32) {
if let Some(order) = self.orders.get(index as usize) {
self.queue.published[order.engine].fetch_max(order.sequence, Ordering::AcqRel);
}
}

pub(crate) fn command_retired(&self, index: u32) -> Result {
let order = self.orders.get(index as usize).ok_or(EINVAL)?;
let previous = order.sequence.checked_sub(1).ok_or(EINVAL)?;
Expand Down
86 changes: 85 additions & 1 deletion drivers/gpu/drm/asahi/g16/params.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,105 @@ pub(crate) fn compute_deadline_ms() -> u32 {
module_parameters::g16_compute_deadline_ms.value()
}

/// Runtime tunables, written through `aurora-g16/tune` (debugfs, root only) as `name=value`.
pub(crate) mod tune {
use core::sync::atomic::{AtomicU32, Ordering};

/// Compute units of a VM that submits compute back to back
/// (2..=[`super::hw::sizes::COMPUTE_UNITS`]); applies to VMs that grow after the change.
pub(crate) static COMPUTE_UNITS: AtomicU32 = AtomicU32::new(6);
/// Compute commands in flight at most (1 = the serialized engine).
pub(crate) static COMPUTE_FLIGHTS: AtomicU32 = AtomicU32::new(16);
/// Completion repoll interval override, us (0: `g16_resnap_us`).
pub(crate) static RESNAP_US: AtomicU32 = AtomicU32::new(0);
/// PB heap of a VM's render arena, MiB (VMs created after a change).
pub(crate) static PB_HEAP_MB: AtomicU32 = AtomicU32::new(64);
/// PB blocks posted to a fresh epoch at most (2048: the whole heap).
pub(crate) static PB_BLOCKS: AtomicU32 = AtomicU32::new(2048);
const NAMES: [(&str, &AtomicU32, u32, u32); 5] = [
("compute_units", &COMPUTE_UNITS, 2, super::hw::sizes::COMPUTE_UNITS as u32),
("compute_flights", &COMPUTE_FLIGHTS, 1, super::hw::sizes::FLIGHTS as u32),
("resnap_us", &RESNAP_US, 0, 1000),
("pb_heap_mb", &PB_HEAP_MB, 8, 256),
("pb_blocks", &PB_BLOCKS, 16, 2048),
];

/// Applies one `name=value` line.
pub(crate) fn set(line: &str) -> kernel::error::Result {
let (name, value) = line.trim().split_once('=').ok_or(kernel::error::code::EINVAL)?;
let value: u32 = value
.trim()
.parse()
.map_err(|_| kernel::error::code::EINVAL)?;
for (n, cell, lo, hi) in NAMES {
if n == name.trim() {
if value < lo || value > hi {
return Err(kernel::error::code::ERANGE);
}
cell.store(value, Ordering::Relaxed);
return Ok(());
}
}
Err(kernel::error::code::ENOENT)
}

pub(crate) fn show(f: &mut impl core::fmt::Write) -> core::fmt::Result {
for (n, cell, _, _) in NAMES {
write!(f, "{}={} ", n, cell.load(Ordering::Relaxed))?;
}
writeln!(f)
}
}

/// PB heap blocks of a new render arena (128 KiB each).
pub(crate) fn pb_heap_blocks() -> u32 {
let mb = tune::PB_HEAP_MB.load(core::sync::atomic::Ordering::Relaxed).clamp(8, 256);
mb * 8
}

/// PB blocks posted to a fresh epoch of a heap of `heap_blocks`.
pub(crate) fn pb_posted_blocks(heap_blocks: u32) -> u32 {
tune::PB_BLOCKS
.load(core::sync::atomic::Ordering::Relaxed)
.clamp(16, 2048)
.min(heap_blocks)
}

/// Compute units of a VM that submits compute back to back.
pub(crate) fn compute_units() -> usize {
(tune::COMPUTE_UNITS.load(core::sync::atomic::Ordering::Relaxed) as usize)
.clamp(2, hw::sizes::COMPUTE_UNITS)
}

/// Compute commands in flight at most.
pub(crate) fn compute_flights() -> usize {
(tune::COMPUTE_FLIGHTS.load(core::sync::atomic::Ordering::Relaxed) as usize)
.clamp(1, hw::sizes::FLIGHTS)
}

/// Repoll while a ready prefix waits for record consumption, us.
pub(crate) fn repoll_us() -> u32 {
module_parameters::g16_repoll_us.value().clamp(5, 100)
}

/// Completion repoll interval, us.
pub(crate) fn resnap_us() -> u32 {
module_parameters::g16_resnap_us.value().clamp(25, 1000)
match tune::RESNAP_US.load(core::sync::atomic::Ordering::Relaxed) {
0 => module_parameters::g16_resnap_us.value().clamp(25, 1000),
us => us.clamp(10, 1000),
}
}

/// Whether firmware halts are answered with the handshake (else fail-stop).
pub(crate) fn recovery() -> bool {
module_parameters::g16_recovery.value() != 0
}

/// Whether a pass slot's retired PB epoch may serve the next render of its VM.
pub(crate) fn pb_reuse() -> bool {
module_parameters::g16_pb_reuse.value() != 0
}

/// Whether the CDM/VDM stream gates are on.
pub(crate) fn stream_gate() -> bool {
module_parameters::g16_stream_gate.value() != 0
Expand Down
Loading
Loading