From 9c9bf14bb586ea59143a72392a5dec13d710bb8d Mon Sep 17 00:00:00 2001 From: Friedrich <261542475+aeries0x@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:02:22 +0100 Subject: [PATCH 1/8] drm/asahi: g16: reuse a pass slot's retired PB epoch Every G16 render gets a fresh parameter buffer epoch at SUBMIT and an InitBM ahead of its TA work, so the firmware rebuilds the allocator state for each render. Reusing the epoch instead cuts a small triangle render at queue depth 1 on a base M4 MacBook Pro (J604) from 1121 to 492 us. With this change, the next render of the same VM on a pass slot gets the slot's owner epoch, without a reseed or InitBM, if that epoch's render retired normally and was exposed. InitBM binds the device-wide slot to one epoch and VM, so this happens only while no other epoch was handed to the slot since; after another VM's render on the slot, after a failed or aborted render, or after any firmware recovery, the next render starts fresh. SUBMIT still allocates a fresh epoch for every render, so one is always at hand. Renders are bound, published and retired one at a time, so the slot's owner is the epoch it is bound to whenever reuse is allowed. The reused epoch is the slot's existing owner lease, so this holds no epoch, VM or admission permit beyond what the lease already holds: a slot binding is tracked by a counter, not by a reference. A host test covers when an epoch may be reused and that nothing else holds it. aurora.g16_pb_reuse=0 restores a fresh epoch and InitBM per render. The stats line counts reused and fresh bindings. Assisted-by: LLM Signed-off-by: Friedrich <261542475+aeries0x@users.noreply.github.com> --- drivers/gpu/drm/asahi/asahi_core.rs | 4 + drivers/gpu/drm/asahi/g16/engine.rs | 44 +++++++-- drivers/gpu/drm/asahi/g16/params.rs | 5 ++ drivers/gpu/drm/asahi/g16/poll.rs | 12 ++- drivers/gpu/drm/asahi/g16/pools.rs | 89 +++++++++++++++++-- drivers/gpu/drm/asahi/g16/recovery.rs | 3 + .../drivers/gpu/asahi-m4/test-resources.py | 24 ++++- 7 files changed, 163 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index b970691651032..a75e537af2b69 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -167,6 +167,10 @@ module! { // permissions: 0o644, description: "Initial TVB size in blocks", }, + g16_pb_reuse: u32 { + default: 1, + description: "G16 parameter buffer: 1 = reuse a pass slot's last retired epoch for the same VM while the slot is still bound to it (default), 0 = a fresh epoch and InitBM per render", + }, robust_isolation: u32 { default: 0, // permissions: 0o644, diff --git a/drivers/gpu/drm/asahi/g16/engine.rs b/drivers/gpu/drm/asahi/g16/engine.rs index 5ee33d71432d1..0c40f185f6280 100644 --- a/drivers/gpu/drm/asahi/g16/engine.rs +++ b/drivers/gpu/drm/asahi/g16/engine.rs @@ -180,6 +180,10 @@ pub(crate) struct Flight { pub(crate) recovery_started_ns: u64, /// Publication time, ns. pub(crate) published_ns: u64, + /// The PB epoch this render prefix uses (its own, or a reused one with `g16_pb_reuse`). + pub(crate) pb: Option>, + /// Its pass slot binding (`Pools::bind`). + pub(crate) pb_binding: u64, } impl Flight { @@ -209,6 +213,8 @@ impl Flight { recovery_suspect: false, recovery_started_ns: 0, published_ns: 0, + pb: None, + pb_binding: 0, } } } @@ -872,12 +878,30 @@ impl Firmware { } /// Addresses and cumulative allocator state of the selected retained render unit. - fn render_refs(&mut self, k: usize, job: &Job, draw: u64) -> Result<(RenderRefs, bool)> { + fn render_refs( + &mut self, + k: usize, + job: &Job, + draw: u64, + ) -> Result<(RenderRefs, bool, Arc, u64)> { let generation = job.vm.generation; let arena = job.resources.render()?; let mut a = arena.pools.prepare(k, draw)?; - let epoch = job.resources.pb(job.progress.sent() as usize)?; - epoch.prepare(k, arena.pools.heap_gva(k)?)?; + // g16_pb_reuse: reuse this VM's retired epoch of the slot: no reseed, no InitBM. + let (epoch, reused) = match self.pools.reusable(params::pb_reuse(), k, &job.vm) { + Some(e) => { + self.pools.stats.reused += 1; + a.init_buffer = false; + (e, true) + } + None => { + let e = job.resources.pb(job.progress.sent() as usize)?.clone(); + e.prepare(k, arena.pools.heap_gva(k)?)?; + self.pools.stats.fresh += 1; + (e, false) + } + }; + let binding = self.pools.bind(k, reused); a.pb_mgr = epoch.manager.va; a.mgr_misc = epoch.misc.va; let r = &self.graph.render; @@ -908,6 +932,8 @@ impl Firmware { usc_base: job.queue.usc_base, }, a.init_buffer, + epoch, + binding, )) } @@ -1033,8 +1059,8 @@ impl Firmware { return Err(EAGAIN); } if render { - job.resources - .pb(index as usize)? + f.pb.as_ref() + .ok_or(EINVAL)? .check_assignment(f.claims.pass[0] as usize)?; } // Reserve lifetime ownership before any counter or producer becomes reachable. @@ -1047,7 +1073,7 @@ impl Firmware { let publication = (|| -> Result { if render { let k = f.claims.pass[0] as usize; - job.resources.pb(index as usize)?.expose(); + f.pb.as_ref().ok_or(EINVAL)?.expose(); job.resources.render()?.pools.publish(k, f.uma_submitted)?; // Notifier service dereferences u32 counters; the host ordering ledger // remains u64. Eight-byte spacing is retained allocation padding. @@ -1263,7 +1289,9 @@ impl Firmware { // Claim performs only a qualified per-use scrub, never allocator/list reset. self.slots.claim_pass(job.resources.render()?, k)?; self.slots.hold_vm(&f.claims, &job.vm)?; - let (refs, init_buffer) = self.render_refs(k, job, draw)?; + let (refs, init_buffer, epoch, binding) = self.render_refs(k, job, draw)?; + f.pb = Some(epoch); + f.pb_binding = binding; let p = &job.resources.render()?.passes[k]; let pass = p.refs(); let geometry = Geometry::new(&cmd.cmd, pass.heap_gva); @@ -1322,7 +1350,7 @@ impl Firmware { if init_buffer { pf.initbm = true; let bm = encode::initbm(job.vm.slot, refs.parameter_id, refs.pb_mgr, target); - let epoch = job.resources.pb(job.progress.sent() as usize)?; + let epoch = f.pb.as_ref().ok_or(EINVAL)?; epoch.initbm.write(0, &bm)?; r.ring[0].w64(8 * pf.ta_head as usize, epoch.initbm.va)?; pf.ta_head = (pf.ta_head + 1) % cap; diff --git a/drivers/gpu/drm/asahi/g16/params.rs b/drivers/gpu/drm/asahi/g16/params.rs index d2ef01e26b3a3..65da293ea718f 100644 --- a/drivers/gpu/drm/asahi/g16/params.rs +++ b/drivers/gpu/drm/asahi/g16/params.rs @@ -43,6 +43,11 @@ pub(crate) fn recovery() -> bool { module_parameters::g16_recovery.value() != 0 } +/// Whether a pass slot's retired PB epoch may serve the next render of its VM. +pub(crate) fn pb_reuse() -> bool { + module_parameters::g16_pb_reuse.value() != 0 +} + /// Whether the CDM/VDM stream gates are on. pub(crate) fn stream_gate() -> bool { module_parameters::g16_stream_gate.value() != 0 diff --git a/drivers/gpu/drm/asahi/g16/poll.rs b/drivers/gpu/drm/asahi/g16/poll.rs index 4e9bfaef02e12..1d91555e6694b 100644 --- a/drivers/gpu/drm/asahi/g16/poll.rs +++ b/drivers/gpu/drm/asahi/g16/poll.rs @@ -494,9 +494,13 @@ impl Firmware { if f.renders != 0 { self.recovery.last_retired[1] = Some(f.ta_target); self.recovery.last_retired[2] = Some(f.fs_target); - let committed = f.job.resources.pb(f.first).and_then(|epoch| { + let committed = f.pb.as_ref().ok_or(EINVAL).and_then(|epoch| { + let k = timestamp_claims.pass[0] as usize; + self.pools.retire_binding(k, epoch)?; + let enabled = params::pb_reuse(); self.pools - .retire_binding(timestamp_claims.pass[0] as usize, epoch) + .retire_epoch(enabled, k, f.pb_binding, result.is_ok()); + Ok(()) }); if let Err(error) = committed { self.retain(&f.job); @@ -518,6 +522,10 @@ impl Firmware { } else { // Only an actual passed abort lacks the full owner receipt. An affected job // with a full normal receipt fails its fence without poisoning healthy storage. + if f.renders != 0 { + // The quarantined slot is not picked again, but keep no epoch of it for reuse. + self.pools.forget(timestamp_claims.pass[0] as usize); + } self.slots.quarantine(&mut f.claims); self.retain(&f.job); self.recovery.aborted_receipts = self.recovery.aborted_receipts.saturating_add(1); diff --git a/drivers/gpu/drm/asahi/g16/pools.rs b/drivers/gpu/drm/asahi/g16/pools.rs index d4a85cc6f3d36..f2ae2f15d4deb 100644 --- a/drivers/gpu/drm/asahi/g16/pools.rs +++ b/drivers/gpu/drm/asahi/g16/pools.rs @@ -2,9 +2,12 @@ //! Private render UMA/heap backing and device-wide PB-slot leases. //! -//! Each VM keeps two persistent UMA/heap units. Each render command installs a fresh -//! PB manager with full InitBM and never reinitializes exposed cursors. The previous -//! PB-slot lease survives until the replacement's complete normal retirement. +//! Each VM keeps two persistent UMA/heap units. With `g16_pb_reuse` (the default), a render +//! reuses its pass slot's lease, its VM's last successfully retired epoch of the slot, while +//! the slot is still bound to it, unchanged (the host writes nothing to it, no InitBM). +//! Otherwise it installs a fresh PB manager with full InitBM and never reinitializes exposed +//! cursors. The previous PB-slot lease survives until the replacement's complete normal +//! retirement. #[path = "pools/lifetime.rs"] pub(crate) mod lifetime; @@ -67,6 +70,10 @@ pub(crate) struct PbStats { #[allow(dead_code)] pub(crate) refused: u64, pub(crate) bad: u64, + /// Pass slot bindings that reused a retired epoch / started a fresh one (a deferred + /// render counts again when it is retried). + pub(crate) reused: u64, + pub(crate) fresh: u64, } pub(crate) struct PoolSet { @@ -259,6 +266,11 @@ pub(crate) struct Pools { pub(crate) dev: ARef, pub(crate) stats: PbStats, owners: [Option>; 2], + /// Per pass slot, a binding count, bumped by each fresh epoch (its InitBM rebinds the + /// slot) and by a firmware recovery. It identifies a binding without holding the epoch. + binds: [u64; 2], + /// Per pass slot, with `g16_pb_reuse`: `owners[k]` may serve the next render of its VM. + keep: [bool; 2], } impl Pools { pub(crate) fn new(dev: &AsahiDrmDevice) -> Self { @@ -266,6 +278,66 @@ impl Pools { dev: dev.as_ref().as_ref().into(), stats: PbStats::default(), owners: [None, None], + binds: [0, 0], + keep: [false, false], + } + } + /// With reuse `enabled` (`g16_pb_reuse`): slot `k`'s owner epoch, if its render retired + /// normally while the slot was still bound to it, it belongs to `vm` and was exposed (its + /// InitBM ran). InitBM binds the device-wide slot to one epoch and VM; a fresh epoch handed + /// to the slot rebinds it, so the owner is then stale and the render starts fresh. Adds no + /// reference beyond the owner lease itself. + pub(crate) fn reusable( + &self, + enabled: bool, + k: usize, + vm: &Arc, + ) -> Option> { + if !enabled || !*self.keep.get(k)? { + return None; + } + let owner = self.owners.get(k)?.as_ref()?; + (Arc::ptr_eq(&owner._vm, vm) && owner.exposed.load(Ordering::Acquire)) + .then(|| owner.clone()) + } + /// Slot `k` is handed to a render's epoch, `reused` or fresh (with InitBM). Returns the + /// binding the render retires with. + pub(crate) fn bind(&mut self, k: usize, reused: bool) -> u64 { + let (Some(binds), Some(keep)) = (self.binds.get_mut(k), self.keep.get_mut(k)) else { + return 0; + }; + if !reused { + *binds = binds.wrapping_add(1); + *keep = false; + } + *binds + } + /// A render on slot `k` with `binding` retired, after its `retire_binding`. If no other + /// epoch was bound and no recovery started since, the next render of the same VM may reuse + /// its epoch with reuse `enabled` and a `completed` render; after a failed one it starts + /// fresh. + /// Renders are bound, published and retired one at a time (`sizes::FLIGHTS` is 1, enforced + /// in `Engine::new`), so `keep[k]` always refers to the epoch the slot is bound to: + /// `owners[k]`. + pub(crate) fn retire_epoch(&mut self, enabled: bool, k: usize, binding: u64, completed: bool) { + if self.binds.get(k) == Some(&binding) { + if let Some(keep) = self.keep.get_mut(k) { + *keep = enabled && completed; + } + } + } + /// Reuse nothing on slot `k` (its render was aborted). + pub(crate) fn forget(&mut self, k: usize) { + if let Some(keep) = self.keep.get_mut(k) { + *keep = false; + } + } + /// A firmware recovery started: no epoch bound so far is reused, also not by a render that + /// retires after it (its binding no longer matches); each slot's next render starts fresh. + pub(crate) fn invalidate(&mut self) { + for (binds, keep) in self.binds.iter_mut().zip(self.keep.iter_mut()) { + *binds = binds.wrapping_add(1); + *keep = false; } } pub(crate) fn retire_binding(&mut self, k: usize, epoch: &Arc) -> Result { @@ -275,16 +347,19 @@ impl Pools { pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { writeln!( f, - "pb: M4 private allocators blocks {} requests {} bad {}", + "pb: M4 private allocators blocks {} requests {} bad {} reused {} fresh {}", buffer::MAX_BLOCKS, self.stats.requests, - self.stats.bad + self.stats.bad, + self.stats.reused, + self.stats.fresh ) } } -/// One never-reinitialized PB manager installation. Even a same-VM command gets a fresh -/// manager and full InitBM; no host reset of exposed firmware cursors is required. +/// One never-reinitialized PB manager installation. Unless `g16_pb_reuse` hands a same-VM +/// command its retired epoch, every command gets a fresh manager and full InitBM; no host +/// reset of exposed firmware cursors is required. pub(crate) struct PbEpoch(crate::cleanup::Deferred); impl core::ops::Deref for PbEpoch { type Target = PbEpochInner; diff --git a/drivers/gpu/drm/asahi/g16/recovery.rs b/drivers/gpu/drm/asahi/g16/recovery.rs index b0fc2ed96976c..18975b812e276 100644 --- a/drivers/gpu/drm/asahi/g16/recovery.rs +++ b/drivers/gpu/drm/asahi/g16/recovery.rs @@ -320,6 +320,9 @@ impl Firmware { } self.recovery.running = true; self.engine.frozen = true; + // A recovery may reset what the firmware keeps for a bound PB epoch, also one whose + // render was not affected: start the next render on each pass slot fresh. + self.pools.invalidate(); self.events.grow.clear(); // These frontiers must be captured before the host answers 2: the // firmware can force the stage target while aborting a work item. diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py index a048447b1436a..fdb5f89dd30d9 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py @@ -68,7 +68,7 @@ def module(text, name): } pub struct Arc{inner:std::sync::Arc} - implArc{pub fn new(v:T,_:u32)->crate::Result{Ok(Self{inner:std::sync::Arc::new(v)})}pub fn strong_count(v:&Self)->usize{std::sync::Arc::strong_count(&v.inner)}} + implArc{pub fn new(v:T,_:u32)->crate::Result{Ok(Self{inner:std::sync::Arc::new(v)})}pub fn strong_count(v:&Self)->usize{std::sync::Arc::strong_count(&v.inner)}pub fn ptr_eq(a:&Self,b:&Self)->bool{std::sync::Arc::ptr_eq(&a.inner,&b.inner)}} implClone for Arc{fn clone(&self)->Self{Self{inner:self.inner.clone()}}} implstd::ops::Deref for Arc{type Target=T;fn deref(&self)->&T{&self.inner}} pub mod aref {pub struct ARef(T);impl From<&crate::device::Device>for ARef{fn from(v:&crate::device::Device)->Self{Self(v.clone())}}} @@ -373,6 +373,28 @@ def module(text, name): b.check_assignment(0).unwrap();b.expose();assert_eq!(sync::Arc::strong_count(&a),2); slots.retire_binding(0,&b).unwrap();assert_eq!(sync::Arc::strong_count(&a),1); } + #[test]fn pb_reuse_needs_the_same_binding_vm_and_a_completed_render_and_holds_nothing(){ + let one=vm(1);let two=vm(2);let factory=g16::object::Factory;let budget=g16::budget::Budget::new().unwrap(); + let a=g16::pools::PbEpoch::new(&factory,&one,g16::budget::Budget::reserve(&budget,1).unwrap().take().unwrap()).unwrap(); + a.prepare(0,0x70c0000000).unwrap();a.check_assignment(0).unwrap();a.expose(); + let mut p=g16::pools::Pools::new(&driver::AsahiDrmDevice(device::Device)); + let n=p.bind(0,false);p.retire_binding(0,&a).unwrap();p.retire_epoch(true,0,n,true); + assert_eq!(sync::Arc::strong_count(&a),2,"only the owner lease"); + let r=p.reusable(true,0,&one).unwrap();assert!(sync::Arc::ptr_eq(&r,&a));drop(r); + assert_eq!(sync::Arc::strong_count(&a),2); + assert!(p.reusable(false,0,&one).is_none(),"g16_pb_reuse=0"); + assert!(p.reusable(true,0,&two).is_none(),"another VM");assert!(p.reusable(true,1,&one).is_none(),"another slot"); + assert_eq!(p.bind(0,true),n);p.retire_epoch(true,0,n,true);assert!(p.reusable(true,0,&one).is_some()); + p.retire_epoch(true,0,n,false);assert!(p.reusable(true,0,&one).is_none(),"failed render"); + p.retire_epoch(true,0,n,true);p.forget(0);assert!(p.reusable(true,0,&one).is_none(),"aborted render"); + let m=p.bind(0,false);assert_ne!(m,n);p.retire_epoch(true,0,n,true); + assert!(p.reusable(true,0,&one).is_none(),"a stale binding after a fresh one"); + p.retire_epoch(true,0,m,true);assert!(p.reusable(true,0,&one).is_some()); + p.invalidate();assert!(p.reusable(true,0,&one).is_none(),"recovery"); + p.retire_epoch(true,0,m,true);assert!(p.reusable(true,0,&one).is_none(),"a render retiring after recovery"); + assert_eq!(p.bind(0,true),m.wrapping_add(1)); + assert_eq!(sync::Arc::strong_count(&a),2); + } } ''' From 07409dc586d07e2a5f2bf41c16240435952c3497 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Thu, 19 Nov 2026 07:15:51 -0500 Subject: [PATCH 2/8] drm/asahi: g16: Keep compute commands in flight G16 published one command at a time and built the next only after the previous one fully retired on the host, so every compute command paid the whole firmware and host round trip: on the J773g Mac mini a 1.5 us dispatch cost ~270 us per submission at any queue depth, and dependent work (one command buffer per dispatch, as Geekbench records it) was bound by it. Queue compute commands behind in-flight compute commands on the one firmware compute queue: - A VM gets more retained compute units, each in its own hardware UMA slot (unit k in slot 1 + k). All units in slot 1, as before, is not safe once commands are queued: the firmware starts the next command while the previous one still runs, and with long commands two owners of one slot then corrupt each other (firmware recovery within seconds on queued ALU work). With one slot per unit, ~115k queued commands ran without a recovery. - The displacement ledger generalizes to N owners: owners retire in publication order, and an owner is reusable once any later publication retired (the firmware finished it before starting that one). Publication stays serialized for the render pass slots. - Every queued command posts its own submit record and doorbell (the firmware does not pick up a WRITE advance without one) behind unconsumed ones, with room checks before anything is visible. - A flight with later compute flights behind it retires on "passed" receipts: its stamp, queue READ cursor and the submit pipe's cursors have moved past it. The newest flight keeps the exact receipt. - A compute command may follow its queue's published (not yet retired) compute predecessors: the firmware compute queue runs commands in publication order, each to its Finalize. Renders and every cross-engine dependency still wait for completion, and nothing is published behind a render or a failed, doomed or recovering flight. - Only the same VM's commands queue behind each other: with six dEQP clients, compute queued behind another VM's lost writes (image and sampler cases failed) and stalled the firmware queue. Nothing queues behind a command with user timestamps either: the host writes them when the command retires, and a query copy queued behind it ran before they existed and never finished. - Recovery matches a snapshot entry against every flight and marks every unfinished one; a failed flight dooms the later flights of its queue (their commands would follow a hole). - Two units per VM as before; a VM that submits compute back to back (three submissions within 2 ms each, or three compute commands in one submission) gets four more, mapped at 0x7090000000, so ordinary clients keep the old memory cost. - Up to 16 jobs per logical queue reach the engine (was 2). - The engine statistics and the compute owner ledger move to the heap: `Firmware` is built on the probe's stack, and its larger frame overflowed the 32 KiB kernel stack. - `aurora-g16/tune` (debugfs, root) sets the depth (compute_flights; 1 = the serialized engine), the units of a growing VM and the completion repoll interval for measurements. J773g Mac mini (G16G B0, 25G83), gpubench, before -> after (macOS 27.2 on the same machine): compute.light d8 268 -> 136 us/submit, d64 269 -> 125 (gap between commands 266 -> 16.5 us; macOS 19.7), compute.heavy alu d4 874 -> 557, d64 871 -> 554 (GPU time 554; macOS 540), compute.heavy bw d16 1667 -> 1275 (macOS 1437). Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/g16/command.rs | 13 +- drivers/gpu/drm/asahi/g16/engine.rs | 230 ++++++++++++++---- drivers/gpu/drm/asahi/g16/hw.rs | 30 ++- drivers/gpu/drm/asahi/g16/job.rs | 15 +- drivers/gpu/drm/asahi/g16/params.rs | 61 ++++- drivers/gpu/drm/asahi/g16/poll.rs | 123 +++++++++- drivers/gpu/drm/asahi/g16/pools/lifetime.rs | 128 ++++++++-- drivers/gpu/drm/asahi/g16/queue.rs | 5 + drivers/gpu/drm/asahi/g16/resources.rs | 79 +++++- drivers/gpu/drm/asahi/g16/slots.rs | 111 ++++++--- drivers/gpu/drm/asahi/timestamp.rs | 8 + .../gpu/asahi-m4/test-command-stream.py | 12 +- .../drivers/gpu/asahi-m4/test-job-failure.py | 10 +- .../drivers/gpu/asahi-m4/test-publication.py | 9 +- .../drivers/gpu/asahi-m4/test-recovery.py | 14 +- .../drivers/gpu/asahi-m4/test-resources.py | 14 +- .../drivers/gpu/asahi-m4/test-stamp-wrap.py | 27 +- .../drivers/gpu/asahi-m4/test-timestamps.py | 2 +- 18 files changed, 726 insertions(+), 165 deletions(-) diff --git a/drivers/gpu/drm/asahi/g16/command.rs b/drivers/gpu/drm/asahi/g16/command.rs index a2b329789444c..ea6f6882eff6a 100644 --- a/drivers/gpu/drm/asahi/g16/command.rs +++ b/drivers/gpu/drm/asahi/g16/command.rs @@ -112,12 +112,19 @@ impl Order { /// A same-engine command cannot jump a hole in its queue's completed frontier. /// Cross-engine historical dependencies end at the named prefix, even while later /// commands in that older submission remain pending. - pub(crate) fn ready(&self, completed: [u64; 2]) -> bool { - completed[self.engine].checked_add(1) == Some(self.sequence) + pub(crate) fn ready(&self, completed: [u64; 2], published: [u64; 2]) -> bool { + // Same-engine compute order and barriers are kept by the firmware compute queue, + // which finishes each command before it starts the next: its published frontier + // suffices. Renders, and every cross-engine dependency, need the completed one. + let mut frontier = completed; + if self.engine == COMPUTE { + frontier[COMPUTE] = published[COMPUTE].max(completed[COMPUTE]); + } + frontier[self.engine].checked_add(1) == Some(self.sequence) && self .wait_through .iter() - .zip(completed) + .zip(frontier) .all(|(prefix, observed)| prefix.is_none_or(|last| observed >= last)) } } diff --git a/drivers/gpu/drm/asahi/g16/engine.rs b/drivers/gpu/drm/asahi/g16/engine.rs index 0c40f185f6280..bc9111ca98419 100644 --- a/drivers/gpu/drm/asahi/g16/engine.rs +++ b/drivers/gpu/drm/asahi/g16/engine.rs @@ -3,10 +3,14 @@ //! Serialized M4 execution over the shared Asahi frontend. //! //! A userspace submission retains all commands, mappings and inputs through its final fence. -//! One command at a time is encoded into one of two retained units, then published to the -//! classic engine queues. Reuse requires a complete wire receipt followed by replacement -//! by the other retired unit in the same UMA slot. TA and FS remain a paired publication -//! with the native explicit TA dependency. This total order satisfies Mesa's command barriers. +//! Each command is encoded into a retained unit, then published to the classic engine queues. +//! A render is published only with nothing else in flight, and nothing is published behind +//! it. Compute commands queue behind in-flight compute commands (up to +//! [`params::compute_flights`]), each in its own unit of UMA slot 1: the firmware runs its +//! compute queue in publication order, each command to its Finalize, so publication order is +//! execution order. A unit is reused only after a later publication fully retired (it +//! displaced the unit). TA and FS remain a paired publication with the native explicit TA +//! dependency. This total order satisfies Mesa's command barriers. //! Firmware failure retains every possibly reachable owner. Completions are signalled only //! after dropping the device lock. @@ -17,10 +21,10 @@ use kernel::{prelude::*, sync::Arc}; use super::channel::Channel; use super::command::{Command, ComputeCommand, RenderCommand}; use super::encode::{self, ComputeRefs, Geometry, RenderRefs}; -use super::fw::channels::SubmitRecord; +use super::fw::channels::{SubmitRecord, RING_SLOTS}; use super::fw::graph::{notifier_page, qstate}; use super::fw::work::{self, BarrierType}; -use super::hw::sizes; +use super::hw::{engine as engine_ids, sizes}; use super::job::{Completions, Job}; use super::slots::Claims; use super::{params, Firmware, Shared}; @@ -54,8 +58,9 @@ pub(crate) fn now_ns() -> u64 { ::ktime_get() as u64 } -/// Counter value carried by a native 25G83 submit-pipe record. -fn counter_ticks() -> u64 { +/// Counter value carried by a native 25G83 submit-pipe record (the 24 MHz counter of the +/// firmware's internal timestamps). +pub(crate) fn counter_ticks() -> u64 { #[cfg(target_arch = "aarch64")] { let value: u64; @@ -180,6 +185,8 @@ pub(crate) struct Flight { pub(crate) recovery_started_ns: u64, /// Publication time, ns. pub(crate) published_ns: u64, + /// Published behind another flight. + pub(crate) behind: bool, /// The PB epoch this render prefix uses (its own, or a reused one with `g16_pb_reuse`). pub(crate) pb: Option>, /// Its pass slot binding (`Pools::bind`). @@ -213,6 +220,7 @@ impl Flight { recovery_suspect: false, recovery_started_ns: 0, published_ns: 0, + behind: false, pb: None, pb_binding: 0, } @@ -297,6 +305,10 @@ pub(crate) struct Stats { /// Why ready jobs stayed unpublished: both flights published, a channel record unconsumed, /// no free slot. pub(crate) hold: [u64; 3], + /// Compute commands published behind in-flight ones. + pub(crate) behind: u64, + /// Most flights in flight at once. + pub(crate) max_flights: u64, } /// Host-side phase timers of publication and polling, on with `debug_flags` bit 49 @@ -414,23 +426,21 @@ pub(crate) struct Engine { /// Progress tracking of the oldest prefix. pub(crate) progress_flight: u64, pub(crate) progress: [Option; 3], - pub(crate) stats: Stats, + /// Boxed with [`Self::prof`]: `Firmware` is built on the probe's stack. + pub(crate) stats: KBox, /// Phase timers (debug_flags bit 49). - pub(crate) prof: Prof, + pub(crate) prof: KBox, /// The `aurora-g16/stats` file. pub(crate) debugfs: Option>>>>, + /// The `aurora-g16/tune` file. + pub(crate) tune_file: Option>>>>, scratch: Option>, } impl Engine { pub(crate) fn new() -> Result { // Wrapped native stamps are safe only while pending references fit within the - // signed half-window. This backend publishes exactly one command at a time, - // retires its full wire/owner receipt, then builds the next command/barrier. - // Do not add submission-ahead without revisiting the dependency lifetime proof. - if sizes::FLIGHTS != 1 { - return Err(EINVAL); - } + // signed half-window: at most FLIGHTS commands are published at once, far inside it. Ok(Self { pending: KVec::new(), flights: KVec::with_capacity(sizes::FLIGHTS, GFP_KERNEL)?, @@ -454,9 +464,10 @@ impl Engine { chan_wait: false, progress_flight: 0, progress: [None; 3], - stats: Stats::default(), - prof: Prof::default(), + stats: KBox::new(Stats::default(), GFP_KERNEL)?, + prof: KBox::new(Prof::default(), GFP_KERNEL)?, debugfs: None, + tune_file: None, // Allocate host encoder buffers before any submitted fence can depend on // the device mutex. Publication must never enter direct reclaim here. scratch: Some(KBox::new(Scratch::new()?, GFP_KERNEL)?), @@ -637,14 +648,19 @@ impl Firmware { // owner, not the next command that happens to occupy the flight list. return Ok(()); } - let f = self.engine.flights.first_mut().ok_or(ENOENT)?; - let matches = match event { - 15 => f.computes != 0 && f.target == target, - 0x70 => f.renders != 0 && f.ta_target == target, - 0x71 => f.renders != 0 && f.fs_target == target, - _ => false, - }; - if !matches || flags & 1 == 0 { + // The named owner: the oldest flight, or with compute queued behind it a later one. + let f = self + .engine + .flights + .iter_mut() + .find(|f| match event { + 15 => f.computes != 0 && f.target == target, + 0x70 => f.renders != 0 && f.ta_target == target, + 0x71 => f.renders != 0 && f.fs_target == target, + _ => false, + }) + .ok_or(ENOENT)?; + if flags & 1 == 0 { return Err(ENOENT); } if flags & 9 == 9 @@ -674,19 +690,24 @@ impl Firmware { _event: u32, fault_asid: Option, ) -> Result { - let Some(f) = self.engine.flights.first_mut() else { - return Ok(()); - }; let reached = super::recovery::logic::stamp_reached; - let unfinished = (f.computes != 0 && !reached(stage[2], f.target, self.engine.target)) - || (f.renders != 0 - && (!reached(stage[0], f.ta_target, self.engine.ta_target) - || !reached(stage[1], f.fs_target, self.engine.fs_target))); - if unfinished { - f.recovery_suspect = true; - f.recovery_started_ns = now_ns(); - if fault_asid.is_some_and(|asid| asid >= 1 && asid == f.job.vm.slot) { - f.doomed = true; + let (target, ta_target, fs_target) = ( + self.engine.target, + self.engine.ta_target, + self.engine.fs_target, + ); + // Every unfinished flight, including compute queued behind the oldest. + for f in self.engine.flights.iter_mut() { + let unfinished = (f.computes != 0 && !reached(stage[2], f.target, target)) + || (f.renders != 0 + && (!reached(stage[0], f.ta_target, ta_target) + || !reached(stage[1], f.fs_target, fs_target))); + if unfinished { + f.recovery_suspect = true; + f.recovery_started_ns = now_ns(); + if fault_asid.is_some_and(|asid| asid >= 1 && asid == f.job.vm.slot) { + f.doomed = true; + } } } Ok(()) @@ -754,8 +775,10 @@ impl Firmware { }; let job = self.engine.pending[index].clone(); // Even a failed job may have an earlier command in flight. An aggregate - // error fence must not race that command's access to user buffers. - if !self.engine.flights.is_empty() { + // error fence must not race that command's access to user buffers: only a + // healthy compute command queues behind in-flight compute; everything else + // (and every failure path below) waits for the engine to drain. + if !self.engine.flights.is_empty() && !self.can_queue_behind(&job) { self.engine.stats.hold[0] += 1; break; } @@ -788,6 +811,20 @@ impl Firmware { } None => None, }; + // A VM whose own units were all quarantined (without the class being exhausted) + // cannot run compute again: cancel its queue like an exhausted class would. + if class_error.is_none() + && matches!(command(&job, job.progress.sent()), Some(Command::Compute(_))) + && job + .resources + .compute_units() + .is_ok_and(|n| self.slots.compute_exhausted_within(n)) + { + self.engine.unpend(&job); + let _ = job.mark_terminal_retired(); + complete(done, &job, Err(ECANCELED)); + continue; + } if let Some((class, error)) = class_error { self.mark_class_unavailable(class, error); self.engine.unpend(&job); @@ -817,6 +854,15 @@ impl Firmware { } continue; } + Err(PublicationError::Unpublished(e)) + if e == EAGAIN && !self.engine.flights.is_empty() => + { + // Waits for an in-flight command: its retirement repolls publication, + // and the flights' own deadlines apply. + let i = if self.engine.chan_wait { 1 } else { 2 }; + self.engine.stats.hold[i] += 1; + break; + } Err(PublicationError::Unpublished(e)) if e == EAGAIN => { let i = if self.engine.chan_wait { 1 } else { 2 }; self.engine.stats.hold[i] += 1; @@ -871,6 +917,52 @@ impl Firmware { result } + /// Whether `job`'s next command may be published behind the flights in flight: it is a + /// healthy compute command whose dependencies are met, every flight is a healthy + /// compute-only prefix of the same VM, and the compute depth is below + /// [`params::compute_flights`]. Renders, another VM's commands and every failure path + /// need an empty engine: under several clients, compute queued behind another VM's + /// lost its writes and stalled the firmware compute queue. + fn can_queue_behind(&self, job: &Arc) -> bool { + let e = &self.engine; + if e.flights.len() >= params::compute_flights() + || e.flights.iter().any(|f| { + f.renders != 0 + || f.doomed + || f.recovery_suspect + || !Arc::ptr_eq(&f.job.vm, &job.vm) + || Self::writes_timestamps(f) + }) + || job.vm.status.get() != 0 + || self.slots.compute_exhausted() + || super::recovery::logic::next_ordinal(e.ordinal).is_none() + || !job + .resources + .compute_units() + .is_ok_and(|units| self.slots.compute_available(units)) + { + return false; + } + let sent = job.progress.sent(); + matches!(command(job, sent), Some(Command::Compute(_))) + && job.check_command_dependencies(sent).is_ok() + } + + /// Whether a flight's commands have user timestamps. The host writes them when the + /// command retires, so a command queued behind it (a query copy, possibly waiting on + /// availability) could run before they exist; it waits for the retirement instead. + fn writes_timestamps(f: &Flight) -> bool { + f.job + .commands + .get(f.first..f.first + f.count) + .is_none_or(|commands| { + commands.iter().any(|command| match command { + Command::Compute(c) => !c.timestamp_outputs.is_empty(), + Command::Render(r) => !r.timestamp_outputs.is_empty(), + }) + }) + } + /// Whether some of `job`'s commands were published: such a job is retained after a /// fail-stop. pub(crate) fn job_partly_published(&self, job: &Job) -> bool { @@ -970,7 +1062,8 @@ impl Firmware { f: &mut Flight, visible: &mut bool, ) -> Result { - if !self.engine.flights.is_empty() { + let behind = !self.engine.flights.is_empty(); + if behind && !self.can_queue_behind(job) { return Err(EAGAIN); } let index = job.progress.sent(); @@ -994,10 +1087,16 @@ impl Firmware { for &channel in channels { let pipe = self.channels.channel(channel); let producer = self.channels.producer(channel); - if pipe.read()? != producer - || pipe.completion()? != producer - || pipe.write()? != producer - { + let busy = if behind { + // Behind in-flight compute: post after its unconsumed records, with room. + pipe.write()? != producer + || producer.wrapping_sub(pipe.read()?) % RING_SLOTS >= sizes::AHEAD_RECORDS + } else { + pipe.read()? != producer + || pipe.completion()? != producer + || pipe.write()? != producer + }; + if busy { self.engine.chan_wait = true; return Err(EAGAIN); } @@ -1017,6 +1116,15 @@ impl Firmware { if prefix.c_head >= 256 || prefix.ta_head >= 256 || prefix.fs_head >= 256 { return Err(super::boot::eproto()); } + if behind { + // Room for one more compute entry behind the unconsumed ones. + let read = self.graph.compute.qstate.r32(qstate::READ)?; + if read >= 256 + || prefix.c_head.wrapping_sub(read) % sizes::QUEUE_CAP >= sizes::QUEUE_CAP - 2 + { + return Err(EAGAIN); + } + } f.c_start = prefix.c_head; f.ta_start = prefix.ta_head; f.fs_start = prefix.fs_head; @@ -1081,7 +1189,8 @@ impl Firmware { self.graph.render.threshold.w32(8, prefix.draw as u32)?; } else { let k = f.claims.slot[0] as usize; - job.resources.compute()?.compute[k] + job.resources + .compute_unit(k)? .counter .w64(0, f.uma_submitted)?; self.graph.compute.threshold.w32(0, prefix.n as u32)?; @@ -1102,6 +1211,8 @@ impl Firmware { )?; } wmb(); + // Every command posts its own record: the firmware does not pick up a WRITE + // advance without one, also while the queue is running. for &channel in channels { let (qinfo, head, event, fresh) = match channel { Channel::Compute => ( @@ -1131,7 +1242,11 @@ impl Firmware { event, new_queue: u8::from(fresh), }; - self.channels.post(channel, &record)?; + if behind { + self.channels.post_ahead(channel, &record)?; + } else { + self.channels.post(channel, &record)?; + } self.session.ring(channel)?; } Ok(()) @@ -1162,9 +1277,14 @@ impl Firmware { } self.engine.events = self.events.compute_events; self.engine.stage_seen = self.events.stage_events; + f.behind = behind; self.stat_publish(f); - self.renew_deadlines(); + if !behind { + // A command behind others keeps the oldest's deadline: it starts only after them. + self.renew_deadlines(); + } job.progress.set_sent(index + 1); + job.command_published(index); let mut flight = Flight::new(job.clone()); core::mem::swap(&mut flight, f); self.engine @@ -1176,6 +1296,8 @@ impl Firmware { core::mem::forget(flight); EIO })?; + let s = &mut self.engine.stats; + s.max_flights = s.max_flights.max(self.engine.flights.len() as u64); publication.map_err(|_| EIO) } @@ -1192,11 +1314,11 @@ impl Firmware { let target = encode::stamp(n); let index = self .slots - .claim_compute(job.resources.compute()?, &mut f.claims, 0)? + .claim_compute(&job.resources, &mut f.claims, 0)? .ok_or(EAGAIN)?; let generation = job.vm.generation; self.slots.hold_vm(&f.claims, &job.vm)?; - let s = &job.resources.compute()?.compute[index]; + let s = job.resources.compute_unit(index)?; let g = &self.graph.compute; let before = s.counter.r64(0)?; let submitted = before.checked_add(1).ok_or(EOVERFLOW)?; @@ -1221,7 +1343,9 @@ impl Firmware { usc_base: job.queue.usc_base, record_fw: g.notifier.va + notifier_page::COMPUTE_RECORD as u64, binding_generation: generation, - uma_slot: 1, + // Each unit owns its UMA slot: the firmware starts a queued command while the + // one before it still runs, and two owners of one slot then corrupt each other. + uma_slot: engine_ids::COMPUTE_UMA_SLOT + index as u32, // Preserve the already qualified packet bytes while expressing the native // fields separately. This compatibility identifier is not proof of native // global command-ID allocation or uniqueness after compute-index wrap. @@ -1384,6 +1508,10 @@ impl Firmware { let s = &mut self.engine.stats; s.pub_ns = now_ns(); f.published_ns = s.pub_ns; + if f.behind { + s.behind += 1; + s.overlaps += 1; + } s.prep_ns += s.pub_ns.wrapping_sub(s.prep_from_ns); if s.idle_from_ns != 0 { s.gap_ns += s.pub_ns.wrapping_sub(s.idle_from_ns); diff --git a/drivers/gpu/drm/asahi/g16/hw.rs b/drivers/gpu/drm/asahi/g16/hw.rs index 353199dea75a1..4aec01a3105c5 100644 --- a/drivers/gpu/drm/asahi/g16/hw.rs +++ b/drivers/gpu/drm/asahi/g16/hw.rs @@ -182,10 +182,17 @@ pub(crate) mod client { pub(crate) const CLIENT_APERTURE: Range = 0x7000000000..0x7100000000; pub(crate) const COMPACT_WINDOW: Range = 0x1000000000..0x9000000000; - // Two persistent compute units, each in its own 64 MiB interval and reused across - // application queues. Every alias is inside Mesa's reserved driver hole. + // Persistent compute units, each in its own 64 MiB interval and reused across + // application queues. Units 0 and 1 keep their original placement below the render + // allocators; further units (up to [`super::sizes::COMPUTE_UNITS`]) live between the + // unused CT window and the PB heap, clear of the dynamic PB page-list range + // 0x7040000000..0x7080000000. Every alias is inside Mesa's reserved driver hole. pub(crate) const fn compute_base(k: u64) -> u64 { - 0x7001000000 + k * 0x04000000 + if k < 2 { + 0x7001000000 + k * 0x04000000 + } else { + 0x7090000000 + (k - 2) * 0x04000000 + } } pub(crate) const fn compute_pool(k: u64) -> u64 { compute_base(k) @@ -280,13 +287,20 @@ pub(crate) mod sizes { pub(crate) const PB_BLOCKS: u32 = 2048; pub(crate) const PB_LIST_RECORDS: u32 = 4096; pub(crate) const PB_GROW_STEP: u32 = 10; - /// The qualified M4 protocol permits only one published command at a time. - pub(crate) const FLIGHTS: usize = 1; + /// Published prefixes at most. A render is published only with nothing else in flight; + /// compute commands may queue behind each other on the one firmware compute queue, each + /// in its own retained unit ([`COMPUTE_UNITS`]). + pub(crate) const FLIGHTS: usize = 16; + /// Retained compute units per VM at most, all in hardware UMA slot 1. A unit is reused + /// only after a later publication retired (it displaced the unit), so `n` units keep up + /// to `n - 1` compute commands in flight. + pub(crate) const COMPUTE_UNITS: usize = 8; + const _: () = assert!(super::client::compute_base(COMPUTE_UNITS as u64) <= super::client::PB_HEAP); /// Inherited diagnostic histogram width; M4 does not fold jobs into a compute horizon. pub(crate) const HORIZON_JOBS: u32 = 1; - /// Two jobs may be admitted per logical queue so exact prior-command dependencies can - /// advance independently of a preceding job's aggregate fence. Firmware stays serialized. - pub(crate) const SCHEDULER_CREDITS: u32 = 2; + /// Jobs admitted to the engine per logical queue: enough to keep a queue's compute + /// commands in flight back to back; exact prior-command dependencies still gate each one. + pub(crate) const SCHEDULER_CREDITS: u32 = 16; /// Unconsumed records per 256-entry channel ring a prefix may be posted behind. pub(crate) const AHEAD_RECORDS: u32 = 64; /// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit). diff --git a/drivers/gpu/drm/asahi/g16/job.rs b/drivers/gpu/drm/asahi/g16/job.rs index 31f06c80ab8bf..d79d5892d31f4 100644 --- a/drivers/gpu/drm/asahi/g16/job.rs +++ b/drivers/gpu/drm/asahi/g16/job.rs @@ -290,7 +290,12 @@ impl Job { .completed .each_ref() .map(|value| value.load(Ordering::Acquire)); - if !order.ready(completed) { + let published = self + .queue + .published + .each_ref() + .map(|value| value.load(Ordering::Acquire)); + if !order.ready(completed, published) { return Err(EAGAIN); } Ok(()) @@ -299,6 +304,14 @@ impl Job { /// Advances exactly one queue-local engine ordinal after a full successful wire receipt. /// Called under the device lock before another command is considered. No allocation, /// fence callback or aggregate-job completion occurs here. + /// Advances the queue's published frontier past command `index` (under the device + /// lock, when its first producer became visible). + pub(crate) fn command_published(&self, index: u32) { + if let Some(order) = self.orders.get(index as usize) { + self.queue.published[order.engine].fetch_max(order.sequence, Ordering::AcqRel); + } + } + pub(crate) fn command_retired(&self, index: u32) -> Result { let order = self.orders.get(index as usize).ok_or(EINVAL)?; let previous = order.sequence.checked_sub(1).ok_or(EINVAL)?; diff --git a/drivers/gpu/drm/asahi/g16/params.rs b/drivers/gpu/drm/asahi/g16/params.rs index 65da293ea718f..7ec874cdeab77 100644 --- a/drivers/gpu/drm/asahi/g16/params.rs +++ b/drivers/gpu/drm/asahi/g16/params.rs @@ -28,6 +28,62 @@ pub(crate) fn compute_deadline_ms() -> u32 { module_parameters::g16_compute_deadline_ms.value() } +/// Runtime tunables, written through `aurora-g16/tune` (debugfs, root only) as `name=value`. +pub(crate) mod tune { + use core::sync::atomic::{AtomicU32, Ordering}; + + /// Compute units of a VM that submits compute back to back + /// (2..=[`super::hw::sizes::COMPUTE_UNITS`]); applies to VMs that grow after the change. + pub(crate) static COMPUTE_UNITS: AtomicU32 = AtomicU32::new(6); + /// Compute commands in flight at most (1 = the serialized engine). + pub(crate) static COMPUTE_FLIGHTS: AtomicU32 = AtomicU32::new(16); + /// Completion repoll interval override, us (0: `g16_resnap_us`). + pub(crate) static RESNAP_US: AtomicU32 = AtomicU32::new(0); + const NAMES: [(&str, &AtomicU32, u32, u32); 3] = [ + ("compute_units", &COMPUTE_UNITS, 2, super::hw::sizes::COMPUTE_UNITS as u32), + ("compute_flights", &COMPUTE_FLIGHTS, 1, super::hw::sizes::FLIGHTS as u32), + ("resnap_us", &RESNAP_US, 0, 1000), + ]; + + /// Applies one `name=value` line. + pub(crate) fn set(line: &str) -> kernel::error::Result { + let (name, value) = line.trim().split_once('=').ok_or(kernel::error::code::EINVAL)?; + let value: u32 = value + .trim() + .parse() + .map_err(|_| kernel::error::code::EINVAL)?; + for (n, cell, lo, hi) in NAMES { + if n == name.trim() { + if value < lo || value > hi { + return Err(kernel::error::code::ERANGE); + } + cell.store(value, Ordering::Relaxed); + return Ok(()); + } + } + Err(kernel::error::code::ENOENT) + } + + pub(crate) fn show(f: &mut impl core::fmt::Write) -> core::fmt::Result { + for (n, cell, _, _) in NAMES { + write!(f, "{}={} ", n, cell.load(Ordering::Relaxed))?; + } + writeln!(f) + } +} + +/// Compute units of a VM that submits compute back to back. +pub(crate) fn compute_units() -> usize { + (tune::COMPUTE_UNITS.load(core::sync::atomic::Ordering::Relaxed) as usize) + .clamp(2, hw::sizes::COMPUTE_UNITS) +} + +/// Compute commands in flight at most. +pub(crate) fn compute_flights() -> usize { + (tune::COMPUTE_FLIGHTS.load(core::sync::atomic::Ordering::Relaxed) as usize) + .clamp(1, hw::sizes::FLIGHTS) +} + /// Repoll while a ready prefix waits for record consumption, us. pub(crate) fn repoll_us() -> u32 { module_parameters::g16_repoll_us.value().clamp(5, 100) @@ -35,7 +91,10 @@ pub(crate) fn repoll_us() -> u32 { /// Completion repoll interval, us. pub(crate) fn resnap_us() -> u32 { - module_parameters::g16_resnap_us.value().clamp(25, 1000) + match tune::RESNAP_US.load(core::sync::atomic::Ordering::Relaxed) { + 0 => module_parameters::g16_resnap_us.value().clamp(25, 1000), + us => us.clamp(10, 1000), + } } /// Whether firmware halts are answered with the handshake (else fail-stop). diff --git a/drivers/gpu/drm/asahi/g16/poll.rs b/drivers/gpu/drm/asahi/g16/poll.rs index 1d91555e6694b..ce1a76fffe988 100644 --- a/drivers/gpu/drm/asahi/g16/poll.rs +++ b/drivers/gpu/drm/asahi/g16/poll.rs @@ -56,6 +56,25 @@ impl HrTimerCallback for Shared { } } +/// The `aurora-g16/tune` debugfs file: current tunables. +fn tune_show(_shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { + super::params::tune::show(f) +} + +/// The `aurora-g16/tune` debugfs file: `name=value` lines. +fn tune_write(_shared: &Arc, reader: &mut kernel::uaccess::UserSliceReader) -> Result { + let mut buf = [0u8; 128]; + let len = reader.len().min(buf.len()); + reader.read_slice(&mut buf[..len])?; + let text = core::str::from_utf8(&buf[..len]).map_err(|_| EINVAL)?; + for line in text.split(|c| c == '\n' || c == ' ' || c == ',') { + if !line.trim().is_empty() { + super::params::tune::set(line)?; + } + } + Ok(()) +} + /// The `aurora-g16/stats` debugfs file contents. fn stats_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { let state = shared.state.lock(); @@ -198,10 +217,15 @@ impl Shared { dir.read_callback_file(c_str!("stats"), self.clone(), &stats_show), GFP_KERNEL, )?; + let tune = KBox::pin_init( + dir.read_write_callback_file(c_str!("tune"), self.clone(), &tune_show, &tune_write), + GFP_KERNEL, + )?; let mut state = self.state.lock(); let firmware = state.as_deref_mut().ok_or(ENODEV)?; firmware.engine.shared = Some(self.clone()); firmware.engine.debugfs = Some(file); + firmware.engine.tune_file = Some(tune); Ok(()) } } @@ -212,7 +236,6 @@ pub(crate) fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { } /// Cursor `c` of a ring of `m` entries passed end `h` of entries from `b`, within tail `t`. -#[allow(dead_code)] pub(crate) fn ring_passed(c: u32, b: u32, h: u32, t: u32, m: u32) -> bool { let d = |x: u32| x.wrapping_add(m).wrapping_sub(b) % m; let (dc, dh, dt) = (d(c), d(h), d(t)); @@ -341,6 +364,40 @@ impl Firmware { Ok(true) } + /// In-order receipt of compute flight `f` with later flights queued behind it: the NC + /// READ cursor (and with uncached queue info its consumers) passed its ring entries, + /// and the submit pipe's read and completion cursors passed its record. + fn queue_passed( + &self, + channel: Channel, + state: &super::object::FwObject, + info: &super::object::FwObject, + f: &Flight, + ) -> Result { + let cap = super::hw::sizes::QUEUE_CAP; + let tail = self.engine.head; + if f.head >= cap || f.c_start >= cap || tail >= cap { + return Err(EINVAL); + } + if !ring_passed(state.r32(qstate::READ)?, f.c_start, f.head, tail, cap) { + return Ok(false); + } + if !super::hw::tuning::FW_CACHED { + for offset in [0x20, 0x24, 0x28] { + if !ring_passed(info.r32(offset)?, f.c_start, f.head, tail, cap) { + return Ok(false); + } + } + } + let slots = super::fw::channels::RING_SLOTS; + let record = f.rec[2]; + let first = (record + slots - 1) % slots; + let producer = self.channels.producer(channel); + let pipe = self.channels.channel(channel); + Ok(ring_passed(pipe.read()?, first, record, producer, slots) + && ring_passed(pipe.completion()?, first, record, producer, slots)) + } + /// A full receipt includes stage and notifier stamps, every physical consumer, /// the private allocator completion and the internal timestamp pair. An aborted /// receipt omits only the writes firmware does not make for an aborted command, @@ -359,7 +416,33 @@ impl Firmware { }; rmb(); let mut full = true; - if f.computes != 0 { + // Later compute flights queued behind this one: the queue, pipe and stamp cells + // already moved on, so their receipt is "passed", in order; the newest flight keeps + // the exact receipt. + let behind = self.engine.flights.len() > 1; + if f.computes != 0 && behind { + let g = &self.graph.compute; + let e = &self.engine; + if !stamp_reached(g.fw_stamp.r32(0)?, f.target, e.target) + || !self.queue_passed(Channel::Compute, &g.qstate, &g.qinfo, f)? + { + return Ok(Retirement::Pending); + } + let k = f.claims.slot[0] as usize; + let unit = f.job.resources.compute_unit(k)?; + let submitted = unit.counter.r64(0)?; + let start = unit.its.r64(0)?; + let end = unit.its.r64(8)?; + let stamp = g.notifier.r32(notifier_page::COMPUTE_STAMP)?; + full &= stamp_reached(stamp, f.target, e.target) + && (stamp != f.target + || g.notifier.r32(notifier_page::COMPUTE_CONTEXT)? == f.job.vm.slot) + && submitted != 0 + && unit.work.r64(compute::DESCRIPTOR_COMPLETED)? == submitted + && unit.work.r64(compute::DESCRIPTOR_COUNTER)? == unit.counter.va + && start != 0 + && end > start; + } else if f.computes != 0 { let g = &self.graph.compute; if g.fw_stamp.r32(0)? != f.target || !self.queue_receipt(Channel::Compute, &g.qstate, &g.qinfo, f.head, f.rec[2])? @@ -367,7 +450,7 @@ impl Firmware { return Ok(Retirement::Pending); } let k = f.claims.slot[0] as usize; - let unit = f.job.resources.compute()?.compute.get(k).ok_or(EINVAL)?; + let unit = f.job.resources.compute_unit(k)?; let submitted = unit.counter.r64(0)?; let start = unit.its.r64(0)?; let end = unit.its.r64(8)?; @@ -564,6 +647,15 @@ impl Firmware { let job = f.job.clone(); let p = &job.progress; p.next.store(p.next() + f.count as u32, Ordering::Relaxed); + if result.is_err() { + // Its queue's later commands queued behind it fail with it, as unpublished ones + // are cancelled: they never report a successful receipt after this hole. + for later in self.engine.flights.iter_mut() { + if Arc::ptr_eq(&later.job.queue, &job.queue) { + later.doomed = true; + } + } + } if let Err(error) = result { if p.is_pending() { // Drop its unpublished commands. @@ -662,13 +754,7 @@ impl Firmware { match command { super::command::Command::Compute(command) => { let index = *slots.get(compute).ok_or(EINVAL)? as usize; - let unit = flight - .job - .resources - .compute()? - .compute - .get(index) - .ok_or(EINVAL)?; + let unit = flight.job.resources.compute_unit(index)?; let ticks = [unit.its.r64(0)?, unit.its.r64(8)?, 0, 0]; command .timestamp_outputs @@ -758,10 +844,9 @@ impl Firmware { ); } for k in 0..(f.computes as usize).min(f.claims.slot_count as usize) { - let Some(arena) = f.job.resources.compute.as_ref() else { + let Ok(c) = f.job.resources.compute_unit(f.claims.slot[k] as usize) else { return; }; - let c = &arena.compute[f.claims.slot[k] as usize]; stage(&c.its, 0, &mut s.cdm_ticks, crate::usage::Engine::Compute); } if hi > lo { @@ -998,6 +1083,20 @@ impl Firmware { s.span_ticks / 24 )?; // The phase timers of debug_flags bit 49, and the scrub flush waits. + { + let s = &self.engine.stats; + writeln!( + f, + "pipeline: behind {} max_flights {} compute_published {} hold {}/{}/{}", + s.behind, + s.max_flights, + self.slots.compute_published(), + s.hold[0], + s.hold[1], + s.hold[2] + )?; + super::params::tune::show(f)?; + } let p = &self.engine.prof; writeln!( f, diff --git a/drivers/gpu/drm/asahi/g16/pools/lifetime.rs b/drivers/gpu/drm/asahi/g16/pools/lifetime.rs index baf623b2722d7..2f156546ab307 100644 --- a/drivers/gpu/drm/asahi/g16/pools/lifetime.rs +++ b/drivers/gpu/drm/asahi/g16/pools/lifetime.rs @@ -1,7 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Bounded two-owner displacement ledger. A completion alone never makes an M4 UMA owner -//! reusable: a distinct, fully retired owner must replace it in the same hardware slot. +//! Bounded displacement ledger of the retained owners of one hardware UMA slot. A completion +//! alone never makes an M4 UMA owner reusable: a later, fully retired publication must have +//! replaced it in the same hardware slot. +//! +//! The firmware runs one queue's commands strictly in order, each to its Finalize, so a +//! publication that retired started after every earlier one finished and displaced it. Owners +//! therefore retire in publication order and an owner is reusable once any owner published +//! after it retired. With `pipelined`, owners may be published while earlier ones are still +//! in flight (the compute queue); without it, publication stays serialized against every +//! owner (the render pass slots). +//! //! Core-only so the transition system is executable in host tests. #[derive(Copy, Clone, Debug, Eq, PartialEq)] @@ -33,13 +42,13 @@ pub(crate) struct Counts { } #[derive(Clone)] -pub(crate) struct Lifetime { - units: [Unit; 2], +pub(crate) struct Lifetime { + units: [Unit; N], sequence: u64, - last_owner: Option, + pipelined: bool, } -impl Lifetime { - pub(crate) const fn new() -> Self { +impl Lifetime { + pub(crate) const fn new(pipelined: bool) -> Self { let unit = Unit { state: State::Fresh, previous: State::Fresh, @@ -47,9 +56,9 @@ impl Lifetime { publication: 0, }; Self { - units: [unit; 2], + units: [unit; N], sequence: 0, - last_owner: None, + pipelined, } } #[allow(dead_code)] @@ -70,16 +79,23 @@ impl Lifetime { } counts } + /// Published owners. + pub(crate) fn published(&self) -> u32 { + self.units + .iter() + .filter(|u| u.state == State::Published) + .count() as u32 + } pub(crate) fn available(&self, k: usize) -> bool { let Some(u) = self.units.get(k) else { return false; }; match u.state { State::Fresh => true, - State::Retired => self.last_owner.is_some_and(|b| { - b != k - && self.units[b].state == State::Retired - && self.units[b].publication > u.publication + // A later publication retired: it started only after this owner finished, and + // its load displaced this owner from the hardware slot. + State::Retired => self.units.iter().enumerate().any(|(b, other)| { + b != k && other.state == State::Retired && other.publication > u.publication }), _ => false, } @@ -87,7 +103,11 @@ impl Lifetime { /// No amount of firmware progress can make a unit claimable. The caller must fail /// pending work rather than wait forever after abort/quarantine or sequence exhaustion. pub(crate) fn exhausted(&self) -> bool { - (0..2).all(|k| !self.available(k)) + self.exhausted_within(N) + } + /// [`Self::exhausted`] when only owners `0..n` exist. + pub(crate) fn exhausted_within(&self, n: usize) -> bool { + (0..n.min(N)).all(|k| !self.available(k)) && !self .units .iter() @@ -113,22 +133,30 @@ impl Lifetime { } } pub(crate) fn publish(&mut self, k: usize) -> bool { - if self.units[k].state != State::Preparing || self.sequence == u64::MAX { + if k >= N || self.units[k].state != State::Preparing || self.sequence == u64::MAX { return false; } - // Publication must be serialized against every owner of this hardware slot. - if self.units.iter().any(|u| u.state == State::Published) { + // Without pipelining, publication is serialized against every owner of the slot. + if !self.pipelined && self.units.iter().any(|u| u.state == State::Published) { return false; } self.sequence += 1; self.units[k].publication = self.sequence; self.units[k].state = State::Published; - self.last_owner = Some(k); true } - /// Called only after the caller has established complete, exact wire retirement. + /// Called only after the caller has established complete, exact wire retirement. Owners + /// retire in publication order: `k` must be the oldest published owner. pub(crate) fn retire(&mut self, k: usize) -> bool { - if self.units[k].state != State::Published || self.last_owner != Some(k) { + if k >= N || self.units[k].state != State::Published { + return false; + } + let publication = self.units[k].publication; + if self + .units + .iter() + .any(|u| u.state == State::Published && u.publication < publication) + { return false; } self.units[k].state = State::Retired; @@ -145,7 +173,7 @@ mod tests { use super::*; #[test] fn distinct_retired_displacement_is_required() { - let mut l = Lifetime::new(); + let mut l = Lifetime::<2>::new(false); assert!(l.claim(0)); assert!(l.publish(0)); assert!(!l.claim(0)); @@ -164,7 +192,7 @@ mod tests { } #[test] fn abort_does_not_recycle_or_authorize_displacement() { - let mut l = Lifetime::new(); + let mut l = Lifetime::<2>::new(false); assert!(l.claim(0)); assert!(l.publish(0)); assert!(l.retire(0)); @@ -185,7 +213,7 @@ mod tests { } #[test] fn preparation_cancel_preserves_retirement() { - let mut l = Lifetime::new(); + let mut l = Lifetime::<2>::new(false); assert!(l.claim(0)); l.cancel(0); assert_eq!(l.generation(0), 0); @@ -202,7 +230,7 @@ mod tests { } #[test] fn publication_is_serialized_and_command_count_is_not_binding_generation() { - let mut l = Lifetime::new(); + let mut l = Lifetime::<2>::new(false); assert!(l.claim(0)); assert!(l.publish(0)); assert!(l.claim(1)); @@ -219,4 +247,56 @@ mod tests { assert_eq!(l.generation(1), 1); assert!(l.claim(0)); } + #[test] + fn pipelined_owners_keep_n_minus_one_in_flight() { + let mut l = Lifetime::<4>::new(true); + // Fresh owners publish back to back. + for k in 0..4 { + assert!(l.claim(k)); + assert!(l.publish(k)); + } + assert_eq!(l.published(), 4); + // Out of order retirement is refused. + assert!(!l.retire(1)); + assert!(l.retire(0)); + // Owner 0 retired but nothing after it has: not displaced yet. + assert!(!l.claim(0)); + assert!(l.retire(1)); + assert!(l.claim(0)); // displaced by 1 + assert!(!l.claim(1)); // 2 still in flight + assert!(l.publish(0)); + assert!(l.retire(2)); + assert!(l.claim(1)); + assert!(l.publish(1)); + // Steady state: round robin with up to three in flight. + let mut next = 2; + for _ in 0..1000 { + let oldest = (0..4) + .filter(|&k| l.units[k].state == State::Published) + .min_by_key(|&k| l.units[k].publication) + .unwrap(); + assert!(l.retire(oldest)); + assert!(l.claim(next)); + assert!(l.publish(next)); + assert!(l.published() <= 3); + next = (next + 1) % 4; + } + } + #[test] + fn pipelined_newest_retired_owner_is_never_reusable() { + let mut l = Lifetime::<3>::new(true); + for k in 0..3 { + assert!(l.claim(k)); + assert!(l.publish(k)); + } + for k in 0..3 { + assert!(l.retire(k)); + } + assert!(l.available(0) && l.available(1)); + assert!(!l.available(2)); + assert!(l.claim(1)); + // 2 still proves 0 displaced although 1 is being prepared again. + assert!(l.available(0)); + assert!(!l.available(2)); + } } diff --git a/drivers/gpu/drm/asahi/g16/queue.rs b/drivers/gpu/drm/asahi/g16/queue.rs index 41da5b94c7c50..ada1946bb7983 100644 --- a/drivers/gpu/drm/asahi/g16/queue.rs +++ b/drivers/gpu/drm/asahi/g16/queue.rs @@ -55,6 +55,10 @@ pub(crate) struct QueueState { /// Exact successful firmware receipts, independently for render and compute. These /// survive aggregate-fence publication and never advance on aborted/failed commands. pub(crate) completed: [AtomicU64; 2], + /// Published command frontier per engine. The firmware runs the compute queue in + /// publication order, each command to its Finalize, so a compute command may follow its + /// published (not yet retired) compute predecessors. + pub(crate) published: [AtomicU64; 2], /// Owned-job counts at the queue's last 32 arrivals (compute horizon ahead), recorded by the /// engine when a job arrives. pub(crate) owned: [AtomicU32; 32], @@ -140,6 +144,7 @@ impl Queue { cancelled_at: [const { AtomicU64::new(0) }; 2], turn: AtomicU64::new(0), completed: [const { AtomicU64::new(0) }; 2], + published: [const { AtomicU64::new(0) }; 2], owned: [const { AtomicU32::new(0) }; 32], owned_index: AtomicU32::new(0), }, diff --git a/drivers/gpu/drm/asahi/g16/resources.rs b/drivers/gpu/drm/asahi/g16/resources.rs index 77c47f0770492..af31073f6ac15 100644 --- a/drivers/gpu/drm/asahi/g16/resources.rs +++ b/drivers/gpu/drm/asahi/g16/resources.rs @@ -34,17 +34,30 @@ pub(crate) struct ComputeInner { _bind: mmu::VmBind, _live: Arc, } +/// Compute units every VM gets with its first compute command; more are added +/// ([`Cache::prepare`]) once it submits compute back to back. +pub(crate) const BASE_COMPUTE_UNITS: usize = 2; + +/// Back-to-back compute submissions (each within [`BURST_GAP_NS`] of the previous) after +/// which a VM gets its extension units; a single submission with [`BURST_COMMANDS`] compute +/// commands qualifies at once. +const BURST_SUBMITS: u32 = 3; +const BURST_GAP_NS: u64 = 2_000_000; +const BURST_COMMANDS: usize = 3; + impl ComputeArena { + /// Units `units` (their index fixes their client addresses and hardware UMA slot). fn new( factory: &Factory, bind: &mmu::VmBind, vm: &mmu::Vm, live: &Arc, + units: core::ops::Range, ) -> Result> { let mut objects = factory.objects(vm); - let mut compute = KVec::with_capacity(2, GFP_KERNEL)?; - for k in 0..2 { - compute.push(ComputeSlot::new(&mut objects, k)?, GFP_KERNEL)?; + let mut compute = KVec::with_capacity(units.len(), GFP_KERNEL)?; + for k in units { + compute.push(ComputeSlot::new(&mut objects, k as u64)?, GFP_KERNEL)?; } objects.finish_private()?; Ok(Arc::new( @@ -116,7 +129,12 @@ impl RenderArena { #[derive(Default)] struct Ready { compute: Option>, + /// Units from [`BASE_COMPUTE_UNITS`] on, for a VM that keeps compute queued. + compute_ext: Option>, render: Option>, + /// Last compute submission and the back-to-back run it continues. + last_compute_ns: u64, + burst: u32, } #[pin_data] pub(crate) struct Cache { @@ -141,14 +159,18 @@ impl Cache { ) } pub(crate) fn prepare(&self, vm: &Arc, commands: &[Command]) -> Result { - let compute = commands.iter().any(|c| matches!(c, Command::Compute(_))); + let computes = commands + .iter() + .filter(|c| matches!(c, Command::Compute(_))) + .count(); + let compute = computes != 0; let render_count = commands .iter() .filter(|c| matches!(c, Command::Render(_))) .count(); let render = render_count != 0; let mut epochs = super::budget::Budget::reserve(&self.budget, render_count.try_into()?)?; - let (compute, render) = { + let (compute, compute_ext, render) = { // Never held by publication, completion, recovery or fence signalling. let mut ready = self.ready.lock(); if vm.status.get() != 0 || vm.vm.is_closed() { @@ -160,8 +182,34 @@ impl Cache { &self.bind, &vm.vm, &self.live, + 0..BASE_COMPUTE_UNITS, )?); } + if compute { + // SAFETY: ktime_get has no preconditions. + let now = unsafe { kernel::bindings::ktime_get() } as u64; + ready.burst = if now.wrapping_sub(ready.last_compute_ns) < BURST_GAP_NS { + ready.burst.saturating_add(1) + } else { + 0 + }; + ready.last_compute_ns = now; + let units = super::params::compute_units(); + if ready.compute_ext.is_none() + && units > BASE_COMPUTE_UNITS + && (ready.burst >= BURST_SUBMITS || computes >= BURST_COMMANDS) + { + // Best effort: without them the VM keeps one command in flight. + ready.compute_ext = ComputeArena::new( + &self.factory, + &self.bind, + &vm.vm, + &self.live, + BASE_COMPUTE_UNITS..units, + ) + .ok(); + } + } if render && ready.render.is_none() { ready.render = Some(RenderArena::new( &self.factory, @@ -176,6 +224,7 @@ impl Cache { } ( if compute { ready.compute.clone() } else { None }, + if compute { ready.compute_ext.clone() } else { None }, if render { ready.render.clone() } else { None }, ) }; @@ -195,6 +244,7 @@ impl Cache { } Ok(Prepared { compute, + compute_ext, render, pb, }) @@ -204,6 +254,8 @@ impl Cache { /// Immutable references carried by an accepted job. No fallible lazy work in the engine. pub(crate) struct Prepared { pub(crate) compute: Option>, + /// The VM's extension compute units when the job was prepared. + pub(crate) compute_ext: Option>, pub(crate) render: Option>, pb: KVec>>, } @@ -211,6 +263,23 @@ impl Prepared { pub(crate) fn compute(&self) -> Result<&ComputeArena> { self.compute.as_deref().ok_or(EINVAL) } + /// Compute units this job can use. + pub(crate) fn compute_units(&self) -> Result { + Ok(self.compute()?.compute.len() + + self.compute_ext.as_deref().map_or(0, |e| e.compute.len())) + } + /// Compute unit `k`: the VM's base units, then its extension units. + pub(crate) fn compute_unit(&self, k: usize) -> Result<&ComputeSlot> { + let base = &self.compute()?.compute; + match base.get(k) { + Some(unit) => Ok(unit), + None => self + .compute_ext + .as_deref() + .and_then(|e| e.compute.get(k - base.len())) + .ok_or(EINVAL), + } + } pub(crate) fn render(&self) -> Result<&RenderArena> { self.render.as_deref().ok_or(EINVAL) } diff --git a/drivers/gpu/drm/asahi/g16/slots.rs b/drivers/gpu/drm/asahi/g16/slots.rs index b4a48ed2907ae..79db116659499 100644 --- a/drivers/gpu/drm/asahi/g16/slots.rs +++ b/drivers/gpu/drm/asahi/g16/slots.rs @@ -2,8 +2,9 @@ //! Per-VM M4 command storage and device-wide displacement lifetime tracking. //! -//! Both compute units alternate hardware UMA slot 1; both render units alternate slot 33. -//! Firmware retirement and a later distinct retired owner are both required before reuse. +//! The compute units share hardware UMA slot 1 and may be in flight together, in publication +//! order; both render units alternate slot 33, one at a time. Firmware retirement and a later +//! distinct retired owner are both required before reuse. //! Private backing and old VM leases survive until a distinct replacement fully retires. //! Aborted storage remains mapped and quarantined. Pool cursors are never scrubbed. @@ -370,20 +371,27 @@ pub(crate) struct SlotStats { pub(crate) pass_scrubs: u64, } +/// Displacement ledger and VM leases of the compute units (see [`Slots`]). +pub(crate) struct ComputeOwners { + lifetime: Lifetime<{ sizes::COMPUTE_UNITS }>, + owner: [Option>; sizes::COMPUTE_UNITS], + pending_retarget: [Option; sizes::COMPUTE_UNITS], + pending_owner: [Option>; sizes::COMPUTE_UNITS], +} + pub(crate) struct Slots { pub(crate) cursor: u32, pub(crate) pass_busy: u32, pub(crate) pass_cursor: u32, pub(crate) stats: SlotStats, - compute_lifetime: Lifetime, - pass_lifetime: Lifetime, + /// The compute owners, on the heap: `Firmware` (and so `Slots`) is built on the probe's + /// stack, which has no room for it. + compute: KBox, + pass_lifetime: Lifetime<2>, // Last-owner leases remain bound through retirement and displacement. Replaced only // when the displaced unit is safely rebound to a new publication. - compute_owner: [Option>; 2], pass_owner: [Option>; 2], - pending_compute_retarget: [Option; 2], pending_pass_retarget: [Option; 2], - pending_compute_owner: [Option>; 2], pending_pass_owner: [Option>; 2], } impl Slots { @@ -393,13 +401,18 @@ impl Slots { pass_busy: 0, pass_cursor: 0, stats: SlotStats::default(), - compute_lifetime: Lifetime::new(), - pass_lifetime: Lifetime::new(), - compute_owner: [None, None], + compute: KBox::new( + ComputeOwners { + lifetime: Lifetime::new(true), + owner: [const { None }; sizes::COMPUTE_UNITS], + pending_retarget: [None; sizes::COMPUTE_UNITS], + pending_owner: [const { None }; sizes::COMPUTE_UNITS], + }, + GFP_KERNEL, + )?, + pass_lifetime: Lifetime::new(false), pass_owner: [None, None], - pending_compute_retarget: [None, None], pending_pass_retarget: [None, None], - pending_compute_owner: [None, None], pending_pass_owner: [None, None], }) } @@ -408,10 +421,10 @@ impl Slots { } #[allow(dead_code)] pub(crate) fn compute_generation(&self, k: usize) -> Result { - if k >= 2 { + if k >= sizes::COMPUTE_UNITS { return Err(EINVAL); } - Ok(self.compute_lifetime.generation(k)) + Ok(self.compute.lifetime.generation(k)) } #[allow(dead_code)] pub(crate) fn pass_generation(&self, k: usize) -> Result { @@ -421,39 +434,57 @@ impl Slots { Ok(self.pass_lifetime.generation(k)) } pub(crate) fn compute_exhausted(&self) -> bool { - self.compute_lifetime.exhausted() + // Ledger indices above the most units a VM gets never have an owner. + self.compute + .lifetime + .exhausted_within(super::params::compute_units()) + } + /// Whether none of the first `n` compute units (a VM's own) can ever be claimed again. + pub(crate) fn compute_exhausted_within(&self, n: usize) -> bool { + self.compute.lifetime.exhausted_within(n) } pub(crate) fn render_exhausted(&self) -> bool { self.pass_lifetime.exhausted() } pub(crate) fn lifetime_counts(&self) -> [super::pools::lifetime::Counts; 2] { - [self.compute_lifetime.counts(), self.pass_lifetime.counts()] + [self.compute.lifetime.counts(), self.pass_lifetime.counts()] + } + /// Whether one of the first `n` compute units can be claimed now. + pub(crate) fn compute_available(&self, n: usize) -> bool { + (0..n.min(sizes::COMPUTE_UNITS)).any(|k| self.compute.lifetime.available(k)) + } + /// Compute units published and not yet retired. + pub(crate) fn compute_published(&self) -> u32 { + self.compute.lifetime.published() } #[allow(dead_code)] pub(crate) fn free_compute(&self) -> u32 { - (0..2) - .filter(|&k| self.compute_lifetime.available(k)) + (0..super::params::compute_units()) + .filter(|&k| self.compute.lifetime.available(k)) .count() as u32 } pub(crate) fn claim_compute( &mut self, - arena: &super::resources::ComputeArena, + units: &super::resources::Prepared, f: &mut Claims, _j: u32, ) -> Result> { if f.slot_count != 0 { return Ok(None); } - for i in 0..2 { - let k = (self.cursor as usize + i) % 2; - if self.compute_lifetime.claim(k) { - if let Err(e) = arena.compute[k].scrub() { - self.compute_lifetime.cancel(k); + // Round robin over this VM's units: the next one in publication order is the + // oldest, the first to be displaced by a later retired publication. + let n = units.compute_units()?.min(sizes::COMPUTE_UNITS); + for i in 0..n { + let k = (self.cursor as usize + i) % n; + if self.compute.lifetime.claim(k) { + if let Err(e) = units.compute_unit(k).and_then(|unit| unit.scrub()) { + self.compute.lifetime.cancel(k); return Err(e); } f.slot[0] = k as u32; f.slot_count = 1; - self.cursor = (1 - k) as u32; + self.cursor = ((k + 1) % n) as u32; return Ok(Some(k)); } } @@ -522,7 +553,7 @@ impl Slots { /// Full wire retirement is a separate condition for hardware qualification. pub(crate) fn note_published(&mut self, f: &Claims) { for &k in &f.slot[..f.slot_count as usize] { - if let Some(proof) = self.pending_compute_retarget[k as usize].take() { + if let Some(proof) = self.compute.pending_retarget[k as usize].take() { self.stats.cross_vm_retargets[0] = self.stats.cross_vm_retargets[0].saturating_add(1); self.stats.last_retarget[0] = proof; @@ -543,17 +574,19 @@ impl Slots { /// A previous lease is replaced only after the new publication fully retires. pub(crate) fn hold_vm(&mut self, f: &Claims, vm: &Arc) -> Result { if f.slot_count > 2 - || f.slot[..f.slot_count as usize].iter().any(|&k| k >= 2) + || f.slot[..f.slot_count as usize] + .iter() + .any(|&k| k as usize >= sizes::COMPUTE_UNITS) || f.pass_mask & !3 != 0 { return Err(EINVAL); } for &k in &f.slot[..f.slot_count as usize] { let k = k as usize; - self.pending_compute_retarget[k] = self.compute_owner[k] + self.compute.pending_retarget[k] = self.compute.owner[k] .as_ref() .and_then(|old| Self::vm_retarget(k, old, vm)); - self.pending_compute_owner[k] = Some(vm.clone()); + self.compute.pending_owner[k] = Some(vm.clone()); } for k in 0..2 { if f.pass_mask & (1 << k) != 0 { @@ -569,7 +602,7 @@ impl Slots { /// Reserve the actual same-slot publication. Call before producers become visible. pub(crate) fn publish(&mut self, f: &Claims) -> Result { // Validate on a private copy so a failure changes no ownership. - let mut c = self.compute_lifetime.clone(); + let mut c = self.compute.lifetime.clone(); let mut p = self.pass_lifetime.clone(); for &k in &f.slot[..f.slot_count as usize] { if !c.publish(k as usize) { @@ -581,13 +614,13 @@ impl Slots { return Err(EBUSY); } } - self.compute_lifetime = c; + self.compute.lifetime = c; self.pass_lifetime = p; Ok(()) } /// Full wire retirement is a precondition; completion fences alone are insufficient. pub(crate) fn retire(&mut self, f: &mut Claims) -> Result { - let mut c = self.compute_lifetime.clone(); + let mut c = self.compute.lifetime.clone(); let mut p = self.pass_lifetime.clone(); for &k in &f.slot[..f.slot_count as usize] { if !c.retire(k as usize) { @@ -599,10 +632,10 @@ impl Slots { return Err(EINVAL); } } - self.compute_lifetime = c; + self.compute.lifetime = c; self.pass_lifetime = p; for &k in &f.slot[..f.slot_count as usize] { - self.compute_owner[k as usize] = self.pending_compute_owner[k as usize].take(); + self.compute.owner[k as usize] = self.compute.pending_owner[k as usize].take(); } for k in 0..2 { if f.pass_mask & (1 << k) != 0 { @@ -617,8 +650,8 @@ impl Slots { } pub(crate) fn quarantine(&mut self, f: &mut Claims) { for &k in &f.slot[..f.slot_count as usize] { - self.compute_lifetime.quarantine(k as usize); - self.pending_compute_retarget[k as usize] = None; + self.compute.lifetime.quarantine(k as usize); + self.compute.pending_retarget[k as usize] = None; } for k in 0..2 { if f.pass_mask & (1 << k) != 0 { @@ -634,9 +667,9 @@ impl Slots { /// Cancels preparation only. Published units require retire() or quarantine(). pub(crate) fn release(&mut self, f: &mut Claims) { for &k in &f.slot[..f.slot_count as usize] { - self.compute_lifetime.cancel(k as usize); - self.pending_compute_retarget[k as usize] = None; - self.pending_compute_owner[k as usize] = None; + self.compute.lifetime.cancel(k as usize); + self.compute.pending_retarget[k as usize] = None; + self.compute.pending_owner[k as usize] = None; } for k in 0..2 { if f.pass_mask & (1 << k) != 0 { diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 2a18ebbeb5d9c..7118cffde3301 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -203,6 +203,14 @@ pub(crate) struct Destinations(Option bool { + self.0 + .as_deref() + .is_none_or(|destinations| destinations.iter().all(Option::is_none)) + } + /// G16 retains its eight-byte destination alignment and EINVAL contract. #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn resolve_aligned( diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py index b0c92c63fd028..22ff751a2e415 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py @@ -356,8 +356,16 @@ let order=Order::new([4,8],[2,1],[1,NONE],[3,7],RENDER).unwrap(); assert_eq!(order.sequence,7); assert_eq!(order.wait_through,[Some(5),Some(7)]); - assert!(!order.ready([6,6])); - assert!(order.ready([6,7])); + assert!(!order.ready([6,6],[6,6])); + assert!(order.ready([6,7],[6,7])); + // A compute command may follow published compute predecessors (the firmware compute + // queue runs them in order); every cross-engine dependency still needs completion. + let compute=Order::new([0,2],[0,0],[NONE,NONE],[0,2],COMPUTE).unwrap(); + assert!(!compute.ready([0,1],[0,1])); + assert!(compute.ready([0,1],[0,2])); + let render=Order::new([0,2],[0,0],[NONE,NONE],[0,2],RENDER).unwrap(); + assert!(!render.ready([0,1],[0,2])); + assert!(render.ready([0,2],[0,2])); } #[test] fn geometry_capacity_boundary_uses_the_real_encoder() { diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-job-failure.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-job-failure.py index 62c21f5aad575..efe5a60fd2cfd 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-job-failure.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-job-failure.py @@ -43,7 +43,7 @@ def function(text, name): type Result = core::result::Result; const EIO:Error=Error(-5); const ENODEV:Error=Error(-19); const ECANCELED:Error=Error(-125); const ETIMEDOUT:Error=Error(-110);const ENOMEM:Error=Error(-12);const EAGAIN:Error=Error(-11); const EINVAL:Error=Error(-22); -const EOVERFLOW:Error=Error(-75); const NONE:u16=u16::MAX; +const EOVERFLOW:Error=Error(-75); const NONE:u16=u16::MAX; const COMPUTE:usize=1; ORDER_SOURCE pub mod task { pub const MAX_SCHEDULE_TIMEOUT:i64=i64::MAX; } struct InputState { status:AtomicI32,waits:AtomicUsize,interrupted:AtomicI64,gate:Mutex<()>,changed:Condvar } @@ -96,7 +96,7 @@ def function(text, name): } } mod command { pub(crate) use super::Order; } -struct QueueState { completed:[AtomicU64;2],cancelled_at:[AtomicU64;2] } +struct QueueState { completed:[AtomicU64;2],published:[AtomicU64;2],cancelled_at:[AtomicU64;2] } impl QueueState { QUEUE_METHODS } struct Job { id:u64,vm:Vm,_guard:Guard,progress:Progress,started:AtomicBool,accepted:AtomicBool,fence:Fence,commands:Vec<()>,inputs:Vec,orders:Vec,queue:Arc,full_retired:AtomicU32,terminal_retired:AtomicBool } struct Shared { crashed:AtomicBool,submitted:Mutex>,submit_result:Result } @@ -116,7 +116,7 @@ def function(text, name): } fn job_with_inputs(id:u64,status:Arc,inputs:Vec)->Arc { let orders=vec![Order::new([0,0],[0,0],[NONE;2],[0;2],1).unwrap(),Order::new([0,0],[0,1],[NONE;2],[0;2],1).unwrap()]; - job_with_orders(id,status,inputs,orders,Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]})) + job_with_orders(id,status,inputs,orders,Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],published:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]})) } fn job_with_orders(id:u64,status:Arc,inputs:Vec,orders:Vec,queue:Arc)->Arc { Arc::new(Job{id,vm:Vm{status:status.clone()},_guard:Guard(AtomicBool::new(false)), @@ -278,7 +278,7 @@ def function(text, name): } #[test] fn cancellation_checks_both_command_ordinal_and_cross_engine_failed_prefix() { - let queue=Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]}); + let queue=Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],published:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]}); let cutoff=Order::new([0,2],[0,0],[NONE;2],[0;2],1).unwrap();queue.cancel(&[cutoff],0); let earlier=Order::new([0,1],[0,0],[NONE;2],[0;2],1).unwrap(); let independent_render=Order::new([0,3],[0,0],[NONE;2],[0;2],0).unwrap(); @@ -292,7 +292,7 @@ def function(text, name): } #[test] fn exact_prefix_does_not_wait_for_the_aggregate_or_next_ordinal() { - let queue=Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]}); + let queue=Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],published:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]}); let status=Arc::new(Status::new()); let old=job_with_orders(1,status.clone(),vec![],vec![ Order::new([0,0],[0,0],[NONE;2],[0;2],1).unwrap(), diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-publication.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-publication.py index 6f388b4693fd4..029fee2ee9650 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-publication.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-publication.py @@ -39,7 +39,8 @@ def item(declaration): fn count(&self)->u32{self.count} fn set_pending(&self,v:bool){self.pending.store(v,Ordering::Relaxed)} } -struct Job { id:u32,vm:Vm,queue:Queue,progress:Progress,commands:Vec,dependency:Result,terminal_marked:AtomicBool } +struct Resources;impl Resources{fn compute_units(&self)->Result{Ok(2)}} +struct Job { id:u32,vm:Vm,queue:Queue,progress:Progress,commands:Vec,dependency:Result,terminal_marked:AtomicBool,resources:Resources } impl Job { fn check_command_dependencies(&self,_index:u32)->Result{self.dependency} fn mark_terminal_retired(&self)->Result{assert!(!self.progress.pending.load(Ordering::Relaxed));self.terminal_marked.store(true,Ordering::Relaxed);Ok(())} @@ -64,6 +65,7 @@ def item(declaration): impl Slots{ fn compute_exhausted(&self)->bool{self.compute_exhausted} fn render_exhausted(&self)->bool{self.render_exhausted} + fn compute_exhausted_within(&self,_units:usize)->bool{self.compute_exhausted} fn release(&mut self,_claims:&mut Claims){self.releases+=1;} fn quarantine(&mut self,_claims:&mut Claims){self.quarantines+=1;} } @@ -114,6 +116,9 @@ def item(declaration): } impl Firmware { fn mark_class_unavailable(&mut self,class:usize,error:Error){self.unavailable.push((class,error));} + // Queuing behind in-flight compute is exercised on hardware; this recorder keeps the + // serialized selection and error isolation under test. + fn can_queue_behind(&self,_job:&Arc)->bool{false} fn service_power(&self)->Result{Ok(())} fn drain_rings(&mut self)->Result{Ok(())} fn check_hangs(&mut self,_done:&mut Completions)->Result{Ok(())} @@ -144,7 +149,7 @@ def item(declaration): Arc::new(Job{id,vm:Vm{status:Status(AtomicU32::new(0)),slot:id}, queue:Queue{turn:AtomicU64::new(0),priority:0}, progress:Progress{sent:AtomicU32::new(sent),count:sent+1,pending:AtomicBool::new(true)}, - commands:vec![if render{Command::Render(())}else{Command::Compute(())};sent as usize+1],dependency,terminal_marked:AtomicBool::new(false)}) + commands:vec![if render{Command::Render(())}else{Command::Compute(())};sent as usize+1],dependency,terminal_marked:AtomicBool::new(false),resources:Resources}) } fn firmware(jobs:Vec>,stages:Vec<(Result,bool)>)->Firmware{ Firmware{session:Session{shared:Shared{sleeping:AtomicBool::new(false)}},objects:Checks,recovery:Recovery{error:None,publication_wait_deadline:None}, diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py index 36f4cdeac268e..031d918925c00 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py @@ -31,7 +31,7 @@ // This is the retirement policy used by poll.rs: an affected fence does not // erase the independently authenticated owner receipt. No synthetic receipt // is generated here; the hardware poller remains responsible for that proof. - fn finish(owners: &mut Lifetime, unit: usize, receipt: Retirement, affected: bool) -> bool { + fn finish(owners: &mut Lifetime, unit: usize, receipt: Retirement, affected: bool) -> bool { assert!(receipt.complete()); if receipt.owner_retired() { assert!(owners.retire(unit)); @@ -43,7 +43,7 @@ #[test] fn ten_thousand_affected_full_receipts_preserve_displacement() { - let mut owners = Lifetime::new(); + let mut owners = Lifetime::<2>::new(false); for number in 0..10_000 { let unit = number % 2; assert!(owners.claim(unit)); @@ -58,7 +58,7 @@ #[test] fn actual_abort_never_supplies_a_displacement_receipt() { - let mut owners = Lifetime::new(); + let mut owners = Lifetime::<2>::new(false); assert!(owners.claim(0)); assert!(owners.publish(0)); assert!(!finish(&mut owners, 0, Retirement::Full, false)); @@ -77,8 +77,8 @@ #[test] fn exhausted_compute_ledger_does_not_poison_render_ledger() { - let mut compute = Lifetime::new(); - let mut render = Lifetime::new(); + let mut compute = Lifetime::<2>::new(false); + let mut render = Lifetime::<2>::new(false); assert!(compute.claim(0)); assert!(compute.publish(0)); finish(&mut compute, 0, Retirement::Aborted, true); @@ -101,7 +101,7 @@ #[test] fn two_aborts_stay_bounded_and_cannot_be_reset_by_preparation_cancel() { - let mut owners = Lifetime::new(); + let mut owners = Lifetime::<2>::new(false); for unit in 0..2 { assert!(owners.claim(unit)); assert!(owners.publish(unit)); @@ -115,7 +115,7 @@ #[test] fn repeated_handshakes_and_storm_limit_preserve_owner_accounting() { - let mut owners = Lifetime::new(); + let mut owners = Lifetime::<2>::new(false); let (mut window, mut count) = (1, 0); for number in 0..logic::WINDOW_LIMIT { assert!(logic::admit(100, &mut window, &mut count)); diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py index fdb5f89dd30d9..3b806174b3cab 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py @@ -44,7 +44,11 @@ def module(text, name): pub const ETIMEDOUT:Error=Error(110);pub const ERESTARTSYS:Error=Error(512); pub const GFP_KERNEL:u32=0; pub type Result = std::result::Result; -pub mod prelude {pub use crate::{Result,KVec,GFP_KERNEL,EINVAL,EAGAIN,E2BIG,EBUSY,EOVERFLOW,EIO,ETIMEDOUT,ERESTARTSYS};} +pub mod prelude {pub use crate::{Result,KVec,KBox,GFP_KERNEL,EINVAL,EAGAIN,E2BIG,EBUSY,EOVERFLOW,EIO,ETIMEDOUT,ERESTARTSYS};} +pub struct KBox(Box); +implKBox{pub fn new(x:T,_:u32)->Result{Ok(Self(Box::new(x)))}} +implstd::ops::Deref for KBox{type Target=T;fn deref(&self)->&T{&self.0}} +implstd::ops::DerefMut for KBox{fn deref_mut(&mut self)->&mut T{&mut self.0}} pub struct KVec(Vec); implKVec{pub fn new()->Self{Self(Vec::new())} pub fn with_capacity(n:usize,_:u32)->Result{Ok(Self(Vec::with_capacity(n)))} pub fn push(&mut self,x:T,_:u32)->Result{self.0.push(x);Ok(())}} implstd::ops::Deref for KVec{type Target=Vec;fn deref(&self)->&Vec{&self.0}} @@ -143,10 +147,18 @@ def module(text, name): pub mod resources { use core::sync::atomic::{AtomicU64,Ordering};use crate::*; pub struct ComputeArena{pub compute:KVec} + // A job's compute units; here one arena. + pub type Prepared=ComputeArena; + impl ComputeArena{ + pub fn compute_units(&self)->Result{Ok(self.compute.len())} + pub fn compute_unit(&self,k:usize)->Result<&super::slots::ComputeSlot>{self.compute.get(k).ok_or(EINVAL)} + } pub struct RenderArena{pub passes:KVec,pub tpc_dirty:[AtomicU64;2],pub tilemap_dirty:[AtomicU64;2]} impl RenderArena{ __SET_DIRTY__ } } pub mod boot {pub fn eproto()->crate::Error{crate::Error(71)}} + // Arenas here have the two base units. + pub mod params {pub fn compute_units()->usize{2}} pub struct Events{pub grow:crate::KVec<[u32;18]>} pub struct Firmware{pub pools:pools::Pools,pub events:Events} pub mod budget{__BUDGET__} diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-stamp-wrap.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-stamp-wrap.py index fc1682600f23f..788090c6c639e 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-stamp-wrap.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-stamp-wrap.py @@ -43,10 +43,11 @@ def function(text, name): struct Graph{compute:CGraph,render:RGraph} struct Vm{slot:u32}struct Job{vm:Vm,resources:Resources} struct Resources{compute:Vec,passes:Vec,pools:Pools} -impl Resources{fn compute(&self)->Result<&Self>{Ok(self)}fn render(&self)->Result<&Self>{Ok(self)}} +impl Resources{fn compute(&self)->Result<&Self>{Ok(self)}fn render(&self)->Result<&Self>{Ok(self)}fn compute_unit(&self,k:usize)->Result<&CUnit>{self.compute.get(k).ok_or(EINVAL)}} struct Claims{slot:[u32;1],pass:[u32;1]} struct Flight{computes:u32,renders:u32,target:u32,ta_target:u32,fs_target:u32,job:Job,claims:Claims,head:u32,ta_head:u32,fs_head:u32,rec:[u32;3],recovery_suspect:bool,recovery_started_ns:u64,doomed:bool} -struct Engine{flights:Vec} +struct Engine{flights:Vec,target:u32} +fn stamp_reached(observed:u32,own:u32,tail:u32)->bool{logic::stamp_reached(observed,own,tail)} impl Engine{fn oldest(&self)->Option<&Flight>{self.flights.first()}} struct CUnit{counter:Object,its:Object,work:Object} struct RUnit{scene:Object} @@ -58,6 +59,7 @@ def function(text, name): fn rmb(){}fn now_ns()->u64{100_000_000} impl Firmware{ fn queue_receipt(&self,_ch:Channel,_state:&Object,_info:&Object,_head:u32,_rec:u32)->Result{Ok(self.queue_passed)} + fn queue_passed(&self,_ch:Channel,_state:&Object,_info:&Object,_f:&Flight)->Result{Ok(self.queue_passed)} __RETIRED__ __RETIRED_SNAPSHOT__ } @@ -66,7 +68,7 @@ def function(text, name): let mut f=Firmware{ graph:Graph{compute:CGraph{fw_stamp:Object::default(),qstate:Object::default(),qinfo:Object::default(),notifier:Object::default()}, render:RGraph{fw_stamp:Default::default(),qstate:Default::default(),qinfo:Default::default(),notifier:Object::default()}}, - engine:Engine{flights:vec![Flight{computes:u32::from(!render),renders:u32::from(render),target,ta_target:target,fs_target:target,job:Job{vm:Vm{slot:2},resources:Resources{compute:vec![CUnit{counter:Object::default(),its:Object::default(),work:Object::default()}],passes:vec![RUnit{scene:Object::default()}],pools:Pools{full:true}}},claims:Claims{slot:[0],pass:[0]},head:1,ta_head:1,fs_head:1,rec:[1;3],recovery_suspect:false,recovery_started_ns:0,doomed:false}]}, + engine:Engine{flights:vec![Flight{computes:u32::from(!render),renders:u32::from(render),target,ta_target:target,fs_target:target,job:Job{vm:Vm{slot:2},resources:Resources{compute:vec![CUnit{counter:Object::default(),its:Object::default(),work:Object::default()}],passes:vec![RUnit{scene:Object::default()}],pools:Pools{full:true}}},claims:Claims{slot:[0],pass:[0]},head:1,ta_head:1,fs_head:1,rec:[1;3],recovery_suspect:false,recovery_started_ns:0,doomed:false}],target}, slots:Slots{scrub:Scrub{compute:vec![CUnit{counter:Object::default(),its:Object::default(),work:Object::default()}],passes:vec![RUnit{scene:Object::default()}]}}, pools:Pools{full:true},queue_passed:true, }; @@ -113,6 +115,25 @@ def function(text, name): } }} } +#[test]fn queued_compute_flight_retires_on_passed_receipts_in_order(){ + for n in [0x7fffff,0x800000,0xffffff,0x1000000,0x1000001]{ + let mut f=firmware(false,n); + let later=firmware(false,n+1).engine.flights.remove(0); + f.engine.flights.push(later);f.engine.target=encode::stamp(n+1); + // The command behind it finished too: the shared stamp and notifier cells name it. + f.graph.compute.fw_stamp.set(0,u64::from(encode::stamp(n+1))); + f.graph.compute.notifier.set(notifier_page::COMPUTE_STAMP,u64::from(encode::stamp(n+1))); + f.graph.compute.notifier.set(notifier_page::COMPUTE_CONTEXT,9); + assert_eq!(f.retired(),Ok(Retirement::Full)); + f.graph.compute.fw_stamp.set(0,u64::from(encode::stamp(n-1))); + assert_eq!(f.retired(),Ok(Retirement::Pending)); + f.graph.compute.fw_stamp.set(0,u64::from(encode::stamp(n))); + assert_eq!(f.retired(),Ok(Retirement::Full)); + f.queue_passed=false;assert_eq!(f.retired(),Ok(Retirement::Pending)); + f.queue_passed=true;f.engine.flights[0].job.resources.compute[0].work.set(compute::DESCRIPTOR_COMPLETED,n-1); + assert_eq!(f.retired(),Ok(Retirement::Pending)); + } +} #[test]fn changing_packed_receipt_remains_pending_and_stable_retry_can_retire(){ let mut f=firmware(false,1);f.engine.flights[0].job.resources.compute[0].its.unstable=true; assert_eq!(f.retired(),Ok(Retirement::Pending)); diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-timestamps.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-timestamps.py index 9a159e849e86b..2abd69cf194e1 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-timestamps.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-timestamps.py @@ -161,7 +161,7 @@ def function(source, marker): pub struct Scrub{pub compute:Vec,pub passes:Vec} pub struct Slots{pub scrub:Scrub}pub struct Firmware{pub slots:Slots} pub struct Resources{pub compute:Vec,pub passes:Vec} - impl Resources{pub fn compute(&self)->Result<&Self>{Ok(self)}pub fn render(&self)->Result<&Self>{Ok(self)}} + impl Resources{pub fn compute(&self)->Result<&Self>{Ok(self)}pub fn render(&self)->Result<&Self>{Ok(self)}pub fn compute_unit(&self,k:usize)->Result<&Compute>{self.compute.get(k).ok_or(super::EINVAL)}} pub struct Job{pub commands:Vec,pub resources:Resources} pub struct Flight{pub job:Job,pub first:usize,pub count:usize,pub renders:u32} pub mod poll{use super::*;impl Firmware{ __RETIRE__ }} From 3969c68efe5db3052358fc53453a8655661314ad Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Thu, 19 Nov 2026 07:15:59 -0500 Subject: [PATCH 3/8] drm/asahi: g16: Share one PB heap between a VM's render allocators Every rendering VM pinned two 256 MiB parameter-buffer heaps, one per render allocator (pass slot), so each rendering process cost ~0.5 GiB of unevictable memory and a few dozen clients (a dEQP-EGL run at six jobs) exhausted a 24 GiB Mac mini. macOS 27.2 on the same machine adds ~24 MiB of wired memory per rendering process. A render is published only while nothing else is in flight and retires with its PB blocks returned to its manager, so the two pass slots' PB managers never hand out heap blocks at the same time. Back both with one heap at the first slot's address. Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/g16/pools.rs | 23 +++++++++++-------- .../drivers/gpu/asahi-m4/test-resources.py | 3 ++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/asahi/g16/pools.rs b/drivers/gpu/drm/asahi/g16/pools.rs index f2ae2f15d4deb..0559f25847778 100644 --- a/drivers/gpu/drm/asahi/g16/pools.rs +++ b/drivers/gpu/drm/asahi/g16/pools.rs @@ -2,7 +2,9 @@ //! Private render UMA/heap backing and device-wide PB-slot leases. //! -//! Each VM keeps two persistent UMA/heap units. With `g16_pb_reuse` (the default), a render +//! Each VM keeps two persistent UMA units and one PB heap they share: a render is published +//! only while no other command is in flight and retires with its blocks returned, so the two +//! pass slots' PB managers never hand out heap blocks at the same time. With `g16_pb_reuse` (the default), a render //! reuses its pass slot's lease, its VM's last successfully retired epoch of the slot, while //! the slot is still bound to it, unchanged (the host writes nothing to it, no InitBM). //! Otherwise it installs a fresh PB manager with full InitBM and never reinitializes exposed @@ -129,7 +131,7 @@ fn write_records( Ok(()) } impl Allocator { - fn new(o: &mut Objects, k: u64) -> Result { + fn new(o: &mut Objects, k: u64, heap: &Arc) -> Result { let base = client::render_base(k); let nc = |va| Some((va, pte::LEAF_NC)); // Native UMA software metadata is ASC-owned and never needs a client GPU alias. @@ -153,13 +155,7 @@ impl Allocator { let state = alloc(o, 0x20000, Class::Shared, nc(client::render_state(k)), true)?; let list = alloc(o, 0x4000, Class::Shared, nc(client::render_list(k)), true)?; let submitted = alloc(o, 0x40, Class::Fw, None, true)?; - let heap = alloc( - o, - client::PB_HEAP_SIZE as usize, - Class::Shared, - nc(client::pb_heap(k)), - false, - )?; + let heap = heap.clone(); let scene_list = alloc( o, buffer::SCENE_LIST_SIZE, @@ -220,9 +216,16 @@ impl Allocator { } impl PoolSet { pub(crate) fn new(o: &mut Objects) -> Result { + let heap = alloc( + o, + client::PB_HEAP_SIZE as usize, + Class::Shared, + Some((client::pb_heap(0), pte::LEAF_NC)), + false, + )?; let mut units = KVec::with_capacity(2, GFP_KERNEL)?; for k in 0..2 { - units.push(Allocator::new(o, k)?, GFP_KERNEL)?; + units.push(Allocator::new(o, k, &heap)?, GFP_KERNEL)?; } Ok(Self { units }) } diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py index 3b806174b3cab..f5b65553a69b0 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py @@ -224,7 +224,8 @@ def module(text, name): aliases.push((0x1000000000,0x1000010000)); // Per-VM context store. aliases.sort();for a in aliases.windows(2){assert!(a[0].1<=a[1].0,"overlap {:x?} {:x?}",a[0],a[1]);} for (k,s) in arena.compute.compute.iter().enumerate(){assert_eq!(s.work.lower_gva.unwrap()&0x3fff,0);assert_eq!(s.work.size,0x2000);assert_eq!(s.pool.size,0x2000000);assert_eq!(s.pool_state.size,0x20000);assert_eq!(s.page_list.size,0x4000);assert_eq!(s.context_store.size,0x40000);assert_eq!(s.pool_state.r32(0).unwrap()as u64 |((s.pool_state.r32(4).unwrap()as u64)<<32),s.pool.gva.unwrap());assert_eq!(pools.allocator_refs_for(k).unwrap().parameter_id,k as u32);} - assert_eq!(aliases.len(),43); + // One PB heap per VM, shared by both render allocators. + assert_eq!(aliases.len(),42); } #[test]fn work_has_kernel_lower_alias_without_application_alias(){ let mut o=Objects::new();let slots=Slots::new().unwrap();let arena=arenas(&mut o); From 98ddc4df56988dc12b13040e172850ecd3fb7022 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Thu, 19 Nov 2026 08:16:09 -0500 Subject: [PATCH 4/8] drm/asahi: g16: Let a closed VM's arenas go; slot leases keep their own A hardware slot's last owner stays bound until a later owner displaces it, because the firmware may still write that owner's descriptor then. The leases held the owner's whole VM state, and with it every arena of the VM: a closed client whose compute unit was the last in some UMA slot kept its 256 MiB PB heap and its compute pools pinned until another VM happened to use that slot. With more compute units, slots used only by clients that queue compute are displaced rarely, and closed clients stayed pinned. Each lease now holds exactly the arena of its unit (compute) or pass slot (render) beside the VM state, and the VM's cache lets go of its arenas when the client closes the VM. What jobs and leases still need, they hold themselves; the rest is freed at close. Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/g16/engine.rs | 3 ++ drivers/gpu/drm/asahi/g16/resources.rs | 35 +++++++++++++++++-- drivers/gpu/drm/asahi/g16/slots.rs | 30 ++++++++++++++++ drivers/gpu/drm/asahi/g16/vm.rs | 5 +++ .../drivers/gpu/asahi-m4/test-resources.py | 2 ++ .../drivers/gpu/asahi-m4/test-vm-registry.py | 10 ++++-- 6 files changed, 80 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g16/engine.rs b/drivers/gpu/drm/asahi/g16/engine.rs index bc9111ca98419..5f0312c1ff70d 100644 --- a/drivers/gpu/drm/asahi/g16/engine.rs +++ b/drivers/gpu/drm/asahi/g16/engine.rs @@ -1174,6 +1174,7 @@ impl Firmware { // Reserve lifetime ownership before any counter or producer becomes reachable. job.vm.vm.mark_binding_published()?; self.slots.hold_vm(&f.claims, &job.vm)?; + self.slots.hold_arenas(&f.claims, &job.resources); self.slots.publish(&f.claims)?; // Even a failed first store may have become visible. From this boundary preserve // every owner and return explicit uncertainty rather than guessing from errno. @@ -1318,6 +1319,7 @@ impl Firmware { .ok_or(EAGAIN)?; let generation = job.vm.generation; self.slots.hold_vm(&f.claims, &job.vm)?; + self.slots.hold_arenas(&f.claims, &job.resources); let s = job.resources.compute_unit(index)?; let g = &self.graph.compute; let before = s.counter.r64(0)?; @@ -1413,6 +1415,7 @@ impl Firmware { // Claim performs only a qualified per-use scrub, never allocator/list reset. self.slots.claim_pass(job.resources.render()?, k)?; self.slots.hold_vm(&f.claims, &job.vm)?; + self.slots.hold_arenas(&f.claims, &job.resources); let (refs, init_buffer, epoch, binding) = self.render_refs(k, job, draw)?; f.pb = Some(epoch); f.pb_binding = binding; diff --git a/drivers/gpu/drm/asahi/g16/resources.rs b/drivers/gpu/drm/asahi/g16/resources.rs index af31073f6ac15..201a387357c7a 100644 --- a/drivers/gpu/drm/asahi/g16/resources.rs +++ b/drivers/gpu/drm/asahi/g16/resources.rs @@ -13,7 +13,7 @@ use super::{ vm::VmState, }; use crate::mmu; -use core::sync::atomic::{AtomicU64, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use kernel::{ new_mutex, prelude::*, @@ -140,6 +140,8 @@ struct Ready { pub(crate) struct Cache { #[pin] ready: Mutex, + /// The VM's client closed: no arena is kept here any more. + closed: AtomicBool, pub(crate) factory: Factory, bind: mmu::VmBind, live: Arc, @@ -154,10 +156,26 @@ impl Cache { let budget = super::budget::Budget::new()?; Arc::pin_init( pin_init!(Self { ready <- new_mutex!(Ready::default(), "g16::resources"), + closed: AtomicBool::new(false), factory, bind, live, budget }), GFP_KERNEL, ) } + /// The VM's client closed. Lets go of the arenas: what jobs and the hardware slots' + /// last-owner leases still need, they hold themselves. + pub(crate) fn close(&self) { + self.closed.store(true, Ordering::Release); + let ready = { + let mut ready = self.ready.lock(); + ( + ready.compute.take(), + ready.compute_ext.take(), + ready.render.take(), + ) + }; + drop(ready); + } + pub(crate) fn prepare(&self, vm: &Arc, commands: &[Command]) -> Result { let computes = commands .iter() @@ -173,7 +191,7 @@ impl Cache { let (compute, compute_ext, render) = { // Never held by publication, completion, recovery or fence signalling. let mut ready = self.ready.lock(); - if vm.status.get() != 0 || vm.vm.is_closed() { + if vm.status.get() != 0 || vm.vm.is_closed() || self.closed.load(Ordering::Acquire) { return Err(EIO); } if compute && ready.compute.is_none() { @@ -268,6 +286,19 @@ impl Prepared { Ok(self.compute()?.compute.len() + self.compute_ext.as_deref().map_or(0, |e| e.compute.len())) } + /// The arena holding compute unit `k`. + pub(crate) fn compute_arena(&self, k: usize) -> Option> { + let base = self.compute.as_ref()?; + if k < base.compute.len() { + Some(base.clone()) + } else { + self.compute_ext.clone() + } + } + /// The render arena. + pub(crate) fn render_arena(&self) -> Option> { + self.render.clone() + } /// Compute unit `k`: the VM's base units, then its extension units. pub(crate) fn compute_unit(&self, k: usize) -> Result<&ComputeSlot> { let base = &self.compute()?.compute; diff --git a/drivers/gpu/drm/asahi/g16/slots.rs b/drivers/gpu/drm/asahi/g16/slots.rs index 79db116659499..1bbfca906de18 100644 --- a/drivers/gpu/drm/asahi/g16/slots.rs +++ b/drivers/gpu/drm/asahi/g16/slots.rs @@ -377,6 +377,10 @@ pub(crate) struct ComputeOwners { owner: [Option>; sizes::COMPUTE_UNITS], pending_retarget: [Option; sizes::COMPUTE_UNITS], pending_owner: [Option>; sizes::COMPUTE_UNITS], + /// The arena holding each owner's unit: the firmware may still write the unit's + /// descriptor when a later owner displaces it, after its VM's client closed. + arena: [Option>; sizes::COMPUTE_UNITS], + pending_arena: [Option>; sizes::COMPUTE_UNITS], } pub(crate) struct Slots { @@ -393,6 +397,9 @@ pub(crate) struct Slots { pass_owner: [Option>; 2], pending_pass_retarget: [Option; 2], pending_pass_owner: [Option>; 2], + /// The render arena of each pass slot's owner (see [`ComputeOwners::arena`]). + pass_arena: [Option>; 2], + pending_pass_arena: [Option>; 2], } impl Slots { pub(crate) fn new() -> Result { @@ -407,6 +414,8 @@ impl Slots { owner: [const { None }; sizes::COMPUTE_UNITS], pending_retarget: [None; sizes::COMPUTE_UNITS], pending_owner: [const { None }; sizes::COMPUTE_UNITS], + arena: [const { None }; sizes::COMPUTE_UNITS], + pending_arena: [const { None }; sizes::COMPUTE_UNITS], }, GFP_KERNEL, )?, @@ -414,6 +423,8 @@ impl Slots { pass_owner: [None, None], pending_pass_retarget: [None, None], pending_pass_owner: [None, None], + pass_arena: [None, None], + pending_pass_arena: [None, None], }) } pub(crate) fn init(&self) -> Result { @@ -599,6 +610,21 @@ impl Slots { Ok(()) } + /// With [`Self::hold_vm`]: the claimed units' arenas, held with their VM leases. The VM's + /// own cache lets go of its arenas when its client closes. + pub(crate) fn hold_arenas(&mut self, f: &Claims, resources: &super::resources::Prepared) { + for &k in &f.slot[..(f.slot_count as usize).min(f.slot.len())] { + if let Some(cell) = self.compute.pending_arena.get_mut(k as usize) { + *cell = resources.compute_arena(k as usize); + } + } + for k in 0..2 { + if f.pass_mask & (1 << k) != 0 { + self.pending_pass_arena[k] = resources.render_arena(); + } + } + } + /// Reserve the actual same-slot publication. Call before producers become visible. pub(crate) fn publish(&mut self, f: &Claims) -> Result { // Validate on a private copy so a failure changes no ownership. @@ -636,10 +662,12 @@ impl Slots { self.pass_lifetime = p; for &k in &f.slot[..f.slot_count as usize] { self.compute.owner[k as usize] = self.compute.pending_owner[k as usize].take(); + self.compute.arena[k as usize] = self.compute.pending_arena[k as usize].take(); } for k in 0..2 { if f.pass_mask & (1 << k) != 0 { self.pass_owner[k] = self.pending_pass_owner[k].take(); + self.pass_arena[k] = self.pending_pass_arena[k].take(); } } self.pass_busy &= !f.pass_mask; @@ -670,12 +698,14 @@ impl Slots { self.compute.lifetime.cancel(k as usize); self.compute.pending_retarget[k as usize] = None; self.compute.pending_owner[k as usize] = None; + self.compute.pending_arena[k as usize] = None; } for k in 0..2 { if f.pass_mask & (1 << k) != 0 { self.pass_lifetime.cancel(k); self.pending_pass_retarget[k] = None; self.pending_pass_owner[k] = None; + self.pending_pass_arena[k] = None; } } self.pass_busy &= !f.pass_mask; diff --git a/drivers/gpu/drm/asahi/g16/vm.rs b/drivers/gpu/drm/asahi/g16/vm.rs index 5bb4f434454cc..ce75ee6e004fc 100644 --- a/drivers/gpu/drm/asahi/g16/vm.rs +++ b/drivers/gpu/drm/asahi/g16/vm.rs @@ -254,6 +254,11 @@ impl Registry { let index = vms.iter().position(|state| state.id == id); index.and_then(|index| vms.remove(index).ok()) }; + // Arenas still reachable by firmware stay with the slot leases and jobs that hold + // them; the rest (and the spare epochs, which hold the VM) go now. + if let Some(cache) = state.as_ref().and_then(|state| state.resources.as_ref()) { + cache.close(); + } drop(state); } diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py index f5b65553a69b0..f40d5319923cf 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py @@ -152,6 +152,8 @@ def module(text, name): impl ComputeArena{ pub fn compute_units(&self)->Result{Ok(self.compute.len())} pub fn compute_unit(&self,k:usize)->Result<&super::slots::ComputeSlot>{self.compute.get(k).ok_or(EINVAL)} + pub fn compute_arena(&self,_k:usize)->Option>{None} + pub fn render_arena(&self)->Option>{None} } pub struct RenderArena{pub passes:KVec,pub tpc_dirty:[AtomicU64;2],pub tilemap_dirty:[AtomicU64;2]} impl RenderArena{ __SET_DIRTY__ } diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py index add96b48bb335..f4c3dbc4a4988 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py @@ -23,7 +23,8 @@ def item(marker): mod module_parameters{pub struct Param;impl Param{pub fn value(&self)->u32{0}}pub static robust_isolation:Param=Param;} mod types{pub struct ScopeGuard(Option);implScopeGuard{pub fn new(f:F)->Self{Self(Some(f))}pub fn dismiss(mut self){self.0.take();}}implDrop for ScopeGuard{fn drop(&mut self){if let Some(f)=self.0.take(){f()}}}} struct Handle;impl Handle{fn is_closed(&self)->bool{false}} -struct VmState{id:u64,vm:Handle,drops:Arc}impl Drop for VmState{fn drop(&mut self){self.drops.fetch_add(1,Ordering::AcqRel);}} +struct Cache{closes:AtomicU32}impl Cache{fn close(&self){self.closes.fetch_add(1,Ordering::AcqRel);}} +struct VmState{id:u64,vm:Handle,drops:Arc,resources:Option>}impl Drop for VmState{fn drop(&mut self){self.drops.fetch_add(1,Ordering::AcqRel);}} __REGISTRY__ struct Reservation<'a>(Option<&'a Registry>,u64); __RESERVATION_DROP__ @@ -46,10 +47,13 @@ def item(marker): } #[test]fn close_releases_registry_owner_without_another_vm_create(){ let r=Registry::new_test();let drops=Arc::new(AtomicU32::new(0)); - let state=Arc::new(VmState{id:1,vm:Handle,drops:drops.clone()});r.vms.lock().push(state.clone(),0).unwrap(); + let cache=Arc::new(Cache{closes:AtomicU32::new(0)}); + let state=Arc::new(VmState{id:1,vm:Handle,drops:drops.clone(),resources:Some(cache.clone())});r.vms.lock().push(state.clone(),0).unwrap(); r.close(1);assert!(r.vms.lock().is_empty());assert_eq!(drops.load(Ordering::Acquire),0); + // The client's close lets the VM's cache drop its arenas while other owners hold the VM. + assert_eq!(cache.closes.load(Ordering::Acquire),1); drop(state);assert_eq!(drops.load(Ordering::Acquire),1); - r.vms.lock().push(Arc::new(VmState{id:2,vm:Handle,drops:drops.clone()}),0).unwrap(); + r.vms.lock().push(Arc::new(VmState{id:2,vm:Handle,drops:drops.clone(),resources:None}),0).unwrap(); r.close(2);assert_eq!(drops.load(Ordering::Acquire),2); } ''' From 857865f4b89d92917368f8045f9e09e68379d592 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Thu, 19 Nov 2026 08:16:16 -0500 Subject: [PATCH 5/8] drm/asahi: g16: Keep never-exposed PB epochs for the next renders Each render submission allocated a fresh PB epoch: three firmware objects, their GPU and firmware mappings and a shared-L2 synchronisation. With PB epoch reuse most renders never expose that epoch, and its deferred teardown unmaps it again. Render submission cost ~390 us of CPU per vkQueueSubmit on the J773g Mac mini (compute ~25 us). Return a retired job's never-exposed epochs to its VM (up to four, under a spinlock: a job may drop in any context) and hand them to the next render submissions before allocating. A spare keeps the budget permit it was created with. A spare holds its VM, which holds the cache, so the cache drops its spares, and takes no more, when the client closes the VM. Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/g16/pools.rs | 4 ++ drivers/gpu/drm/asahi/g16/resources.rs | 68 +++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g16/pools.rs b/drivers/gpu/drm/asahi/g16/pools.rs index 0559f25847778..9b4fbcfb52e5e 100644 --- a/drivers/gpu/drm/asahi/g16/pools.rs +++ b/drivers/gpu/drm/asahi/g16/pools.rs @@ -453,6 +453,10 @@ impl PbEpoch { pub(crate) fn expose(&self) { self.exposed.store(true, Ordering::Release); } + /// Never handed to the firmware: its storage may serve another render of its VM. + pub(crate) fn unexposed(&self) -> bool { + !self.exposed.load(Ordering::Acquire) + } } impl Firmware { diff --git a/drivers/gpu/drm/asahi/g16/resources.rs b/drivers/gpu/drm/asahi/g16/resources.rs index 201a387357c7a..63238572b27ee 100644 --- a/drivers/gpu/drm/asahi/g16/resources.rs +++ b/drivers/gpu/drm/asahi/g16/resources.rs @@ -15,11 +15,16 @@ use super::{ use crate::mmu; use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use kernel::{ - new_mutex, + new_mutex, new_spinlock, prelude::*, - sync::{Arc, Mutex}, + sync::{Arc, Mutex, SpinLock}, }; +/// Never-exposed PB epochs a VM keeps for its next renders. With PB epoch reuse most +/// renders never expose the epoch prepared for them; allocating and mapping a fresh one per +/// render submit costs far more than the render itself. +const SPARE_EPOCHS: usize = 4; + pub(crate) struct ComputeArena(crate::cleanup::Deferred); impl core::ops::Deref for ComputeArena { type Target = ComputeInner; @@ -140,7 +145,11 @@ struct Ready { pub(crate) struct Cache { #[pin] ready: Mutex, - /// The VM's client closed: no arena is kept here any more. + /// Never-exposed epochs returned by retired jobs (at most [`SPARE_EPOCHS`]). Each holds + /// the VM, which holds this cache: [`Self::close`] breaks that cycle. + #[pin] + spares: SpinLock>>, + /// The VM's client closed: no arena or spare is kept here any more. closed: AtomicBool, pub(crate) factory: Factory, bind: mmu::VmBind, @@ -154,17 +163,34 @@ impl Cache { live: Arc, ) -> Result> { let budget = super::budget::Budget::new()?; + let spares = KVec::with_capacity(SPARE_EPOCHS, GFP_KERNEL)?; Arc::pin_init( pin_init!(Self { ready <- new_mutex!(Ready::default(), "g16::resources"), + spares <- new_spinlock!(spares, "g16::spare_epochs"), closed: AtomicBool::new(false), factory, bind, live, budget }), GFP_KERNEL, ) } - /// The VM's client closed. Lets go of the arenas: what jobs and the hardware slots' - /// last-owner leases still need, they hold themselves. + /// Returns a retired job's never-exposed epoch for the next renders. Any context. + fn give_back(&self, epoch: Arc) { + let mut spares = self.spares.lock(); + // Without room, or once the client closed, the epoch drops: its deferred cleanup + // unmaps it as before. + if !self.closed.load(Ordering::Acquire) { + let _ = spares.push_within_capacity(epoch); + } + } + + /// The VM's client closed. Lets go of the arenas and spare epochs: what jobs and the + /// hardware slots' last-owner leases still need, they hold themselves. pub(crate) fn close(&self) { - self.closed.store(true, Ordering::Release); + let spares = { + let mut spares = self.spares.lock(); + self.closed.store(true, Ordering::Release); + core::mem::take(&mut *spares) + }; + drop(spares); let ready = { let mut ready = self.ready.lock(); ( @@ -176,7 +202,11 @@ impl Cache { drop(ready); } - pub(crate) fn prepare(&self, vm: &Arc, commands: &[Command]) -> Result { + pub(crate) fn prepare( + self: &Arc, + vm: &Arc, + commands: &[Command], + ) -> Result { let computes = commands .iter() .filter(|c| matches!(c, Command::Compute(_))) @@ -250,7 +280,13 @@ impl Cache { for command in commands { pb.push( if matches!(command, Command::Render(_)) { - Some(PbEpoch::new(&self.factory, vm, epochs.take()?)?) + // A spare keeps the budget permit it was created with; the reservation + // returns the unused credit when it drops. + let spare = self.spares.lock().pop(); + Some(match spare { + Some(epoch) if epoch.unexposed() => epoch, + _ => PbEpoch::new(&self.factory, vm, epochs.take()?)?, + }) } else { None }, @@ -265,6 +301,7 @@ impl Cache { compute_ext, render, pb, + cache: Some(self.clone()), }) } } @@ -276,6 +313,21 @@ pub(crate) struct Prepared { pub(crate) compute_ext: Option>, pub(crate) render: Option>, pb: KVec>>, + /// The VM cache that unused epochs return to. + cache: Option>, +} + +impl Drop for Prepared { + fn drop(&mut self) { + let Some(cache) = self.cache.take() else { + return; + }; + for epoch in self.pb.iter_mut().filter_map(Option::take) { + if epoch.unexposed() { + cache.give_back(epoch); + } + } + } } impl Prepared { pub(crate) fn compute(&self) -> Result<&ComputeArena> { From 2e96e4177c9325d07ceb4d3eae8cefd237715455 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Thu, 19 Nov 2026 08:19:59 -0500 Subject: [PATCH 6/8] drm/asahi: g16: Size the PB heap at 64 MiB The parameter-buffer heap was the whole 256 MiB window, fully backed and pinned for every rendering VM. The 25G83 firmware copes with a PB that runs out without the host: with only 64 blocks (8 MiB) posted, heavy and tiler-overflow renders still complete, no growth event arrives, and counters in the PB manager move only for the scenes that overran it. Back 64 MiB of the window by default and name only the backed part in the page and block lists. `aurora-g16/tune` sets the heap of VMs created later (pb_heap_mb, 8..256) and caps the blocks a fresh epoch posts (pb_blocks), for measurements. Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/g16/engine.rs | 2 +- drivers/gpu/drm/asahi/g16/params.rs | 22 +++++++++++++- drivers/gpu/drm/asahi/g16/pools.rs | 29 ++++++++++++++----- .../drivers/gpu/asahi-m4/test-resources.py | 10 +++---- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g16/engine.rs b/drivers/gpu/drm/asahi/g16/engine.rs index 5f0312c1ff70d..c431061332cec 100644 --- a/drivers/gpu/drm/asahi/g16/engine.rs +++ b/drivers/gpu/drm/asahi/g16/engine.rs @@ -988,7 +988,7 @@ impl Firmware { } None => { let e = job.resources.pb(job.progress.sent() as usize)?.clone(); - e.prepare(k, arena.pools.heap_gva(k)?)?; + e.prepare(k, arena.pools.heap_gva(k)?, arena.pools.heap_blocks())?; self.pools.stats.fresh += 1; (e, false) } diff --git a/drivers/gpu/drm/asahi/g16/params.rs b/drivers/gpu/drm/asahi/g16/params.rs index 7ec874cdeab77..3532f8b7923c1 100644 --- a/drivers/gpu/drm/asahi/g16/params.rs +++ b/drivers/gpu/drm/asahi/g16/params.rs @@ -39,10 +39,16 @@ pub(crate) mod tune { pub(crate) static COMPUTE_FLIGHTS: AtomicU32 = AtomicU32::new(16); /// Completion repoll interval override, us (0: `g16_resnap_us`). pub(crate) static RESNAP_US: AtomicU32 = AtomicU32::new(0); - const NAMES: [(&str, &AtomicU32, u32, u32); 3] = [ + /// PB heap of a VM's render arena, MiB (VMs created after a change). + pub(crate) static PB_HEAP_MB: AtomicU32 = AtomicU32::new(64); + /// PB blocks posted to a fresh epoch at most (2048: the whole heap). + pub(crate) static PB_BLOCKS: AtomicU32 = AtomicU32::new(2048); + const NAMES: [(&str, &AtomicU32, u32, u32); 5] = [ ("compute_units", &COMPUTE_UNITS, 2, super::hw::sizes::COMPUTE_UNITS as u32), ("compute_flights", &COMPUTE_FLIGHTS, 1, super::hw::sizes::FLIGHTS as u32), ("resnap_us", &RESNAP_US, 0, 1000), + ("pb_heap_mb", &PB_HEAP_MB, 8, 256), + ("pb_blocks", &PB_BLOCKS, 16, 2048), ]; /// Applies one `name=value` line. @@ -72,6 +78,20 @@ pub(crate) mod tune { } } +/// PB heap blocks of a new render arena (128 KiB each). +pub(crate) fn pb_heap_blocks() -> u32 { + let mb = tune::PB_HEAP_MB.load(core::sync::atomic::Ordering::Relaxed).clamp(8, 256); + mb * 8 +} + +/// PB blocks posted to a fresh epoch of a heap of `heap_blocks`. +pub(crate) fn pb_posted_blocks(heap_blocks: u32) -> u32 { + tune::PB_BLOCKS + .load(core::sync::atomic::Ordering::Relaxed) + .clamp(16, 2048) + .min(heap_blocks) +} + /// Compute units of a VM that submits compute back to back. pub(crate) fn compute_units() -> usize { (tune::COMPUTE_UNITS.load(core::sync::atomic::Ordering::Relaxed) as usize) diff --git a/drivers/gpu/drm/asahi/g16/pools.rs b/drivers/gpu/drm/asahi/g16/pools.rs index 9b4fbcfb52e5e..a1f46bc9c8a23 100644 --- a/drivers/gpu/drm/asahi/g16/pools.rs +++ b/drivers/gpu/drm/asahi/g16/pools.rs @@ -80,6 +80,8 @@ pub(crate) struct PbStats { pub(crate) struct PoolSet { units: KVec, + /// Blocks of the shared heap (the PB managers' lists cover the whole 256 MiB window). + heap_blocks: u32, } fn alloc( @@ -216,9 +218,12 @@ impl Allocator { } impl PoolSet { pub(crate) fn new(o: &mut Objects) -> Result { + // The firmware copes with a full PB without the host (its overflow counters move and + // the scene completes), so the heap is sized well below the 256 MiB window. + let heap_blocks = super::params::pb_heap_blocks().min(buffer::MAX_BLOCKS); let heap = alloc( o, - client::PB_HEAP_SIZE as usize, + heap_blocks as usize * buffer::BLOCK_BYTES as usize, Class::Shared, Some((client::pb_heap(0), pte::LEAF_NC)), false, @@ -227,7 +232,7 @@ impl PoolSet { for k in 0..2 { units.push(Allocator::new(o, k, &heap)?, GFP_KERNEL)?; } - Ok(Self { units }) + Ok(Self { units, heap_blocks }) } #[allow(dead_code)] pub(crate) fn allocator_refs_for(&self, k: usize) -> Result { @@ -262,6 +267,9 @@ impl PoolSet { pub(crate) fn heap_gva(&self, k: usize) -> Result { gva(&self.units.get(k).ok_or(EINVAL)?.heap) } + pub(crate) fn heap_blocks(&self) -> u32 { + self.heap_blocks + } } /// Device-wide physical PB-slot leases. Replacement requires a complete normal receipt. @@ -417,18 +425,23 @@ impl PbEpoch { /// Seed only unpublished storage. A retry may choose another physical slot; after /// exposure neither the manager nor any of its firmware-mutated lists is reset. - pub(crate) fn prepare(&self, k: usize, heap: u64) -> Result { - if k >= 2 || self.exposed.load(Ordering::Acquire) { + pub(crate) fn prepare(&self, k: usize, heap: u64, heap_blocks: u32) -> Result { + if k >= 2 || self.exposed.load(Ordering::Acquire) || heap_blocks > buffer::MAX_BLOCKS { return Err(EINVAL); } + // Only the backed part of the heap is named; the rest of the lists stays zero. + let pages = heap_blocks * buffer::PAGES_PER_BLOCK; write_records(&self.pages, buffer::MAX_PAGES as usize, |i| { - buffer::page_record(heap, i as u32).to_le_bytes() + let i = i as u32; + (if i < pages { buffer::page_record(heap, i) } else { 0 }).to_le_bytes() })?; write_records(&self.blocks, buffer::MAX_BLOCKS as usize, |i| { - u64::from(buffer::block_record(heap, i as u32)).to_le_bytes() + let b = i as u32; + u64::from(if b < heap_blocks { buffer::block_record(heap, b) } else { 0 }).to_le_bytes() })?; - self.ring.w32(0, buffer::MAX_BLOCKS)?; - self.ring.w32(4, buffer::MAX_BLOCKS)?; + let posted = super::params::pb_posted_blocks(heap_blocks); + self.ring.w32(0, posted)?; + self.ring.w32(4, posted)?; self.counter.w64(0, 1)?; self.manager.write( 0, diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py index f40d5319923cf..ced68b4114095 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py @@ -160,7 +160,7 @@ def module(text, name): } pub mod boot {pub fn eproto()->crate::Error{crate::Error(71)}} // Arenas here have the two base units. - pub mod params {pub fn compute_units()->usize{2}} + pub mod params {pub fn compute_units()->usize{2}pub fn pb_heap_blocks()->u32{2048}pub fn pb_posted_blocks(heap:u32)->u32{heap}} pub struct Events{pub grow:crate::KVec<[u32;18]>} pub struct Firmware{pub pools:pools::Pools,pub events:Events} pub mod budget{__BUDGET__} @@ -377,10 +377,10 @@ def module(text, name): #[test]fn pb_installations_are_fresh_and_exposed_records_cannot_be_reinitialized(){ let vm=vm(1);let factory=g16::object::Factory;let budget=g16::budget::Budget::new().unwrap(); let a=g16::pools::PbEpoch::new(&factory,&vm,g16::budget::Budget::reserve(&budget,1).unwrap().take().unwrap()).unwrap(); - a.prepare(0,0x70c0000000).unwrap();a.check_assignment(0).unwrap();a.expose(); + a.prepare(0,0x70c0000000,2048).unwrap();a.check_assignment(0).unwrap();a.expose(); a.manager.w64(0xb0,0x123456789abcdef0).unwrap();let saved=a.manager.snapshot(); - assert_eq!(a.prepare(0,0x70c0000000),Err(EINVAL));assert_eq!(saved,a.manager.snapshot()); - let b=g16::pools::PbEpoch::new(&factory,&vm,g16::budget::Budget::reserve(&budget,1).unwrap().take().unwrap()).unwrap();b.prepare(0,0x70c0000000).unwrap(); + assert_eq!(a.prepare(0,0x70c0000000,2048),Err(EINVAL));assert_eq!(saved,a.manager.snapshot()); + let b=g16::pools::PbEpoch::new(&factory,&vm,g16::budget::Budget::reserve(&budget,1).unwrap().take().unwrap()).unwrap();b.prepare(0,0x70c0000000,2048).unwrap(); assert!(!std::ptr::eq(&a.manager.data,&b.manager.data)); assert_eq!(b.manager.class,g16::object::Class::Fw);assert_eq!(b.manager.gva,None); let mut slots=g16::pools::Pools::new(&driver::AsahiDrmDevice(device::Device)); @@ -391,7 +391,7 @@ def module(text, name): #[test]fn pb_reuse_needs_the_same_binding_vm_and_a_completed_render_and_holds_nothing(){ let one=vm(1);let two=vm(2);let factory=g16::object::Factory;let budget=g16::budget::Budget::new().unwrap(); let a=g16::pools::PbEpoch::new(&factory,&one,g16::budget::Budget::reserve(&budget,1).unwrap().take().unwrap()).unwrap(); - a.prepare(0,0x70c0000000).unwrap();a.check_assignment(0).unwrap();a.expose(); + a.prepare(0,0x70c0000000,2048).unwrap();a.check_assignment(0).unwrap();a.expose(); let mut p=g16::pools::Pools::new(&driver::AsahiDrmDevice(device::Device)); let n=p.bind(0,false);p.retire_binding(0,&a).unwrap();p.retire_epoch(true,0,n,true); assert_eq!(sync::Arc::strong_count(&a),2,"only the owner lease"); From eb9d5fe625ea6c46db171a32e022064b73a5d9ce Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Thu, 19 Nov 2026 10:00:11 -0500 Subject: [PATCH 7/8] drm/asahi: g16: Skip the submit-path publication while a render runs Nothing is published behind a render, so a job submitted while one is in flight waits for the poll pass after its retirement anyway. With more jobs admitted per queue, each such submit still ran a full publication pass (ring drain, health and power checks) under the device lock and delayed the retire-to-publish path: gpubench render.light at depth 8 took 389-396 us per submission. Enqueue only; the next poll publishes it (344 us; stock 350, macOS 47). Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/g16/poll.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g16/poll.rs b/drivers/gpu/drm/asahi/g16/poll.rs index ce1a76fffe988..e384e30a02411 100644 --- a/drivers/gpu/drm/asahi/g16/poll.rs +++ b/drivers/gpu/drm/asahi/g16/poll.rs @@ -138,7 +138,11 @@ impl Shared { firmware.engine.shared = Some(self.clone()); } let accepted = firmware.enqueue(job); - if accepted.is_ok() { + // Nothing is published while a render is in flight: the poll pass after its + // retirement publishes this job. Skipping the pass here keeps queued submissions + // from holding the device lock (ring drains, health checks) on that path. + let render_busy = firmware.engine.flights.iter().any(|f| f.renders != 0); + if accepted.is_ok() && !render_busy { if let Err(error) = firmware.publish(&mut done) { firmware.fail_stop(error, &mut done); self.fail_stop(error); From 4b15d49462dfdb60386a84c31e6e780cef4061e9 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Thu, 19 Nov 2026 20:46:29 -0500 Subject: [PATCH 8/8] drm/asahi: g16: Return a firmware-aborted owner's unit to service A firmware recovery marks every unfinished flight suspect, and a suspect the recovery passed without a full receipt is aborted. Its units were quarantined for good. Once every unit a VM may use was quarantined, every compute job of every client was cancelled until a reboot, and rendering the same once both pass slots were. One client's fault with compute in flight could do that; with compute kept in flight, one recovery can abort all six units at once. An aborted owner is now abandoned instead: - Its job fails and its VM's sticky error is latched under the device lock before anything else is published, so no job of that VM is published again and nothing loads the aborted storage. The engine retains the job, which keeps that storage mapped for good. - Its unit returns to service for other storage as a fresh owner. It is not a retired owner, so it proves no displacement: an owner published before it still needs a later full retirement. The recovery passed the aborted command, so nothing runs in its UMA slot when the next owner is published, and a render's next epoch is fresh (the recovery invalidated every PB binding). Quarantine stays for storage that cannot be trusted: a corrupt descriptor identity or a failed retirement. Measured on the J773g (M4, 25G83 firmware, kernel 7.3.0-ARCH-m4drv8), 2026-11-19: four forced firmware recoveries in a row, each aborting compute. Every recovery's owner was abandoned (quarantined 0), compute passed a host-checked dispatch test in a new process after each one with no reboot, and render and compute ran in other processes afterwards. Without this change the same trigger left compute cancelled for every client until a reboot. Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/g16/poll.rs | 15 +++- drivers/gpu/drm/asahi/g16/pools/lifetime.rs | 48 +++++++++++- drivers/gpu/drm/asahi/g16/slots.rs | 37 +++++++-- .../drivers/gpu/asahi-m4/test-recovery.py | 78 ++++++++++++++----- 4 files changed, 147 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/drm/asahi/g16/poll.rs b/drivers/gpu/drm/asahi/g16/poll.rs index e384e30a02411..316d6f69ad2ee 100644 --- a/drivers/gpu/drm/asahi/g16/poll.rs +++ b/drivers/gpu/drm/asahi/g16/poll.rs @@ -609,12 +609,19 @@ impl Firmware { } else { // Only an actual passed abort lacks the full owner receipt. An affected job // with a full normal receipt fails its fence without poisoning healthy storage. + // The aborted storage stays mapped with its retained job and is never used again + // (its VM keeps a sticky error); its units serve other storage, so one client's + // fault does not take compute or rendering away from every other client. if f.renders != 0 { - // The quarantined slot is not picked again, but keep no epoch of it for reuse. + // Keep no epoch of the aborted render for reuse. self.pools.forget(timestamp_claims.pass[0] as usize); } - self.slots.quarantine(&mut f.claims); + // Its VM's sticky error before its units return to service: from here on no job + // of that VM is published (`publish` refuses it), so nothing loads the aborted + // storage again, also not before the job's own completion runs. + f.job.latch_failure(EIO); self.retain(&f.job); + self.slots.abandon(&mut f.claims); self.recovery.aborted_receipts = self.recovery.aborted_receipts.saturating_add(1); } if self.slots.compute_exhausted() { @@ -1005,9 +1012,9 @@ impl Firmware { { writeln!( f, - "{}_owners: fresh {} preparing {} published {} retired {} quarantined {} reusable {}", + "{}_owners: fresh {} preparing {} published {} retired {} quarantined {} reusable {} abandoned {}", name, owners.fresh, owners.preparing, owners.published, owners.retired, - owners.quarantined, owners.reusable + owners.quarantined, owners.reusable, owners.abandoned )?; } for (kind, name) in ["compute", "render"].iter().enumerate() { diff --git a/drivers/gpu/drm/asahi/g16/pools/lifetime.rs b/drivers/gpu/drm/asahi/g16/pools/lifetime.rs index 2f156546ab307..22274177a9cfe 100644 --- a/drivers/gpu/drm/asahi/g16/pools/lifetime.rs +++ b/drivers/gpu/drm/asahi/g16/pools/lifetime.rs @@ -39,6 +39,8 @@ pub(crate) struct Counts { pub(crate) retired: u32, pub(crate) quarantined: u32, pub(crate) reusable: u32, + /// Owners abandoned after a firmware recovery passed them ([`Lifetime::abandon`]). + pub(crate) abandoned: u32, } #[derive(Clone)] @@ -46,6 +48,7 @@ pub(crate) struct Lifetime { units: [Unit; N], sequence: u64, pipelined: bool, + abandoned: u32, } impl Lifetime { pub(crate) const fn new(pipelined: bool) -> Self { @@ -59,6 +62,7 @@ impl Lifetime { units: [unit; N], sequence: 0, pipelined, + abandoned: 0, } } #[allow(dead_code)] @@ -66,7 +70,10 @@ impl Lifetime { self.units[k].generation } pub(crate) fn counts(&self) -> Counts { - let mut counts = Counts::default(); + let mut counts = Counts { + abandoned: self.abandoned, + ..Counts::default() + }; for (k, unit) in self.units.iter().enumerate() { match unit.state { State::Fresh => counts.fresh += 1, @@ -162,16 +169,53 @@ impl Lifetime { self.units[k].state = State::Retired; true } - /// A firmware-aborted owner is retained forever and cannot grant a displacement lease. + /// An owner whose storage can no longer be trusted (corrupt identity or a failed + /// retirement) is retained forever and cannot grant a displacement lease. pub(crate) fn quarantine(&mut self, k: usize) { self.units[k].state = State::Quarantined; } + /// An owner a firmware recovery passed without completing it (an aborted receipt). The + /// firmware no longer runs it, so its index returns to service for other storage, as a + /// fresh owner. Its own storage is never used again: its job failed, which leaves its VM + /// a sticky error (no later publication), and the engine retains the job and so keeps the + /// storage mapped. It is not a retired owner: it never proves that an owner published + /// before it was displaced, which still takes a later full retirement. + pub(crate) fn abandon(&mut self, k: usize) { + let Some(u) = self.units.get_mut(k) else { + return; + }; + if u.state != State::Published { + return; + } + u.state = State::Fresh; + u.previous = State::Fresh; + u.generation = 0; + self.abandoned = self.abandoned.saturating_add(1); + } } #[cfg(test)] mod tests { use super::*; #[test] + fn an_abandoned_owner_returns_to_service_but_displaces_nothing() { + let mut l = Lifetime::<2>::new(false); + assert!(l.claim(0)); + assert!(l.publish(0)); + assert!(l.retire(0)); + assert!(l.claim(1)); + assert!(l.publish(1)); + l.abandon(1); + assert!(!l.exhausted()); + assert!(!l.claim(0)); // The abort is no displacement of owner 0. + assert!(l.claim(1)); // Its index serves other storage. + assert!(l.publish(1)); + assert!(l.retire(1)); + assert!(l.claim(0)); // A later full retirement displaced owner 0. + assert_eq!(l.counts().abandoned, 1); + assert_eq!(l.counts().quarantined, 0); + } + #[test] fn distinct_retired_displacement_is_required() { let mut l = Lifetime::<2>::new(false); assert!(l.claim(0)); diff --git a/drivers/gpu/drm/asahi/g16/slots.rs b/drivers/gpu/drm/asahi/g16/slots.rs index 1bbfca906de18..562c45e4ba945 100644 --- a/drivers/gpu/drm/asahi/g16/slots.rs +++ b/drivers/gpu/drm/asahi/g16/slots.rs @@ -2,11 +2,13 @@ //! Per-VM M4 command storage and device-wide displacement lifetime tracking. //! -//! The compute units share hardware UMA slot 1 and may be in flight together, in publication -//! order; both render units alternate slot 33, one at a time. Firmware retirement and a later -//! distinct retired owner are both required before reuse. +//! Compute unit k owns hardware UMA slot 1 + k; the units may be in flight together, in +//! publication order. Both render units alternate slot 33, one at a time. Firmware retirement +//! and a later distinct retired owner are both required before reuse. //! Private backing and old VM leases survive until a distinct replacement fully retires. -//! Aborted storage remains mapped and quarantined. Pool cursors are never scrubbed. +//! Storage a firmware recovery aborted remains mapped for good (its job is retained) and is +//! never used again; its unit returns to service for other storage (`abandon`). Storage that +//! cannot be trusted is quarantined. Pool cursors are never scrubbed. use core::sync::atomic::Ordering; use kernel::{prelude::*, sync::Arc}; @@ -692,7 +694,32 @@ impl Slots { f.pass_mask = 0; f.ct_count = 0; } - /// Cancels preparation only. Published units require retire() or quarantine(). + /// The firmware's recovery passed the flight of `f` without completing it: its units return + /// to service ([`Lifetime::abandon`]). The flight's own storage leases are dropped; the + /// caller retains its job, which keeps that storage mapped. Each unit's previous owner + /// keeps its lease until a later owner fully retires, as after any publication. + pub(crate) fn abandon(&mut self, f: &mut Claims) { + for &k in &f.slot[..f.slot_count as usize] { + let k = k as usize; + self.compute.lifetime.abandon(k); + self.compute.pending_retarget[k] = None; + self.compute.pending_owner[k] = None; + self.compute.pending_arena[k] = None; + } + for k in 0..2 { + if f.pass_mask & (1 << k) != 0 { + self.pass_lifetime.abandon(k); + self.pending_pass_retarget[k] = None; + self.pending_pass_owner[k] = None; + self.pending_pass_arena[k] = None; + } + } + self.pass_busy &= !f.pass_mask; + f.slot_count = 0; + f.pass_mask = 0; + f.ct_count = 0; + } + /// Cancels preparation only. Published units require retire(), abandon() or quarantine(). pub(crate) fn release(&mut self, f: &mut Claims) { for &k in &f.slot[..f.slot_count as usize] { self.compute.lifetime.cancel(k as usize); diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py index 031d918925c00..d3c3df963d607 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py @@ -29,14 +29,16 @@ use super::logic::{self, Retirement}; // This is the retirement policy used by poll.rs: an affected fence does not - // erase the independently authenticated owner receipt. No synthetic receipt - // is generated here; the hardware poller remains responsible for that proof. + // erase the independently authenticated owner receipt, and an aborted owner is + // abandoned (its job fails, its storage stays retained, its unit serves other + // storage). No synthetic receipt is generated here; the hardware poller remains + // responsible for that proof. fn finish(owners: &mut Lifetime, unit: usize, receipt: Retirement, affected: bool) -> bool { assert!(receipt.complete()); if receipt.owner_retired() { assert!(owners.retire(unit)); } else { - owners.quarantine(unit); + owners.abandon(unit); } receipt.job_failed(affected) } @@ -65,42 +67,45 @@ assert!(owners.claim(1)); assert!(owners.publish(1)); assert!(finish(&mut owners, 1, Retirement::Aborted, false)); + // The abort does not displace owner 0; only a later full retirement does. for _ in 0..10_000 { assert!(!owners.claim(0)); - assert!(!owners.claim(1)); - assert!(owners.exhausted()); + assert!(!owners.exhausted()); } assert_eq!(owners.counts(), Counts { - retired: 1, quarantined: 1, ..Counts::default() + fresh: 1, retired: 1, reusable: 1, abandoned: 1, ..Counts::default() }); + assert!(owners.claim(1)); + assert!(owners.publish(1)); + assert!(!finish(&mut owners, 1, Retirement::Full, false)); + assert!(owners.claim(0)); } #[test] - fn exhausted_compute_ledger_does_not_poison_render_ledger() { + fn an_aborted_compute_owner_exhausts_neither_ledger() { let mut compute = Lifetime::<2>::new(false); let mut render = Lifetime::<2>::new(false); assert!(compute.claim(0)); assert!(compute.publish(0)); finish(&mut compute, 0, Retirement::Aborted, true); - // The remaining fresh owner can run once, but cannot make the aborted - // one reusable. Class exhaustion must remain explicit after this point. - assert!(compute.claim(1)); - assert!(compute.publish(1)); - finish(&mut compute, 1, Retirement::Full, false); - assert!(compute.exhausted()); for number in 0..10_000 { let unit = number % 2; + assert!(compute.claim(unit)); + assert!(compute.publish(unit)); + assert!(!finish(&mut compute, unit, Retirement::Full, false)); + assert!(!compute.exhausted()); assert!(render.claim(unit)); assert!(render.publish(unit)); assert!(!finish(&mut render, unit, Retirement::Full, false)); assert!(!render.exhausted()); } - assert_eq!(compute.counts().quarantined, 1); - assert_eq!(render.counts().quarantined, 0); + assert_eq!(compute.counts().quarantined, 0); + assert_eq!(compute.counts().abandoned, 1); + assert_eq!(render.counts().abandoned, 0); } #[test] - fn two_aborts_stay_bounded_and_cannot_be_reset_by_preparation_cancel() { + fn aborts_cannot_be_reset_by_preparation_cancel_and_leave_both_units_in_service() { let mut owners = Lifetime::<2>::new(false); for unit in 0..2 { assert!(owners.claim(unit)); @@ -108,9 +113,42 @@ finish(&mut owners, unit, Retirement::Aborted, true); owners.cancel(unit); } - assert!(owners.exhausted()); - assert_eq!(owners.counts().quarantined, 2); - assert_eq!(owners.counts().reusable, 0); + assert!(!owners.exhausted()); + assert_eq!(owners.counts().abandoned, 2); + assert_eq!(owners.counts().reusable, 2); + assert!(owners.claim(0)); + owners.cancel(0); + assert!(owners.claim(0)); + } + + #[test] + fn a_recovery_aborting_every_pipelined_unit_leaves_compute_in_service() { + // Six units in flight together (the compute queue). Every third round a recovery + // aborts every flight; compute stays available, with as many units as before. + let mut owners = Lifetime::<6>::new(true); + let mut aborted = 0; + for round in 0..1_000 { + let mut published = Vec::new(); + for unit in 0..6 { + if owners.claim(unit) { + assert!(owners.publish(unit)); + published.push(unit); + } + } + assert!(published.len() >= 5, "round {round}: {published:?}"); + for &unit in &published { + let receipt = if round % 3 == 0 { + aborted += 1; + Retirement::Aborted + } else { + Retirement::Full + }; + finish(&mut owners, unit, receipt, false); + } + assert!(!owners.exhausted()); + } + assert_eq!(owners.counts().quarantined, 0); + assert_eq!(owners.counts().abandoned, aborted); } #[test] @@ -155,7 +193,7 @@ 'result': 'pass', 'tests': tests, 'scope': 'Production recovery decisions and owner ledger; CPU-only tests, no hardware recovery qualification.', - 'known_limit': 'A genuine aborted owner remains quarantined until reboot; two-owner class capacity is bounded.', + 'known_limit': 'Only an owner whose storage cannot be trusted (corrupt identity, failed retirement) remains quarantined until reboot; an aborted owner is abandoned and its unit serves other storage.', 'source': { name: hashlib.sha256((production / name).read_bytes()).hexdigest() for name in ['recovery/logic.rs', 'pools/lifetime.rs', 'poll.rs', 'recovery.rs']