diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index b970691651032..a75e537af2b69 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -167,6 +167,10 @@ module! { // permissions: 0o644, description: "Initial TVB size in blocks", }, + g16_pb_reuse: u32 { + default: 1, + description: "G16 parameter buffer: 1 = reuse a pass slot's last retired epoch for the same VM while the slot is still bound to it (default), 0 = a fresh epoch and InitBM per render", + }, robust_isolation: u32 { default: 0, // permissions: 0o644, diff --git a/drivers/gpu/drm/asahi/g16/command.rs b/drivers/gpu/drm/asahi/g16/command.rs index a2b329789444c..ea6f6882eff6a 100644 --- a/drivers/gpu/drm/asahi/g16/command.rs +++ b/drivers/gpu/drm/asahi/g16/command.rs @@ -112,12 +112,19 @@ impl Order { /// A same-engine command cannot jump a hole in its queue's completed frontier. /// Cross-engine historical dependencies end at the named prefix, even while later /// commands in that older submission remain pending. - pub(crate) fn ready(&self, completed: [u64; 2]) -> bool { - completed[self.engine].checked_add(1) == Some(self.sequence) + pub(crate) fn ready(&self, completed: [u64; 2], published: [u64; 2]) -> bool { + // Same-engine compute order and barriers are kept by the firmware compute queue, + // which finishes each command before it starts the next: its published frontier + // suffices. Renders, and every cross-engine dependency, need the completed one. + let mut frontier = completed; + if self.engine == COMPUTE { + frontier[COMPUTE] = published[COMPUTE].max(completed[COMPUTE]); + } + frontier[self.engine].checked_add(1) == Some(self.sequence) && self .wait_through .iter() - .zip(completed) + .zip(frontier) .all(|(prefix, observed)| prefix.is_none_or(|last| observed >= last)) } } diff --git a/drivers/gpu/drm/asahi/g16/engine.rs b/drivers/gpu/drm/asahi/g16/engine.rs index 5ee33d71432d1..c431061332cec 100644 --- a/drivers/gpu/drm/asahi/g16/engine.rs +++ b/drivers/gpu/drm/asahi/g16/engine.rs @@ -3,10 +3,14 @@ //! Serialized M4 execution over the shared Asahi frontend. //! //! A userspace submission retains all commands, mappings and inputs through its final fence. -//! One command at a time is encoded into one of two retained units, then published to the -//! classic engine queues. Reuse requires a complete wire receipt followed by replacement -//! by the other retired unit in the same UMA slot. TA and FS remain a paired publication -//! with the native explicit TA dependency. This total order satisfies Mesa's command barriers. +//! Each command is encoded into a retained unit, then published to the classic engine queues. +//! A render is published only with nothing else in flight, and nothing is published behind +//! it. Compute commands queue behind in-flight compute commands (up to +//! [`params::compute_flights`]), each in its own unit of UMA slot 1: the firmware runs its +//! compute queue in publication order, each command to its Finalize, so publication order is +//! execution order. A unit is reused only after a later publication fully retired (it +//! displaced the unit). TA and FS remain a paired publication with the native explicit TA +//! dependency. This total order satisfies Mesa's command barriers. //! Firmware failure retains every possibly reachable owner. Completions are signalled only //! after dropping the device lock. @@ -17,10 +21,10 @@ use kernel::{prelude::*, sync::Arc}; use super::channel::Channel; use super::command::{Command, ComputeCommand, RenderCommand}; use super::encode::{self, ComputeRefs, Geometry, RenderRefs}; -use super::fw::channels::SubmitRecord; +use super::fw::channels::{SubmitRecord, RING_SLOTS}; use super::fw::graph::{notifier_page, qstate}; use super::fw::work::{self, BarrierType}; -use super::hw::sizes; +use super::hw::{engine as engine_ids, sizes}; use super::job::{Completions, Job}; use super::slots::Claims; use super::{params, Firmware, Shared}; @@ -54,8 +58,9 @@ pub(crate) fn now_ns() -> u64 { ::ktime_get() as u64 } -/// Counter value carried by a native 25G83 submit-pipe record. -fn counter_ticks() -> u64 { +/// Counter value carried by a native 25G83 submit-pipe record (the 24 MHz counter of the +/// firmware's internal timestamps). +pub(crate) fn counter_ticks() -> u64 { #[cfg(target_arch = "aarch64")] { let value: u64; @@ -180,6 +185,12 @@ pub(crate) struct Flight { pub(crate) recovery_started_ns: u64, /// Publication time, ns. pub(crate) published_ns: u64, + /// Published behind another flight. + pub(crate) behind: bool, + /// The PB epoch this render prefix uses (its own, or a reused one with `g16_pb_reuse`). + pub(crate) pb: Option>, + /// Its pass slot binding (`Pools::bind`). + pub(crate) pb_binding: u64, } impl Flight { @@ -209,6 +220,9 @@ impl Flight { recovery_suspect: false, recovery_started_ns: 0, published_ns: 0, + behind: false, + pb: None, + pb_binding: 0, } } } @@ -291,6 +305,10 @@ pub(crate) struct Stats { /// Why ready jobs stayed unpublished: both flights published, a channel record unconsumed, /// no free slot. pub(crate) hold: [u64; 3], + /// Compute commands published behind in-flight ones. + pub(crate) behind: u64, + /// Most flights in flight at once. + pub(crate) max_flights: u64, } /// Host-side phase timers of publication and polling, on with `debug_flags` bit 49 @@ -408,23 +426,21 @@ pub(crate) struct Engine { /// Progress tracking of the oldest prefix. pub(crate) progress_flight: u64, pub(crate) progress: [Option; 3], - pub(crate) stats: Stats, + /// Boxed with [`Self::prof`]: `Firmware` is built on the probe's stack. + pub(crate) stats: KBox, /// Phase timers (debug_flags bit 49). - pub(crate) prof: Prof, + pub(crate) prof: KBox, /// The `aurora-g16/stats` file. pub(crate) debugfs: Option>>>>, + /// The `aurora-g16/tune` file. + pub(crate) tune_file: Option>>>>, scratch: Option>, } impl Engine { pub(crate) fn new() -> Result { // Wrapped native stamps are safe only while pending references fit within the - // signed half-window. This backend publishes exactly one command at a time, - // retires its full wire/owner receipt, then builds the next command/barrier. - // Do not add submission-ahead without revisiting the dependency lifetime proof. - if sizes::FLIGHTS != 1 { - return Err(EINVAL); - } + // signed half-window: at most FLIGHTS commands are published at once, far inside it. Ok(Self { pending: KVec::new(), flights: KVec::with_capacity(sizes::FLIGHTS, GFP_KERNEL)?, @@ -448,9 +464,10 @@ impl Engine { chan_wait: false, progress_flight: 0, progress: [None; 3], - stats: Stats::default(), - prof: Prof::default(), + stats: KBox::new(Stats::default(), GFP_KERNEL)?, + prof: KBox::new(Prof::default(), GFP_KERNEL)?, debugfs: None, + tune_file: None, // Allocate host encoder buffers before any submitted fence can depend on // the device mutex. Publication must never enter direct reclaim here. scratch: Some(KBox::new(Scratch::new()?, GFP_KERNEL)?), @@ -631,14 +648,19 @@ impl Firmware { // owner, not the next command that happens to occupy the flight list. return Ok(()); } - let f = self.engine.flights.first_mut().ok_or(ENOENT)?; - let matches = match event { - 15 => f.computes != 0 && f.target == target, - 0x70 => f.renders != 0 && f.ta_target == target, - 0x71 => f.renders != 0 && f.fs_target == target, - _ => false, - }; - if !matches || flags & 1 == 0 { + // The named owner: the oldest flight, or with compute queued behind it a later one. + let f = self + .engine + .flights + .iter_mut() + .find(|f| match event { + 15 => f.computes != 0 && f.target == target, + 0x70 => f.renders != 0 && f.ta_target == target, + 0x71 => f.renders != 0 && f.fs_target == target, + _ => false, + }) + .ok_or(ENOENT)?; + if flags & 1 == 0 { return Err(ENOENT); } if flags & 9 == 9 @@ -668,19 +690,24 @@ impl Firmware { _event: u32, fault_asid: Option, ) -> Result { - let Some(f) = self.engine.flights.first_mut() else { - return Ok(()); - }; let reached = super::recovery::logic::stamp_reached; - let unfinished = (f.computes != 0 && !reached(stage[2], f.target, self.engine.target)) - || (f.renders != 0 - && (!reached(stage[0], f.ta_target, self.engine.ta_target) - || !reached(stage[1], f.fs_target, self.engine.fs_target))); - if unfinished { - f.recovery_suspect = true; - f.recovery_started_ns = now_ns(); - if fault_asid.is_some_and(|asid| asid >= 1 && asid == f.job.vm.slot) { - f.doomed = true; + let (target, ta_target, fs_target) = ( + self.engine.target, + self.engine.ta_target, + self.engine.fs_target, + ); + // Every unfinished flight, including compute queued behind the oldest. + for f in self.engine.flights.iter_mut() { + let unfinished = (f.computes != 0 && !reached(stage[2], f.target, target)) + || (f.renders != 0 + && (!reached(stage[0], f.ta_target, ta_target) + || !reached(stage[1], f.fs_target, fs_target))); + if unfinished { + f.recovery_suspect = true; + f.recovery_started_ns = now_ns(); + if fault_asid.is_some_and(|asid| asid >= 1 && asid == f.job.vm.slot) { + f.doomed = true; + } } } Ok(()) @@ -748,8 +775,10 @@ impl Firmware { }; let job = self.engine.pending[index].clone(); // Even a failed job may have an earlier command in flight. An aggregate - // error fence must not race that command's access to user buffers. - if !self.engine.flights.is_empty() { + // error fence must not race that command's access to user buffers: only a + // healthy compute command queues behind in-flight compute; everything else + // (and every failure path below) waits for the engine to drain. + if !self.engine.flights.is_empty() && !self.can_queue_behind(&job) { self.engine.stats.hold[0] += 1; break; } @@ -782,6 +811,20 @@ impl Firmware { } None => None, }; + // A VM whose own units were all quarantined (without the class being exhausted) + // cannot run compute again: cancel its queue like an exhausted class would. + if class_error.is_none() + && matches!(command(&job, job.progress.sent()), Some(Command::Compute(_))) + && job + .resources + .compute_units() + .is_ok_and(|n| self.slots.compute_exhausted_within(n)) + { + self.engine.unpend(&job); + let _ = job.mark_terminal_retired(); + complete(done, &job, Err(ECANCELED)); + continue; + } if let Some((class, error)) = class_error { self.mark_class_unavailable(class, error); self.engine.unpend(&job); @@ -811,6 +854,15 @@ impl Firmware { } continue; } + Err(PublicationError::Unpublished(e)) + if e == EAGAIN && !self.engine.flights.is_empty() => + { + // Waits for an in-flight command: its retirement repolls publication, + // and the flights' own deadlines apply. + let i = if self.engine.chan_wait { 1 } else { 2 }; + self.engine.stats.hold[i] += 1; + break; + } Err(PublicationError::Unpublished(e)) if e == EAGAIN => { let i = if self.engine.chan_wait { 1 } else { 2 }; self.engine.stats.hold[i] += 1; @@ -865,6 +917,52 @@ impl Firmware { result } + /// Whether `job`'s next command may be published behind the flights in flight: it is a + /// healthy compute command whose dependencies are met, every flight is a healthy + /// compute-only prefix of the same VM, and the compute depth is below + /// [`params::compute_flights`]. Renders, another VM's commands and every failure path + /// need an empty engine: under several clients, compute queued behind another VM's + /// lost its writes and stalled the firmware compute queue. + fn can_queue_behind(&self, job: &Arc) -> bool { + let e = &self.engine; + if e.flights.len() >= params::compute_flights() + || e.flights.iter().any(|f| { + f.renders != 0 + || f.doomed + || f.recovery_suspect + || !Arc::ptr_eq(&f.job.vm, &job.vm) + || Self::writes_timestamps(f) + }) + || job.vm.status.get() != 0 + || self.slots.compute_exhausted() + || super::recovery::logic::next_ordinal(e.ordinal).is_none() + || !job + .resources + .compute_units() + .is_ok_and(|units| self.slots.compute_available(units)) + { + return false; + } + let sent = job.progress.sent(); + matches!(command(job, sent), Some(Command::Compute(_))) + && job.check_command_dependencies(sent).is_ok() + } + + /// Whether a flight's commands have user timestamps. The host writes them when the + /// command retires, so a command queued behind it (a query copy, possibly waiting on + /// availability) could run before they exist; it waits for the retirement instead. + fn writes_timestamps(f: &Flight) -> bool { + f.job + .commands + .get(f.first..f.first + f.count) + .is_none_or(|commands| { + commands.iter().any(|command| match command { + Command::Compute(c) => !c.timestamp_outputs.is_empty(), + Command::Render(r) => !r.timestamp_outputs.is_empty(), + }) + }) + } + /// Whether some of `job`'s commands were published: such a job is retained after a /// fail-stop. pub(crate) fn job_partly_published(&self, job: &Job) -> bool { @@ -872,12 +970,30 @@ impl Firmware { } /// Addresses and cumulative allocator state of the selected retained render unit. - fn render_refs(&mut self, k: usize, job: &Job, draw: u64) -> Result<(RenderRefs, bool)> { + fn render_refs( + &mut self, + k: usize, + job: &Job, + draw: u64, + ) -> Result<(RenderRefs, bool, Arc, u64)> { let generation = job.vm.generation; let arena = job.resources.render()?; let mut a = arena.pools.prepare(k, draw)?; - let epoch = job.resources.pb(job.progress.sent() as usize)?; - epoch.prepare(k, arena.pools.heap_gva(k)?)?; + // g16_pb_reuse: reuse this VM's retired epoch of the slot: no reseed, no InitBM. + let (epoch, reused) = match self.pools.reusable(params::pb_reuse(), k, &job.vm) { + Some(e) => { + self.pools.stats.reused += 1; + a.init_buffer = false; + (e, true) + } + None => { + let e = job.resources.pb(job.progress.sent() as usize)?.clone(); + e.prepare(k, arena.pools.heap_gva(k)?, arena.pools.heap_blocks())?; + self.pools.stats.fresh += 1; + (e, false) + } + }; + let binding = self.pools.bind(k, reused); a.pb_mgr = epoch.manager.va; a.mgr_misc = epoch.misc.va; let r = &self.graph.render; @@ -908,6 +1024,8 @@ impl Firmware { usc_base: job.queue.usc_base, }, a.init_buffer, + epoch, + binding, )) } @@ -944,7 +1062,8 @@ impl Firmware { f: &mut Flight, visible: &mut bool, ) -> Result { - if !self.engine.flights.is_empty() { + let behind = !self.engine.flights.is_empty(); + if behind && !self.can_queue_behind(job) { return Err(EAGAIN); } let index = job.progress.sent(); @@ -968,10 +1087,16 @@ impl Firmware { for &channel in channels { let pipe = self.channels.channel(channel); let producer = self.channels.producer(channel); - if pipe.read()? != producer - || pipe.completion()? != producer - || pipe.write()? != producer - { + let busy = if behind { + // Behind in-flight compute: post after its unconsumed records, with room. + pipe.write()? != producer + || producer.wrapping_sub(pipe.read()?) % RING_SLOTS >= sizes::AHEAD_RECORDS + } else { + pipe.read()? != producer + || pipe.completion()? != producer + || pipe.write()? != producer + }; + if busy { self.engine.chan_wait = true; return Err(EAGAIN); } @@ -991,6 +1116,15 @@ impl Firmware { if prefix.c_head >= 256 || prefix.ta_head >= 256 || prefix.fs_head >= 256 { return Err(super::boot::eproto()); } + if behind { + // Room for one more compute entry behind the unconsumed ones. + let read = self.graph.compute.qstate.r32(qstate::READ)?; + if read >= 256 + || prefix.c_head.wrapping_sub(read) % sizes::QUEUE_CAP >= sizes::QUEUE_CAP - 2 + { + return Err(EAGAIN); + } + } f.c_start = prefix.c_head; f.ta_start = prefix.ta_head; f.fs_start = prefix.fs_head; @@ -1033,13 +1167,14 @@ impl Firmware { return Err(EAGAIN); } if render { - job.resources - .pb(index as usize)? + f.pb.as_ref() + .ok_or(EINVAL)? .check_assignment(f.claims.pass[0] as usize)?; } // Reserve lifetime ownership before any counter or producer becomes reachable. job.vm.vm.mark_binding_published()?; self.slots.hold_vm(&f.claims, &job.vm)?; + self.slots.hold_arenas(&f.claims, &job.resources); self.slots.publish(&f.claims)?; // Even a failed first store may have become visible. From this boundary preserve // every owner and return explicit uncertainty rather than guessing from errno. @@ -1047,7 +1182,7 @@ impl Firmware { let publication = (|| -> Result { if render { let k = f.claims.pass[0] as usize; - job.resources.pb(index as usize)?.expose(); + f.pb.as_ref().ok_or(EINVAL)?.expose(); job.resources.render()?.pools.publish(k, f.uma_submitted)?; // Notifier service dereferences u32 counters; the host ordering ledger // remains u64. Eight-byte spacing is retained allocation padding. @@ -1055,7 +1190,8 @@ impl Firmware { self.graph.render.threshold.w32(8, prefix.draw as u32)?; } else { let k = f.claims.slot[0] as usize; - job.resources.compute()?.compute[k] + job.resources + .compute_unit(k)? .counter .w64(0, f.uma_submitted)?; self.graph.compute.threshold.w32(0, prefix.n as u32)?; @@ -1076,6 +1212,8 @@ impl Firmware { )?; } wmb(); + // Every command posts its own record: the firmware does not pick up a WRITE + // advance without one, also while the queue is running. for &channel in channels { let (qinfo, head, event, fresh) = match channel { Channel::Compute => ( @@ -1105,7 +1243,11 @@ impl Firmware { event, new_queue: u8::from(fresh), }; - self.channels.post(channel, &record)?; + if behind { + self.channels.post_ahead(channel, &record)?; + } else { + self.channels.post(channel, &record)?; + } self.session.ring(channel)?; } Ok(()) @@ -1136,9 +1278,14 @@ impl Firmware { } self.engine.events = self.events.compute_events; self.engine.stage_seen = self.events.stage_events; + f.behind = behind; self.stat_publish(f); - self.renew_deadlines(); + if !behind { + // A command behind others keeps the oldest's deadline: it starts only after them. + self.renew_deadlines(); + } job.progress.set_sent(index + 1); + job.command_published(index); let mut flight = Flight::new(job.clone()); core::mem::swap(&mut flight, f); self.engine @@ -1150,6 +1297,8 @@ impl Firmware { core::mem::forget(flight); EIO })?; + let s = &mut self.engine.stats; + s.max_flights = s.max_flights.max(self.engine.flights.len() as u64); publication.map_err(|_| EIO) } @@ -1166,11 +1315,12 @@ impl Firmware { let target = encode::stamp(n); let index = self .slots - .claim_compute(job.resources.compute()?, &mut f.claims, 0)? + .claim_compute(&job.resources, &mut f.claims, 0)? .ok_or(EAGAIN)?; let generation = job.vm.generation; self.slots.hold_vm(&f.claims, &job.vm)?; - let s = &job.resources.compute()?.compute[index]; + self.slots.hold_arenas(&f.claims, &job.resources); + let s = job.resources.compute_unit(index)?; let g = &self.graph.compute; let before = s.counter.r64(0)?; let submitted = before.checked_add(1).ok_or(EOVERFLOW)?; @@ -1195,7 +1345,9 @@ impl Firmware { usc_base: job.queue.usc_base, record_fw: g.notifier.va + notifier_page::COMPUTE_RECORD as u64, binding_generation: generation, - uma_slot: 1, + // Each unit owns its UMA slot: the firmware starts a queued command while the + // one before it still runs, and two owners of one slot then corrupt each other. + uma_slot: engine_ids::COMPUTE_UMA_SLOT + index as u32, // Preserve the already qualified packet bytes while expressing the native // fields separately. This compatibility identifier is not proof of native // global command-ID allocation or uniqueness after compute-index wrap. @@ -1263,7 +1415,10 @@ impl Firmware { // Claim performs only a qualified per-use scrub, never allocator/list reset. self.slots.claim_pass(job.resources.render()?, k)?; self.slots.hold_vm(&f.claims, &job.vm)?; - let (refs, init_buffer) = self.render_refs(k, job, draw)?; + self.slots.hold_arenas(&f.claims, &job.resources); + let (refs, init_buffer, epoch, binding) = self.render_refs(k, job, draw)?; + f.pb = Some(epoch); + f.pb_binding = binding; let p = &job.resources.render()?.passes[k]; let pass = p.refs(); let geometry = Geometry::new(&cmd.cmd, pass.heap_gva); @@ -1322,7 +1477,7 @@ impl Firmware { if init_buffer { pf.initbm = true; let bm = encode::initbm(job.vm.slot, refs.parameter_id, refs.pb_mgr, target); - let epoch = job.resources.pb(job.progress.sent() as usize)?; + let epoch = f.pb.as_ref().ok_or(EINVAL)?; epoch.initbm.write(0, &bm)?; r.ring[0].w64(8 * pf.ta_head as usize, epoch.initbm.va)?; pf.ta_head = (pf.ta_head + 1) % cap; @@ -1356,6 +1511,10 @@ impl Firmware { let s = &mut self.engine.stats; s.pub_ns = now_ns(); f.published_ns = s.pub_ns; + if f.behind { + s.behind += 1; + s.overlaps += 1; + } s.prep_ns += s.pub_ns.wrapping_sub(s.prep_from_ns); if s.idle_from_ns != 0 { s.gap_ns += s.pub_ns.wrapping_sub(s.idle_from_ns); diff --git a/drivers/gpu/drm/asahi/g16/hw.rs b/drivers/gpu/drm/asahi/g16/hw.rs index 353199dea75a1..4aec01a3105c5 100644 --- a/drivers/gpu/drm/asahi/g16/hw.rs +++ b/drivers/gpu/drm/asahi/g16/hw.rs @@ -182,10 +182,17 @@ pub(crate) mod client { pub(crate) const CLIENT_APERTURE: Range = 0x7000000000..0x7100000000; pub(crate) const COMPACT_WINDOW: Range = 0x1000000000..0x9000000000; - // Two persistent compute units, each in its own 64 MiB interval and reused across - // application queues. Every alias is inside Mesa's reserved driver hole. + // Persistent compute units, each in its own 64 MiB interval and reused across + // application queues. Units 0 and 1 keep their original placement below the render + // allocators; further units (up to [`super::sizes::COMPUTE_UNITS`]) live between the + // unused CT window and the PB heap, clear of the dynamic PB page-list range + // 0x7040000000..0x7080000000. Every alias is inside Mesa's reserved driver hole. pub(crate) const fn compute_base(k: u64) -> u64 { - 0x7001000000 + k * 0x04000000 + if k < 2 { + 0x7001000000 + k * 0x04000000 + } else { + 0x7090000000 + (k - 2) * 0x04000000 + } } pub(crate) const fn compute_pool(k: u64) -> u64 { compute_base(k) @@ -280,13 +287,20 @@ pub(crate) mod sizes { pub(crate) const PB_BLOCKS: u32 = 2048; pub(crate) const PB_LIST_RECORDS: u32 = 4096; pub(crate) const PB_GROW_STEP: u32 = 10; - /// The qualified M4 protocol permits only one published command at a time. - pub(crate) const FLIGHTS: usize = 1; + /// Published prefixes at most. A render is published only with nothing else in flight; + /// compute commands may queue behind each other on the one firmware compute queue, each + /// in its own retained unit ([`COMPUTE_UNITS`]). + pub(crate) const FLIGHTS: usize = 16; + /// Retained compute units per VM at most, all in hardware UMA slot 1. A unit is reused + /// only after a later publication retired (it displaced the unit), so `n` units keep up + /// to `n - 1` compute commands in flight. + pub(crate) const COMPUTE_UNITS: usize = 8; + const _: () = assert!(super::client::compute_base(COMPUTE_UNITS as u64) <= super::client::PB_HEAP); /// Inherited diagnostic histogram width; M4 does not fold jobs into a compute horizon. pub(crate) const HORIZON_JOBS: u32 = 1; - /// Two jobs may be admitted per logical queue so exact prior-command dependencies can - /// advance independently of a preceding job's aggregate fence. Firmware stays serialized. - pub(crate) const SCHEDULER_CREDITS: u32 = 2; + /// Jobs admitted to the engine per logical queue: enough to keep a queue's compute + /// commands in flight back to back; exact prior-command dependencies still gate each one. + pub(crate) const SCHEDULER_CREDITS: u32 = 16; /// Unconsumed records per 256-entry channel ring a prefix may be posted behind. pub(crate) const AHEAD_RECORDS: u32 = 64; /// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit). diff --git a/drivers/gpu/drm/asahi/g16/job.rs b/drivers/gpu/drm/asahi/g16/job.rs index 31f06c80ab8bf..d79d5892d31f4 100644 --- a/drivers/gpu/drm/asahi/g16/job.rs +++ b/drivers/gpu/drm/asahi/g16/job.rs @@ -290,7 +290,12 @@ impl Job { .completed .each_ref() .map(|value| value.load(Ordering::Acquire)); - if !order.ready(completed) { + let published = self + .queue + .published + .each_ref() + .map(|value| value.load(Ordering::Acquire)); + if !order.ready(completed, published) { return Err(EAGAIN); } Ok(()) @@ -299,6 +304,14 @@ impl Job { /// Advances exactly one queue-local engine ordinal after a full successful wire receipt. /// Called under the device lock before another command is considered. No allocation, /// fence callback or aggregate-job completion occurs here. + /// Advances the queue's published frontier past command `index` (under the device + /// lock, when its first producer became visible). + pub(crate) fn command_published(&self, index: u32) { + if let Some(order) = self.orders.get(index as usize) { + self.queue.published[order.engine].fetch_max(order.sequence, Ordering::AcqRel); + } + } + pub(crate) fn command_retired(&self, index: u32) -> Result { let order = self.orders.get(index as usize).ok_or(EINVAL)?; let previous = order.sequence.checked_sub(1).ok_or(EINVAL)?; diff --git a/drivers/gpu/drm/asahi/g16/params.rs b/drivers/gpu/drm/asahi/g16/params.rs index d2ef01e26b3a3..3532f8b7923c1 100644 --- a/drivers/gpu/drm/asahi/g16/params.rs +++ b/drivers/gpu/drm/asahi/g16/params.rs @@ -28,6 +28,82 @@ pub(crate) fn compute_deadline_ms() -> u32 { module_parameters::g16_compute_deadline_ms.value() } +/// Runtime tunables, written through `aurora-g16/tune` (debugfs, root only) as `name=value`. +pub(crate) mod tune { + use core::sync::atomic::{AtomicU32, Ordering}; + + /// Compute units of a VM that submits compute back to back + /// (2..=[`super::hw::sizes::COMPUTE_UNITS`]); applies to VMs that grow after the change. + pub(crate) static COMPUTE_UNITS: AtomicU32 = AtomicU32::new(6); + /// Compute commands in flight at most (1 = the serialized engine). + pub(crate) static COMPUTE_FLIGHTS: AtomicU32 = AtomicU32::new(16); + /// Completion repoll interval override, us (0: `g16_resnap_us`). + pub(crate) static RESNAP_US: AtomicU32 = AtomicU32::new(0); + /// PB heap of a VM's render arena, MiB (VMs created after a change). + pub(crate) static PB_HEAP_MB: AtomicU32 = AtomicU32::new(64); + /// PB blocks posted to a fresh epoch at most (2048: the whole heap). + pub(crate) static PB_BLOCKS: AtomicU32 = AtomicU32::new(2048); + const NAMES: [(&str, &AtomicU32, u32, u32); 5] = [ + ("compute_units", &COMPUTE_UNITS, 2, super::hw::sizes::COMPUTE_UNITS as u32), + ("compute_flights", &COMPUTE_FLIGHTS, 1, super::hw::sizes::FLIGHTS as u32), + ("resnap_us", &RESNAP_US, 0, 1000), + ("pb_heap_mb", &PB_HEAP_MB, 8, 256), + ("pb_blocks", &PB_BLOCKS, 16, 2048), + ]; + + /// Applies one `name=value` line. + pub(crate) fn set(line: &str) -> kernel::error::Result { + let (name, value) = line.trim().split_once('=').ok_or(kernel::error::code::EINVAL)?; + let value: u32 = value + .trim() + .parse() + .map_err(|_| kernel::error::code::EINVAL)?; + for (n, cell, lo, hi) in NAMES { + if n == name.trim() { + if value < lo || value > hi { + return Err(kernel::error::code::ERANGE); + } + cell.store(value, Ordering::Relaxed); + return Ok(()); + } + } + Err(kernel::error::code::ENOENT) + } + + pub(crate) fn show(f: &mut impl core::fmt::Write) -> core::fmt::Result { + for (n, cell, _, _) in NAMES { + write!(f, "{}={} ", n, cell.load(Ordering::Relaxed))?; + } + writeln!(f) + } +} + +/// PB heap blocks of a new render arena (128 KiB each). +pub(crate) fn pb_heap_blocks() -> u32 { + let mb = tune::PB_HEAP_MB.load(core::sync::atomic::Ordering::Relaxed).clamp(8, 256); + mb * 8 +} + +/// PB blocks posted to a fresh epoch of a heap of `heap_blocks`. +pub(crate) fn pb_posted_blocks(heap_blocks: u32) -> u32 { + tune::PB_BLOCKS + .load(core::sync::atomic::Ordering::Relaxed) + .clamp(16, 2048) + .min(heap_blocks) +} + +/// Compute units of a VM that submits compute back to back. +pub(crate) fn compute_units() -> usize { + (tune::COMPUTE_UNITS.load(core::sync::atomic::Ordering::Relaxed) as usize) + .clamp(2, hw::sizes::COMPUTE_UNITS) +} + +/// Compute commands in flight at most. +pub(crate) fn compute_flights() -> usize { + (tune::COMPUTE_FLIGHTS.load(core::sync::atomic::Ordering::Relaxed) as usize) + .clamp(1, hw::sizes::FLIGHTS) +} + /// Repoll while a ready prefix waits for record consumption, us. pub(crate) fn repoll_us() -> u32 { module_parameters::g16_repoll_us.value().clamp(5, 100) @@ -35,7 +111,10 @@ pub(crate) fn repoll_us() -> u32 { /// Completion repoll interval, us. pub(crate) fn resnap_us() -> u32 { - module_parameters::g16_resnap_us.value().clamp(25, 1000) + match tune::RESNAP_US.load(core::sync::atomic::Ordering::Relaxed) { + 0 => module_parameters::g16_resnap_us.value().clamp(25, 1000), + us => us.clamp(10, 1000), + } } /// Whether firmware halts are answered with the handshake (else fail-stop). @@ -43,6 +122,11 @@ pub(crate) fn recovery() -> bool { module_parameters::g16_recovery.value() != 0 } +/// Whether a pass slot's retired PB epoch may serve the next render of its VM. +pub(crate) fn pb_reuse() -> bool { + module_parameters::g16_pb_reuse.value() != 0 +} + /// Whether the CDM/VDM stream gates are on. pub(crate) fn stream_gate() -> bool { module_parameters::g16_stream_gate.value() != 0 diff --git a/drivers/gpu/drm/asahi/g16/poll.rs b/drivers/gpu/drm/asahi/g16/poll.rs index 4e9bfaef02e12..316d6f69ad2ee 100644 --- a/drivers/gpu/drm/asahi/g16/poll.rs +++ b/drivers/gpu/drm/asahi/g16/poll.rs @@ -56,6 +56,25 @@ impl HrTimerCallback for Shared { } } +/// The `aurora-g16/tune` debugfs file: current tunables. +fn tune_show(_shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { + super::params::tune::show(f) +} + +/// The `aurora-g16/tune` debugfs file: `name=value` lines. +fn tune_write(_shared: &Arc, reader: &mut kernel::uaccess::UserSliceReader) -> Result { + let mut buf = [0u8; 128]; + let len = reader.len().min(buf.len()); + reader.read_slice(&mut buf[..len])?; + let text = core::str::from_utf8(&buf[..len]).map_err(|_| EINVAL)?; + for line in text.split(|c| c == '\n' || c == ' ' || c == ',') { + if !line.trim().is_empty() { + super::params::tune::set(line)?; + } + } + Ok(()) +} + /// The `aurora-g16/stats` debugfs file contents. fn stats_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { let state = shared.state.lock(); @@ -119,7 +138,11 @@ impl Shared { firmware.engine.shared = Some(self.clone()); } let accepted = firmware.enqueue(job); - if accepted.is_ok() { + // Nothing is published while a render is in flight: the poll pass after its + // retirement publishes this job. Skipping the pass here keeps queued submissions + // from holding the device lock (ring drains, health checks) on that path. + let render_busy = firmware.engine.flights.iter().any(|f| f.renders != 0); + if accepted.is_ok() && !render_busy { if let Err(error) = firmware.publish(&mut done) { firmware.fail_stop(error, &mut done); self.fail_stop(error); @@ -198,10 +221,15 @@ impl Shared { dir.read_callback_file(c_str!("stats"), self.clone(), &stats_show), GFP_KERNEL, )?; + let tune = KBox::pin_init( + dir.read_write_callback_file(c_str!("tune"), self.clone(), &tune_show, &tune_write), + GFP_KERNEL, + )?; let mut state = self.state.lock(); let firmware = state.as_deref_mut().ok_or(ENODEV)?; firmware.engine.shared = Some(self.clone()); firmware.engine.debugfs = Some(file); + firmware.engine.tune_file = Some(tune); Ok(()) } } @@ -212,7 +240,6 @@ pub(crate) fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { } /// Cursor `c` of a ring of `m` entries passed end `h` of entries from `b`, within tail `t`. -#[allow(dead_code)] pub(crate) fn ring_passed(c: u32, b: u32, h: u32, t: u32, m: u32) -> bool { let d = |x: u32| x.wrapping_add(m).wrapping_sub(b) % m; let (dc, dh, dt) = (d(c), d(h), d(t)); @@ -341,6 +368,40 @@ impl Firmware { Ok(true) } + /// In-order receipt of compute flight `f` with later flights queued behind it: the NC + /// READ cursor (and with uncached queue info its consumers) passed its ring entries, + /// and the submit pipe's read and completion cursors passed its record. + fn queue_passed( + &self, + channel: Channel, + state: &super::object::FwObject, + info: &super::object::FwObject, + f: &Flight, + ) -> Result { + let cap = super::hw::sizes::QUEUE_CAP; + let tail = self.engine.head; + if f.head >= cap || f.c_start >= cap || tail >= cap { + return Err(EINVAL); + } + if !ring_passed(state.r32(qstate::READ)?, f.c_start, f.head, tail, cap) { + return Ok(false); + } + if !super::hw::tuning::FW_CACHED { + for offset in [0x20, 0x24, 0x28] { + if !ring_passed(info.r32(offset)?, f.c_start, f.head, tail, cap) { + return Ok(false); + } + } + } + let slots = super::fw::channels::RING_SLOTS; + let record = f.rec[2]; + let first = (record + slots - 1) % slots; + let producer = self.channels.producer(channel); + let pipe = self.channels.channel(channel); + Ok(ring_passed(pipe.read()?, first, record, producer, slots) + && ring_passed(pipe.completion()?, first, record, producer, slots)) + } + /// A full receipt includes stage and notifier stamps, every physical consumer, /// the private allocator completion and the internal timestamp pair. An aborted /// receipt omits only the writes firmware does not make for an aborted command, @@ -359,7 +420,33 @@ impl Firmware { }; rmb(); let mut full = true; - if f.computes != 0 { + // Later compute flights queued behind this one: the queue, pipe and stamp cells + // already moved on, so their receipt is "passed", in order; the newest flight keeps + // the exact receipt. + let behind = self.engine.flights.len() > 1; + if f.computes != 0 && behind { + let g = &self.graph.compute; + let e = &self.engine; + if !stamp_reached(g.fw_stamp.r32(0)?, f.target, e.target) + || !self.queue_passed(Channel::Compute, &g.qstate, &g.qinfo, f)? + { + return Ok(Retirement::Pending); + } + let k = f.claims.slot[0] as usize; + let unit = f.job.resources.compute_unit(k)?; + let submitted = unit.counter.r64(0)?; + let start = unit.its.r64(0)?; + let end = unit.its.r64(8)?; + let stamp = g.notifier.r32(notifier_page::COMPUTE_STAMP)?; + full &= stamp_reached(stamp, f.target, e.target) + && (stamp != f.target + || g.notifier.r32(notifier_page::COMPUTE_CONTEXT)? == f.job.vm.slot) + && submitted != 0 + && unit.work.r64(compute::DESCRIPTOR_COMPLETED)? == submitted + && unit.work.r64(compute::DESCRIPTOR_COUNTER)? == unit.counter.va + && start != 0 + && end > start; + } else if f.computes != 0 { let g = &self.graph.compute; if g.fw_stamp.r32(0)? != f.target || !self.queue_receipt(Channel::Compute, &g.qstate, &g.qinfo, f.head, f.rec[2])? @@ -367,7 +454,7 @@ impl Firmware { return Ok(Retirement::Pending); } let k = f.claims.slot[0] as usize; - let unit = f.job.resources.compute()?.compute.get(k).ok_or(EINVAL)?; + let unit = f.job.resources.compute_unit(k)?; let submitted = unit.counter.r64(0)?; let start = unit.its.r64(0)?; let end = unit.its.r64(8)?; @@ -494,9 +581,13 @@ impl Firmware { if f.renders != 0 { self.recovery.last_retired[1] = Some(f.ta_target); self.recovery.last_retired[2] = Some(f.fs_target); - let committed = f.job.resources.pb(f.first).and_then(|epoch| { + let committed = f.pb.as_ref().ok_or(EINVAL).and_then(|epoch| { + let k = timestamp_claims.pass[0] as usize; + self.pools.retire_binding(k, epoch)?; + let enabled = params::pb_reuse(); self.pools - .retire_binding(timestamp_claims.pass[0] as usize, epoch) + .retire_epoch(enabled, k, f.pb_binding, result.is_ok()); + Ok(()) }); if let Err(error) = committed { self.retain(&f.job); @@ -518,8 +609,19 @@ impl Firmware { } else { // Only an actual passed abort lacks the full owner receipt. An affected job // with a full normal receipt fails its fence without poisoning healthy storage. - self.slots.quarantine(&mut f.claims); + // The aborted storage stays mapped with its retained job and is never used again + // (its VM keeps a sticky error); its units serve other storage, so one client's + // fault does not take compute or rendering away from every other client. + if f.renders != 0 { + // Keep no epoch of the aborted render for reuse. + self.pools.forget(timestamp_claims.pass[0] as usize); + } + // Its VM's sticky error before its units return to service: from here on no job + // of that VM is published (`publish` refuses it), so nothing loads the aborted + // storage again, also not before the job's own completion runs. + f.job.latch_failure(EIO); self.retain(&f.job); + self.slots.abandon(&mut f.claims); self.recovery.aborted_receipts = self.recovery.aborted_receipts.saturating_add(1); } if self.slots.compute_exhausted() { @@ -556,6 +658,15 @@ impl Firmware { let job = f.job.clone(); let p = &job.progress; p.next.store(p.next() + f.count as u32, Ordering::Relaxed); + if result.is_err() { + // Its queue's later commands queued behind it fail with it, as unpublished ones + // are cancelled: they never report a successful receipt after this hole. + for later in self.engine.flights.iter_mut() { + if Arc::ptr_eq(&later.job.queue, &job.queue) { + later.doomed = true; + } + } + } if let Err(error) = result { if p.is_pending() { // Drop its unpublished commands. @@ -654,13 +765,7 @@ impl Firmware { match command { super::command::Command::Compute(command) => { let index = *slots.get(compute).ok_or(EINVAL)? as usize; - let unit = flight - .job - .resources - .compute()? - .compute - .get(index) - .ok_or(EINVAL)?; + let unit = flight.job.resources.compute_unit(index)?; let ticks = [unit.its.r64(0)?, unit.its.r64(8)?, 0, 0]; command .timestamp_outputs @@ -750,10 +855,9 @@ impl Firmware { ); } for k in 0..(f.computes as usize).min(f.claims.slot_count as usize) { - let Some(arena) = f.job.resources.compute.as_ref() else { + let Ok(c) = f.job.resources.compute_unit(f.claims.slot[k] as usize) else { return; }; - let c = &arena.compute[f.claims.slot[k] as usize]; stage(&c.its, 0, &mut s.cdm_ticks, crate::usage::Engine::Compute); } if hi > lo { @@ -908,9 +1012,9 @@ impl Firmware { { writeln!( f, - "{}_owners: fresh {} preparing {} published {} retired {} quarantined {} reusable {}", + "{}_owners: fresh {} preparing {} published {} retired {} quarantined {} reusable {} abandoned {}", name, owners.fresh, owners.preparing, owners.published, owners.retired, - owners.quarantined, owners.reusable + owners.quarantined, owners.reusable, owners.abandoned )?; } for (kind, name) in ["compute", "render"].iter().enumerate() { @@ -990,6 +1094,20 @@ impl Firmware { s.span_ticks / 24 )?; // The phase timers of debug_flags bit 49, and the scrub flush waits. + { + let s = &self.engine.stats; + writeln!( + f, + "pipeline: behind {} max_flights {} compute_published {} hold {}/{}/{}", + s.behind, + s.max_flights, + self.slots.compute_published(), + s.hold[0], + s.hold[1], + s.hold[2] + )?; + super::params::tune::show(f)?; + } let p = &self.engine.prof; writeln!( f, diff --git a/drivers/gpu/drm/asahi/g16/pools.rs b/drivers/gpu/drm/asahi/g16/pools.rs index d4a85cc6f3d36..a1f46bc9c8a23 100644 --- a/drivers/gpu/drm/asahi/g16/pools.rs +++ b/drivers/gpu/drm/asahi/g16/pools.rs @@ -2,9 +2,14 @@ //! Private render UMA/heap backing and device-wide PB-slot leases. //! -//! Each VM keeps two persistent UMA/heap units. Each render command installs a fresh -//! PB manager with full InitBM and never reinitializes exposed cursors. The previous -//! PB-slot lease survives until the replacement's complete normal retirement. +//! Each VM keeps two persistent UMA units and one PB heap they share: a render is published +//! only while no other command is in flight and retires with its blocks returned, so the two +//! pass slots' PB managers never hand out heap blocks at the same time. With `g16_pb_reuse` (the default), a render +//! reuses its pass slot's lease, its VM's last successfully retired epoch of the slot, while +//! the slot is still bound to it, unchanged (the host writes nothing to it, no InitBM). +//! Otherwise it installs a fresh PB manager with full InitBM and never reinitializes exposed +//! cursors. The previous PB-slot lease survives until the replacement's complete normal +//! retirement. #[path = "pools/lifetime.rs"] pub(crate) mod lifetime; @@ -67,10 +72,16 @@ pub(crate) struct PbStats { #[allow(dead_code)] pub(crate) refused: u64, pub(crate) bad: u64, + /// Pass slot bindings that reused a retired epoch / started a fresh one (a deferred + /// render counts again when it is retried). + pub(crate) reused: u64, + pub(crate) fresh: u64, } pub(crate) struct PoolSet { units: KVec, + /// Blocks of the shared heap (the PB managers' lists cover the whole 256 MiB window). + heap_blocks: u32, } fn alloc( @@ -122,7 +133,7 @@ fn write_records( Ok(()) } impl Allocator { - fn new(o: &mut Objects, k: u64) -> Result { + fn new(o: &mut Objects, k: u64, heap: &Arc) -> Result { let base = client::render_base(k); let nc = |va| Some((va, pte::LEAF_NC)); // Native UMA software metadata is ASC-owned and never needs a client GPU alias. @@ -146,13 +157,7 @@ impl Allocator { let state = alloc(o, 0x20000, Class::Shared, nc(client::render_state(k)), true)?; let list = alloc(o, 0x4000, Class::Shared, nc(client::render_list(k)), true)?; let submitted = alloc(o, 0x40, Class::Fw, None, true)?; - let heap = alloc( - o, - client::PB_HEAP_SIZE as usize, - Class::Shared, - nc(client::pb_heap(k)), - false, - )?; + let heap = heap.clone(); let scene_list = alloc( o, buffer::SCENE_LIST_SIZE, @@ -213,11 +218,21 @@ impl Allocator { } impl PoolSet { pub(crate) fn new(o: &mut Objects) -> Result { + // The firmware copes with a full PB without the host (its overflow counters move and + // the scene completes), so the heap is sized well below the 256 MiB window. + let heap_blocks = super::params::pb_heap_blocks().min(buffer::MAX_BLOCKS); + let heap = alloc( + o, + heap_blocks as usize * buffer::BLOCK_BYTES as usize, + Class::Shared, + Some((client::pb_heap(0), pte::LEAF_NC)), + false, + )?; let mut units = KVec::with_capacity(2, GFP_KERNEL)?; for k in 0..2 { - units.push(Allocator::new(o, k)?, GFP_KERNEL)?; + units.push(Allocator::new(o, k, &heap)?, GFP_KERNEL)?; } - Ok(Self { units }) + Ok(Self { units, heap_blocks }) } #[allow(dead_code)] pub(crate) fn allocator_refs_for(&self, k: usize) -> Result { @@ -252,6 +267,9 @@ impl PoolSet { pub(crate) fn heap_gva(&self, k: usize) -> Result { gva(&self.units.get(k).ok_or(EINVAL)?.heap) } + pub(crate) fn heap_blocks(&self) -> u32 { + self.heap_blocks + } } /// Device-wide physical PB-slot leases. Replacement requires a complete normal receipt. @@ -259,6 +277,11 @@ pub(crate) struct Pools { pub(crate) dev: ARef, pub(crate) stats: PbStats, owners: [Option>; 2], + /// Per pass slot, a binding count, bumped by each fresh epoch (its InitBM rebinds the + /// slot) and by a firmware recovery. It identifies a binding without holding the epoch. + binds: [u64; 2], + /// Per pass slot, with `g16_pb_reuse`: `owners[k]` may serve the next render of its VM. + keep: [bool; 2], } impl Pools { pub(crate) fn new(dev: &AsahiDrmDevice) -> Self { @@ -266,6 +289,66 @@ impl Pools { dev: dev.as_ref().as_ref().into(), stats: PbStats::default(), owners: [None, None], + binds: [0, 0], + keep: [false, false], + } + } + /// With reuse `enabled` (`g16_pb_reuse`): slot `k`'s owner epoch, if its render retired + /// normally while the slot was still bound to it, it belongs to `vm` and was exposed (its + /// InitBM ran). InitBM binds the device-wide slot to one epoch and VM; a fresh epoch handed + /// to the slot rebinds it, so the owner is then stale and the render starts fresh. Adds no + /// reference beyond the owner lease itself. + pub(crate) fn reusable( + &self, + enabled: bool, + k: usize, + vm: &Arc, + ) -> Option> { + if !enabled || !*self.keep.get(k)? { + return None; + } + let owner = self.owners.get(k)?.as_ref()?; + (Arc::ptr_eq(&owner._vm, vm) && owner.exposed.load(Ordering::Acquire)) + .then(|| owner.clone()) + } + /// Slot `k` is handed to a render's epoch, `reused` or fresh (with InitBM). Returns the + /// binding the render retires with. + pub(crate) fn bind(&mut self, k: usize, reused: bool) -> u64 { + let (Some(binds), Some(keep)) = (self.binds.get_mut(k), self.keep.get_mut(k)) else { + return 0; + }; + if !reused { + *binds = binds.wrapping_add(1); + *keep = false; + } + *binds + } + /// A render on slot `k` with `binding` retired, after its `retire_binding`. If no other + /// epoch was bound and no recovery started since, the next render of the same VM may reuse + /// its epoch with reuse `enabled` and a `completed` render; after a failed one it starts + /// fresh. + /// Renders are bound, published and retired one at a time (`sizes::FLIGHTS` is 1, enforced + /// in `Engine::new`), so `keep[k]` always refers to the epoch the slot is bound to: + /// `owners[k]`. + pub(crate) fn retire_epoch(&mut self, enabled: bool, k: usize, binding: u64, completed: bool) { + if self.binds.get(k) == Some(&binding) { + if let Some(keep) = self.keep.get_mut(k) { + *keep = enabled && completed; + } + } + } + /// Reuse nothing on slot `k` (its render was aborted). + pub(crate) fn forget(&mut self, k: usize) { + if let Some(keep) = self.keep.get_mut(k) { + *keep = false; + } + } + /// A firmware recovery started: no epoch bound so far is reused, also not by a render that + /// retires after it (its binding no longer matches); each slot's next render starts fresh. + pub(crate) fn invalidate(&mut self) { + for (binds, keep) in self.binds.iter_mut().zip(self.keep.iter_mut()) { + *binds = binds.wrapping_add(1); + *keep = false; } } pub(crate) fn retire_binding(&mut self, k: usize, epoch: &Arc) -> Result { @@ -275,16 +358,19 @@ impl Pools { pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { writeln!( f, - "pb: M4 private allocators blocks {} requests {} bad {}", + "pb: M4 private allocators blocks {} requests {} bad {} reused {} fresh {}", buffer::MAX_BLOCKS, self.stats.requests, - self.stats.bad + self.stats.bad, + self.stats.reused, + self.stats.fresh ) } } -/// One never-reinitialized PB manager installation. Even a same-VM command gets a fresh -/// manager and full InitBM; no host reset of exposed firmware cursors is required. +/// One never-reinitialized PB manager installation. Unless `g16_pb_reuse` hands a same-VM +/// command its retired epoch, every command gets a fresh manager and full InitBM; no host +/// reset of exposed firmware cursors is required. pub(crate) struct PbEpoch(crate::cleanup::Deferred); impl core::ops::Deref for PbEpoch { type Target = PbEpochInner; @@ -339,18 +425,23 @@ impl PbEpoch { /// Seed only unpublished storage. A retry may choose another physical slot; after /// exposure neither the manager nor any of its firmware-mutated lists is reset. - pub(crate) fn prepare(&self, k: usize, heap: u64) -> Result { - if k >= 2 || self.exposed.load(Ordering::Acquire) { + pub(crate) fn prepare(&self, k: usize, heap: u64, heap_blocks: u32) -> Result { + if k >= 2 || self.exposed.load(Ordering::Acquire) || heap_blocks > buffer::MAX_BLOCKS { return Err(EINVAL); } + // Only the backed part of the heap is named; the rest of the lists stays zero. + let pages = heap_blocks * buffer::PAGES_PER_BLOCK; write_records(&self.pages, buffer::MAX_PAGES as usize, |i| { - buffer::page_record(heap, i as u32).to_le_bytes() + let i = i as u32; + (if i < pages { buffer::page_record(heap, i) } else { 0 }).to_le_bytes() })?; write_records(&self.blocks, buffer::MAX_BLOCKS as usize, |i| { - u64::from(buffer::block_record(heap, i as u32)).to_le_bytes() + let b = i as u32; + u64::from(if b < heap_blocks { buffer::block_record(heap, b) } else { 0 }).to_le_bytes() })?; - self.ring.w32(0, buffer::MAX_BLOCKS)?; - self.ring.w32(4, buffer::MAX_BLOCKS)?; + let posted = super::params::pb_posted_blocks(heap_blocks); + self.ring.w32(0, posted)?; + self.ring.w32(4, posted)?; self.counter.w64(0, 1)?; self.manager.write( 0, @@ -375,6 +466,10 @@ impl PbEpoch { pub(crate) fn expose(&self) { self.exposed.store(true, Ordering::Release); } + /// Never handed to the firmware: its storage may serve another render of its VM. + pub(crate) fn unexposed(&self) -> bool { + !self.exposed.load(Ordering::Acquire) + } } impl Firmware { diff --git a/drivers/gpu/drm/asahi/g16/pools/lifetime.rs b/drivers/gpu/drm/asahi/g16/pools/lifetime.rs index baf623b2722d7..22274177a9cfe 100644 --- a/drivers/gpu/drm/asahi/g16/pools/lifetime.rs +++ b/drivers/gpu/drm/asahi/g16/pools/lifetime.rs @@ -1,7 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Bounded two-owner displacement ledger. A completion alone never makes an M4 UMA owner -//! reusable: a distinct, fully retired owner must replace it in the same hardware slot. +//! Bounded displacement ledger of the retained owners of one hardware UMA slot. A completion +//! alone never makes an M4 UMA owner reusable: a later, fully retired publication must have +//! replaced it in the same hardware slot. +//! +//! The firmware runs one queue's commands strictly in order, each to its Finalize, so a +//! publication that retired started after every earlier one finished and displaced it. Owners +//! therefore retire in publication order and an owner is reusable once any owner published +//! after it retired. With `pipelined`, owners may be published while earlier ones are still +//! in flight (the compute queue); without it, publication stays serialized against every +//! owner (the render pass slots). +//! //! Core-only so the transition system is executable in host tests. #[derive(Copy, Clone, Debug, Eq, PartialEq)] @@ -30,16 +39,19 @@ pub(crate) struct Counts { pub(crate) retired: u32, pub(crate) quarantined: u32, pub(crate) reusable: u32, + /// Owners abandoned after a firmware recovery passed them ([`Lifetime::abandon`]). + pub(crate) abandoned: u32, } #[derive(Clone)] -pub(crate) struct Lifetime { - units: [Unit; 2], +pub(crate) struct Lifetime { + units: [Unit; N], sequence: u64, - last_owner: Option, + pipelined: bool, + abandoned: u32, } -impl Lifetime { - pub(crate) const fn new() -> Self { +impl Lifetime { + pub(crate) const fn new(pipelined: bool) -> Self { let unit = Unit { state: State::Fresh, previous: State::Fresh, @@ -47,9 +59,10 @@ impl Lifetime { publication: 0, }; Self { - units: [unit; 2], + units: [unit; N], sequence: 0, - last_owner: None, + pipelined, + abandoned: 0, } } #[allow(dead_code)] @@ -57,7 +70,10 @@ impl Lifetime { self.units[k].generation } pub(crate) fn counts(&self) -> Counts { - let mut counts = Counts::default(); + let mut counts = Counts { + abandoned: self.abandoned, + ..Counts::default() + }; for (k, unit) in self.units.iter().enumerate() { match unit.state { State::Fresh => counts.fresh += 1, @@ -70,16 +86,23 @@ impl Lifetime { } counts } + /// Published owners. + pub(crate) fn published(&self) -> u32 { + self.units + .iter() + .filter(|u| u.state == State::Published) + .count() as u32 + } pub(crate) fn available(&self, k: usize) -> bool { let Some(u) = self.units.get(k) else { return false; }; match u.state { State::Fresh => true, - State::Retired => self.last_owner.is_some_and(|b| { - b != k - && self.units[b].state == State::Retired - && self.units[b].publication > u.publication + // A later publication retired: it started only after this owner finished, and + // its load displaced this owner from the hardware slot. + State::Retired => self.units.iter().enumerate().any(|(b, other)| { + b != k && other.state == State::Retired && other.publication > u.publication }), _ => false, } @@ -87,7 +110,11 @@ impl Lifetime { /// No amount of firmware progress can make a unit claimable. The caller must fail /// pending work rather than wait forever after abort/quarantine or sequence exhaustion. pub(crate) fn exhausted(&self) -> bool { - (0..2).all(|k| !self.available(k)) + self.exhausted_within(N) + } + /// [`Self::exhausted`] when only owners `0..n` exist. + pub(crate) fn exhausted_within(&self, n: usize) -> bool { + (0..n.min(N)).all(|k| !self.available(k)) && !self .units .iter() @@ -113,39 +140,84 @@ impl Lifetime { } } pub(crate) fn publish(&mut self, k: usize) -> bool { - if self.units[k].state != State::Preparing || self.sequence == u64::MAX { + if k >= N || self.units[k].state != State::Preparing || self.sequence == u64::MAX { return false; } - // Publication must be serialized against every owner of this hardware slot. - if self.units.iter().any(|u| u.state == State::Published) { + // Without pipelining, publication is serialized against every owner of the slot. + if !self.pipelined && self.units.iter().any(|u| u.state == State::Published) { return false; } self.sequence += 1; self.units[k].publication = self.sequence; self.units[k].state = State::Published; - self.last_owner = Some(k); true } - /// Called only after the caller has established complete, exact wire retirement. + /// Called only after the caller has established complete, exact wire retirement. Owners + /// retire in publication order: `k` must be the oldest published owner. pub(crate) fn retire(&mut self, k: usize) -> bool { - if self.units[k].state != State::Published || self.last_owner != Some(k) { + if k >= N || self.units[k].state != State::Published { + return false; + } + let publication = self.units[k].publication; + if self + .units + .iter() + .any(|u| u.state == State::Published && u.publication < publication) + { return false; } self.units[k].state = State::Retired; true } - /// A firmware-aborted owner is retained forever and cannot grant a displacement lease. + /// An owner whose storage can no longer be trusted (corrupt identity or a failed + /// retirement) is retained forever and cannot grant a displacement lease. pub(crate) fn quarantine(&mut self, k: usize) { self.units[k].state = State::Quarantined; } + /// An owner a firmware recovery passed without completing it (an aborted receipt). The + /// firmware no longer runs it, so its index returns to service for other storage, as a + /// fresh owner. Its own storage is never used again: its job failed, which leaves its VM + /// a sticky error (no later publication), and the engine retains the job and so keeps the + /// storage mapped. It is not a retired owner: it never proves that an owner published + /// before it was displaced, which still takes a later full retirement. + pub(crate) fn abandon(&mut self, k: usize) { + let Some(u) = self.units.get_mut(k) else { + return; + }; + if u.state != State::Published { + return; + } + u.state = State::Fresh; + u.previous = State::Fresh; + u.generation = 0; + self.abandoned = self.abandoned.saturating_add(1); + } } #[cfg(test)] mod tests { use super::*; #[test] + fn an_abandoned_owner_returns_to_service_but_displaces_nothing() { + let mut l = Lifetime::<2>::new(false); + assert!(l.claim(0)); + assert!(l.publish(0)); + assert!(l.retire(0)); + assert!(l.claim(1)); + assert!(l.publish(1)); + l.abandon(1); + assert!(!l.exhausted()); + assert!(!l.claim(0)); // The abort is no displacement of owner 0. + assert!(l.claim(1)); // Its index serves other storage. + assert!(l.publish(1)); + assert!(l.retire(1)); + assert!(l.claim(0)); // A later full retirement displaced owner 0. + assert_eq!(l.counts().abandoned, 1); + assert_eq!(l.counts().quarantined, 0); + } + #[test] fn distinct_retired_displacement_is_required() { - let mut l = Lifetime::new(); + let mut l = Lifetime::<2>::new(false); assert!(l.claim(0)); assert!(l.publish(0)); assert!(!l.claim(0)); @@ -164,7 +236,7 @@ mod tests { } #[test] fn abort_does_not_recycle_or_authorize_displacement() { - let mut l = Lifetime::new(); + let mut l = Lifetime::<2>::new(false); assert!(l.claim(0)); assert!(l.publish(0)); assert!(l.retire(0)); @@ -185,7 +257,7 @@ mod tests { } #[test] fn preparation_cancel_preserves_retirement() { - let mut l = Lifetime::new(); + let mut l = Lifetime::<2>::new(false); assert!(l.claim(0)); l.cancel(0); assert_eq!(l.generation(0), 0); @@ -202,7 +274,7 @@ mod tests { } #[test] fn publication_is_serialized_and_command_count_is_not_binding_generation() { - let mut l = Lifetime::new(); + let mut l = Lifetime::<2>::new(false); assert!(l.claim(0)); assert!(l.publish(0)); assert!(l.claim(1)); @@ -219,4 +291,56 @@ mod tests { assert_eq!(l.generation(1), 1); assert!(l.claim(0)); } + #[test] + fn pipelined_owners_keep_n_minus_one_in_flight() { + let mut l = Lifetime::<4>::new(true); + // Fresh owners publish back to back. + for k in 0..4 { + assert!(l.claim(k)); + assert!(l.publish(k)); + } + assert_eq!(l.published(), 4); + // Out of order retirement is refused. + assert!(!l.retire(1)); + assert!(l.retire(0)); + // Owner 0 retired but nothing after it has: not displaced yet. + assert!(!l.claim(0)); + assert!(l.retire(1)); + assert!(l.claim(0)); // displaced by 1 + assert!(!l.claim(1)); // 2 still in flight + assert!(l.publish(0)); + assert!(l.retire(2)); + assert!(l.claim(1)); + assert!(l.publish(1)); + // Steady state: round robin with up to three in flight. + let mut next = 2; + for _ in 0..1000 { + let oldest = (0..4) + .filter(|&k| l.units[k].state == State::Published) + .min_by_key(|&k| l.units[k].publication) + .unwrap(); + assert!(l.retire(oldest)); + assert!(l.claim(next)); + assert!(l.publish(next)); + assert!(l.published() <= 3); + next = (next + 1) % 4; + } + } + #[test] + fn pipelined_newest_retired_owner_is_never_reusable() { + let mut l = Lifetime::<3>::new(true); + for k in 0..3 { + assert!(l.claim(k)); + assert!(l.publish(k)); + } + for k in 0..3 { + assert!(l.retire(k)); + } + assert!(l.available(0) && l.available(1)); + assert!(!l.available(2)); + assert!(l.claim(1)); + // 2 still proves 0 displaced although 1 is being prepared again. + assert!(l.available(0)); + assert!(!l.available(2)); + } } diff --git a/drivers/gpu/drm/asahi/g16/queue.rs b/drivers/gpu/drm/asahi/g16/queue.rs index 41da5b94c7c50..ada1946bb7983 100644 --- a/drivers/gpu/drm/asahi/g16/queue.rs +++ b/drivers/gpu/drm/asahi/g16/queue.rs @@ -55,6 +55,10 @@ pub(crate) struct QueueState { /// Exact successful firmware receipts, independently for render and compute. These /// survive aggregate-fence publication and never advance on aborted/failed commands. pub(crate) completed: [AtomicU64; 2], + /// Published command frontier per engine. The firmware runs the compute queue in + /// publication order, each command to its Finalize, so a compute command may follow its + /// published (not yet retired) compute predecessors. + pub(crate) published: [AtomicU64; 2], /// Owned-job counts at the queue's last 32 arrivals (compute horizon ahead), recorded by the /// engine when a job arrives. pub(crate) owned: [AtomicU32; 32], @@ -140,6 +144,7 @@ impl Queue { cancelled_at: [const { AtomicU64::new(0) }; 2], turn: AtomicU64::new(0), completed: [const { AtomicU64::new(0) }; 2], + published: [const { AtomicU64::new(0) }; 2], owned: [const { AtomicU32::new(0) }; 32], owned_index: AtomicU32::new(0), }, diff --git a/drivers/gpu/drm/asahi/g16/recovery.rs b/drivers/gpu/drm/asahi/g16/recovery.rs index b0fc2ed96976c..18975b812e276 100644 --- a/drivers/gpu/drm/asahi/g16/recovery.rs +++ b/drivers/gpu/drm/asahi/g16/recovery.rs @@ -320,6 +320,9 @@ impl Firmware { } self.recovery.running = true; self.engine.frozen = true; + // A recovery may reset what the firmware keeps for a bound PB epoch, also one whose + // render was not affected: start the next render on each pass slot fresh. + self.pools.invalidate(); self.events.grow.clear(); // These frontiers must be captured before the host answers 2: the // firmware can force the stage target while aborting a work item. diff --git a/drivers/gpu/drm/asahi/g16/resources.rs b/drivers/gpu/drm/asahi/g16/resources.rs index 77c47f0770492..63238572b27ee 100644 --- a/drivers/gpu/drm/asahi/g16/resources.rs +++ b/drivers/gpu/drm/asahi/g16/resources.rs @@ -13,13 +13,18 @@ use super::{ vm::VmState, }; use crate::mmu; -use core::sync::atomic::{AtomicU64, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use kernel::{ - new_mutex, + new_mutex, new_spinlock, prelude::*, - sync::{Arc, Mutex}, + sync::{Arc, Mutex, SpinLock}, }; +/// Never-exposed PB epochs a VM keeps for its next renders. With PB epoch reuse most +/// renders never expose the epoch prepared for them; allocating and mapping a fresh one per +/// render submit costs far more than the render itself. +const SPARE_EPOCHS: usize = 4; + pub(crate) struct ComputeArena(crate::cleanup::Deferred); impl core::ops::Deref for ComputeArena { type Target = ComputeInner; @@ -34,17 +39,30 @@ pub(crate) struct ComputeInner { _bind: mmu::VmBind, _live: Arc, } +/// Compute units every VM gets with its first compute command; more are added +/// ([`Cache::prepare`]) once it submits compute back to back. +pub(crate) const BASE_COMPUTE_UNITS: usize = 2; + +/// Back-to-back compute submissions (each within [`BURST_GAP_NS`] of the previous) after +/// which a VM gets its extension units; a single submission with [`BURST_COMMANDS`] compute +/// commands qualifies at once. +const BURST_SUBMITS: u32 = 3; +const BURST_GAP_NS: u64 = 2_000_000; +const BURST_COMMANDS: usize = 3; + impl ComputeArena { + /// Units `units` (their index fixes their client addresses and hardware UMA slot). fn new( factory: &Factory, bind: &mmu::VmBind, vm: &mmu::Vm, live: &Arc, + units: core::ops::Range, ) -> Result> { let mut objects = factory.objects(vm); - let mut compute = KVec::with_capacity(2, GFP_KERNEL)?; - for k in 0..2 { - compute.push(ComputeSlot::new(&mut objects, k)?, GFP_KERNEL)?; + let mut compute = KVec::with_capacity(units.len(), GFP_KERNEL)?; + for k in units { + compute.push(ComputeSlot::new(&mut objects, k as u64)?, GFP_KERNEL)?; } objects.finish_private()?; Ok(Arc::new( @@ -116,12 +134,23 @@ impl RenderArena { #[derive(Default)] struct Ready { compute: Option>, + /// Units from [`BASE_COMPUTE_UNITS`] on, for a VM that keeps compute queued. + compute_ext: Option>, render: Option>, + /// Last compute submission and the back-to-back run it continues. + last_compute_ns: u64, + burst: u32, } #[pin_data] pub(crate) struct Cache { #[pin] ready: Mutex, + /// Never-exposed epochs returned by retired jobs (at most [`SPARE_EPOCHS`]). Each holds + /// the VM, which holds this cache: [`Self::close`] breaks that cycle. + #[pin] + spares: SpinLock>>, + /// The VM's client closed: no arena or spare is kept here any more. + closed: AtomicBool, pub(crate) factory: Factory, bind: mmu::VmBind, live: Arc, @@ -134,24 +163,65 @@ impl Cache { live: Arc, ) -> Result> { let budget = super::budget::Budget::new()?; + let spares = KVec::with_capacity(SPARE_EPOCHS, GFP_KERNEL)?; Arc::pin_init( pin_init!(Self { ready <- new_mutex!(Ready::default(), "g16::resources"), + spares <- new_spinlock!(spares, "g16::spare_epochs"), + closed: AtomicBool::new(false), factory, bind, live, budget }), GFP_KERNEL, ) } - pub(crate) fn prepare(&self, vm: &Arc, commands: &[Command]) -> Result { - let compute = commands.iter().any(|c| matches!(c, Command::Compute(_))); + /// Returns a retired job's never-exposed epoch for the next renders. Any context. + fn give_back(&self, epoch: Arc) { + let mut spares = self.spares.lock(); + // Without room, or once the client closed, the epoch drops: its deferred cleanup + // unmaps it as before. + if !self.closed.load(Ordering::Acquire) { + let _ = spares.push_within_capacity(epoch); + } + } + + /// The VM's client closed. Lets go of the arenas and spare epochs: what jobs and the + /// hardware slots' last-owner leases still need, they hold themselves. + pub(crate) fn close(&self) { + let spares = { + let mut spares = self.spares.lock(); + self.closed.store(true, Ordering::Release); + core::mem::take(&mut *spares) + }; + drop(spares); + let ready = { + let mut ready = self.ready.lock(); + ( + ready.compute.take(), + ready.compute_ext.take(), + ready.render.take(), + ) + }; + drop(ready); + } + + pub(crate) fn prepare( + self: &Arc, + vm: &Arc, + commands: &[Command], + ) -> Result { + let computes = commands + .iter() + .filter(|c| matches!(c, Command::Compute(_))) + .count(); + let compute = computes != 0; let render_count = commands .iter() .filter(|c| matches!(c, Command::Render(_))) .count(); let render = render_count != 0; let mut epochs = super::budget::Budget::reserve(&self.budget, render_count.try_into()?)?; - let (compute, render) = { + let (compute, compute_ext, render) = { // Never held by publication, completion, recovery or fence signalling. let mut ready = self.ready.lock(); - if vm.status.get() != 0 || vm.vm.is_closed() { + if vm.status.get() != 0 || vm.vm.is_closed() || self.closed.load(Ordering::Acquire) { return Err(EIO); } if compute && ready.compute.is_none() { @@ -160,8 +230,34 @@ impl Cache { &self.bind, &vm.vm, &self.live, + 0..BASE_COMPUTE_UNITS, )?); } + if compute { + // SAFETY: ktime_get has no preconditions. + let now = unsafe { kernel::bindings::ktime_get() } as u64; + ready.burst = if now.wrapping_sub(ready.last_compute_ns) < BURST_GAP_NS { + ready.burst.saturating_add(1) + } else { + 0 + }; + ready.last_compute_ns = now; + let units = super::params::compute_units(); + if ready.compute_ext.is_none() + && units > BASE_COMPUTE_UNITS + && (ready.burst >= BURST_SUBMITS || computes >= BURST_COMMANDS) + { + // Best effort: without them the VM keeps one command in flight. + ready.compute_ext = ComputeArena::new( + &self.factory, + &self.bind, + &vm.vm, + &self.live, + BASE_COMPUTE_UNITS..units, + ) + .ok(); + } + } if render && ready.render.is_none() { ready.render = Some(RenderArena::new( &self.factory, @@ -176,6 +272,7 @@ impl Cache { } ( if compute { ready.compute.clone() } else { None }, + if compute { ready.compute_ext.clone() } else { None }, if render { ready.render.clone() } else { None }, ) }; @@ -183,7 +280,13 @@ impl Cache { for command in commands { pb.push( if matches!(command, Command::Render(_)) { - Some(PbEpoch::new(&self.factory, vm, epochs.take()?)?) + // A spare keeps the budget permit it was created with; the reservation + // returns the unused credit when it drops. + let spare = self.spares.lock().pop(); + Some(match spare { + Some(epoch) if epoch.unexposed() => epoch, + _ => PbEpoch::new(&self.factory, vm, epochs.take()?)?, + }) } else { None }, @@ -195,8 +298,10 @@ impl Cache { } Ok(Prepared { compute, + compute_ext, render, pb, + cache: Some(self.clone()), }) } } @@ -204,13 +309,60 @@ impl Cache { /// Immutable references carried by an accepted job. No fallible lazy work in the engine. pub(crate) struct Prepared { pub(crate) compute: Option>, + /// The VM's extension compute units when the job was prepared. + pub(crate) compute_ext: Option>, pub(crate) render: Option>, pb: KVec>>, + /// The VM cache that unused epochs return to. + cache: Option>, +} + +impl Drop for Prepared { + fn drop(&mut self) { + let Some(cache) = self.cache.take() else { + return; + }; + for epoch in self.pb.iter_mut().filter_map(Option::take) { + if epoch.unexposed() { + cache.give_back(epoch); + } + } + } } impl Prepared { pub(crate) fn compute(&self) -> Result<&ComputeArena> { self.compute.as_deref().ok_or(EINVAL) } + /// Compute units this job can use. + pub(crate) fn compute_units(&self) -> Result { + Ok(self.compute()?.compute.len() + + self.compute_ext.as_deref().map_or(0, |e| e.compute.len())) + } + /// The arena holding compute unit `k`. + pub(crate) fn compute_arena(&self, k: usize) -> Option> { + let base = self.compute.as_ref()?; + if k < base.compute.len() { + Some(base.clone()) + } else { + self.compute_ext.clone() + } + } + /// The render arena. + pub(crate) fn render_arena(&self) -> Option> { + self.render.clone() + } + /// Compute unit `k`: the VM's base units, then its extension units. + pub(crate) fn compute_unit(&self, k: usize) -> Result<&ComputeSlot> { + let base = &self.compute()?.compute; + match base.get(k) { + Some(unit) => Ok(unit), + None => self + .compute_ext + .as_deref() + .and_then(|e| e.compute.get(k - base.len())) + .ok_or(EINVAL), + } + } pub(crate) fn render(&self) -> Result<&RenderArena> { self.render.as_deref().ok_or(EINVAL) } diff --git a/drivers/gpu/drm/asahi/g16/slots.rs b/drivers/gpu/drm/asahi/g16/slots.rs index b4a48ed2907ae..562c45e4ba945 100644 --- a/drivers/gpu/drm/asahi/g16/slots.rs +++ b/drivers/gpu/drm/asahi/g16/slots.rs @@ -2,10 +2,13 @@ //! Per-VM M4 command storage and device-wide displacement lifetime tracking. //! -//! Both compute units alternate hardware UMA slot 1; both render units alternate slot 33. -//! Firmware retirement and a later distinct retired owner are both required before reuse. +//! Compute unit k owns hardware UMA slot 1 + k; the units may be in flight together, in +//! publication order. Both render units alternate slot 33, one at a time. Firmware retirement +//! and a later distinct retired owner are both required before reuse. //! Private backing and old VM leases survive until a distinct replacement fully retires. -//! Aborted storage remains mapped and quarantined. Pool cursors are never scrubbed. +//! Storage a firmware recovery aborted remains mapped for good (its job is retained) and is +//! never used again; its unit returns to service for other storage (`abandon`). Storage that +//! cannot be trusted is quarantined. Pool cursors are never scrubbed. use core::sync::atomic::Ordering; use kernel::{prelude::*, sync::Arc}; @@ -370,21 +373,35 @@ pub(crate) struct SlotStats { pub(crate) pass_scrubs: u64, } +/// Displacement ledger and VM leases of the compute units (see [`Slots`]). +pub(crate) struct ComputeOwners { + lifetime: Lifetime<{ sizes::COMPUTE_UNITS }>, + owner: [Option>; sizes::COMPUTE_UNITS], + pending_retarget: [Option; sizes::COMPUTE_UNITS], + pending_owner: [Option>; sizes::COMPUTE_UNITS], + /// The arena holding each owner's unit: the firmware may still write the unit's + /// descriptor when a later owner displaces it, after its VM's client closed. + arena: [Option>; sizes::COMPUTE_UNITS], + pending_arena: [Option>; sizes::COMPUTE_UNITS], +} + pub(crate) struct Slots { pub(crate) cursor: u32, pub(crate) pass_busy: u32, pub(crate) pass_cursor: u32, pub(crate) stats: SlotStats, - compute_lifetime: Lifetime, - pass_lifetime: Lifetime, + /// The compute owners, on the heap: `Firmware` (and so `Slots`) is built on the probe's + /// stack, which has no room for it. + compute: KBox, + pass_lifetime: Lifetime<2>, // Last-owner leases remain bound through retirement and displacement. Replaced only // when the displaced unit is safely rebound to a new publication. - compute_owner: [Option>; 2], pass_owner: [Option>; 2], - pending_compute_retarget: [Option; 2], pending_pass_retarget: [Option; 2], - pending_compute_owner: [Option>; 2], pending_pass_owner: [Option>; 2], + /// The render arena of each pass slot's owner (see [`ComputeOwners::arena`]). + pass_arena: [Option>; 2], + pending_pass_arena: [Option>; 2], } impl Slots { pub(crate) fn new() -> Result { @@ -393,14 +410,23 @@ impl Slots { pass_busy: 0, pass_cursor: 0, stats: SlotStats::default(), - compute_lifetime: Lifetime::new(), - pass_lifetime: Lifetime::new(), - compute_owner: [None, None], + compute: KBox::new( + ComputeOwners { + lifetime: Lifetime::new(true), + owner: [const { None }; sizes::COMPUTE_UNITS], + pending_retarget: [None; sizes::COMPUTE_UNITS], + pending_owner: [const { None }; sizes::COMPUTE_UNITS], + arena: [const { None }; sizes::COMPUTE_UNITS], + pending_arena: [const { None }; sizes::COMPUTE_UNITS], + }, + GFP_KERNEL, + )?, + pass_lifetime: Lifetime::new(false), pass_owner: [None, None], - pending_compute_retarget: [None, None], pending_pass_retarget: [None, None], - pending_compute_owner: [None, None], pending_pass_owner: [None, None], + pass_arena: [None, None], + pending_pass_arena: [None, None], }) } pub(crate) fn init(&self) -> Result { @@ -408,10 +434,10 @@ impl Slots { } #[allow(dead_code)] pub(crate) fn compute_generation(&self, k: usize) -> Result { - if k >= 2 { + if k >= sizes::COMPUTE_UNITS { return Err(EINVAL); } - Ok(self.compute_lifetime.generation(k)) + Ok(self.compute.lifetime.generation(k)) } #[allow(dead_code)] pub(crate) fn pass_generation(&self, k: usize) -> Result { @@ -421,39 +447,57 @@ impl Slots { Ok(self.pass_lifetime.generation(k)) } pub(crate) fn compute_exhausted(&self) -> bool { - self.compute_lifetime.exhausted() + // Ledger indices above the most units a VM gets never have an owner. + self.compute + .lifetime + .exhausted_within(super::params::compute_units()) + } + /// Whether none of the first `n` compute units (a VM's own) can ever be claimed again. + pub(crate) fn compute_exhausted_within(&self, n: usize) -> bool { + self.compute.lifetime.exhausted_within(n) } pub(crate) fn render_exhausted(&self) -> bool { self.pass_lifetime.exhausted() } pub(crate) fn lifetime_counts(&self) -> [super::pools::lifetime::Counts; 2] { - [self.compute_lifetime.counts(), self.pass_lifetime.counts()] + [self.compute.lifetime.counts(), self.pass_lifetime.counts()] + } + /// Whether one of the first `n` compute units can be claimed now. + pub(crate) fn compute_available(&self, n: usize) -> bool { + (0..n.min(sizes::COMPUTE_UNITS)).any(|k| self.compute.lifetime.available(k)) + } + /// Compute units published and not yet retired. + pub(crate) fn compute_published(&self) -> u32 { + self.compute.lifetime.published() } #[allow(dead_code)] pub(crate) fn free_compute(&self) -> u32 { - (0..2) - .filter(|&k| self.compute_lifetime.available(k)) + (0..super::params::compute_units()) + .filter(|&k| self.compute.lifetime.available(k)) .count() as u32 } pub(crate) fn claim_compute( &mut self, - arena: &super::resources::ComputeArena, + units: &super::resources::Prepared, f: &mut Claims, _j: u32, ) -> Result> { if f.slot_count != 0 { return Ok(None); } - for i in 0..2 { - let k = (self.cursor as usize + i) % 2; - if self.compute_lifetime.claim(k) { - if let Err(e) = arena.compute[k].scrub() { - self.compute_lifetime.cancel(k); + // Round robin over this VM's units: the next one in publication order is the + // oldest, the first to be displaced by a later retired publication. + let n = units.compute_units()?.min(sizes::COMPUTE_UNITS); + for i in 0..n { + let k = (self.cursor as usize + i) % n; + if self.compute.lifetime.claim(k) { + if let Err(e) = units.compute_unit(k).and_then(|unit| unit.scrub()) { + self.compute.lifetime.cancel(k); return Err(e); } f.slot[0] = k as u32; f.slot_count = 1; - self.cursor = (1 - k) as u32; + self.cursor = ((k + 1) % n) as u32; return Ok(Some(k)); } } @@ -522,7 +566,7 @@ impl Slots { /// Full wire retirement is a separate condition for hardware qualification. pub(crate) fn note_published(&mut self, f: &Claims) { for &k in &f.slot[..f.slot_count as usize] { - if let Some(proof) = self.pending_compute_retarget[k as usize].take() { + if let Some(proof) = self.compute.pending_retarget[k as usize].take() { self.stats.cross_vm_retargets[0] = self.stats.cross_vm_retargets[0].saturating_add(1); self.stats.last_retarget[0] = proof; @@ -543,17 +587,19 @@ impl Slots { /// A previous lease is replaced only after the new publication fully retires. pub(crate) fn hold_vm(&mut self, f: &Claims, vm: &Arc) -> Result { if f.slot_count > 2 - || f.slot[..f.slot_count as usize].iter().any(|&k| k >= 2) + || f.slot[..f.slot_count as usize] + .iter() + .any(|&k| k as usize >= sizes::COMPUTE_UNITS) || f.pass_mask & !3 != 0 { return Err(EINVAL); } for &k in &f.slot[..f.slot_count as usize] { let k = k as usize; - self.pending_compute_retarget[k] = self.compute_owner[k] + self.compute.pending_retarget[k] = self.compute.owner[k] .as_ref() .and_then(|old| Self::vm_retarget(k, old, vm)); - self.pending_compute_owner[k] = Some(vm.clone()); + self.compute.pending_owner[k] = Some(vm.clone()); } for k in 0..2 { if f.pass_mask & (1 << k) != 0 { @@ -566,10 +612,25 @@ impl Slots { Ok(()) } + /// With [`Self::hold_vm`]: the claimed units' arenas, held with their VM leases. The VM's + /// own cache lets go of its arenas when its client closes. + pub(crate) fn hold_arenas(&mut self, f: &Claims, resources: &super::resources::Prepared) { + for &k in &f.slot[..(f.slot_count as usize).min(f.slot.len())] { + if let Some(cell) = self.compute.pending_arena.get_mut(k as usize) { + *cell = resources.compute_arena(k as usize); + } + } + for k in 0..2 { + if f.pass_mask & (1 << k) != 0 { + self.pending_pass_arena[k] = resources.render_arena(); + } + } + } + /// Reserve the actual same-slot publication. Call before producers become visible. pub(crate) fn publish(&mut self, f: &Claims) -> Result { // Validate on a private copy so a failure changes no ownership. - let mut c = self.compute_lifetime.clone(); + let mut c = self.compute.lifetime.clone(); let mut p = self.pass_lifetime.clone(); for &k in &f.slot[..f.slot_count as usize] { if !c.publish(k as usize) { @@ -581,13 +642,13 @@ impl Slots { return Err(EBUSY); } } - self.compute_lifetime = c; + self.compute.lifetime = c; self.pass_lifetime = p; Ok(()) } /// Full wire retirement is a precondition; completion fences alone are insufficient. pub(crate) fn retire(&mut self, f: &mut Claims) -> Result { - let mut c = self.compute_lifetime.clone(); + let mut c = self.compute.lifetime.clone(); let mut p = self.pass_lifetime.clone(); for &k in &f.slot[..f.slot_count as usize] { if !c.retire(k as usize) { @@ -599,14 +660,16 @@ impl Slots { return Err(EINVAL); } } - self.compute_lifetime = c; + self.compute.lifetime = c; self.pass_lifetime = p; for &k in &f.slot[..f.slot_count as usize] { - self.compute_owner[k as usize] = self.pending_compute_owner[k as usize].take(); + self.compute.owner[k as usize] = self.compute.pending_owner[k as usize].take(); + self.compute.arena[k as usize] = self.compute.pending_arena[k as usize].take(); } for k in 0..2 { if f.pass_mask & (1 << k) != 0 { self.pass_owner[k] = self.pending_pass_owner[k].take(); + self.pass_arena[k] = self.pending_pass_arena[k].take(); } } self.pass_busy &= !f.pass_mask; @@ -617,8 +680,8 @@ impl Slots { } pub(crate) fn quarantine(&mut self, f: &mut Claims) { for &k in &f.slot[..f.slot_count as usize] { - self.compute_lifetime.quarantine(k as usize); - self.pending_compute_retarget[k as usize] = None; + self.compute.lifetime.quarantine(k as usize); + self.compute.pending_retarget[k as usize] = None; } for k in 0..2 { if f.pass_mask & (1 << k) != 0 { @@ -631,18 +694,45 @@ impl Slots { f.pass_mask = 0; f.ct_count = 0; } - /// Cancels preparation only. Published units require retire() or quarantine(). + /// The firmware's recovery passed the flight of `f` without completing it: its units return + /// to service ([`Lifetime::abandon`]). The flight's own storage leases are dropped; the + /// caller retains its job, which keeps that storage mapped. Each unit's previous owner + /// keeps its lease until a later owner fully retires, as after any publication. + pub(crate) fn abandon(&mut self, f: &mut Claims) { + for &k in &f.slot[..f.slot_count as usize] { + let k = k as usize; + self.compute.lifetime.abandon(k); + self.compute.pending_retarget[k] = None; + self.compute.pending_owner[k] = None; + self.compute.pending_arena[k] = None; + } + for k in 0..2 { + if f.pass_mask & (1 << k) != 0 { + self.pass_lifetime.abandon(k); + self.pending_pass_retarget[k] = None; + self.pending_pass_owner[k] = None; + self.pending_pass_arena[k] = None; + } + } + self.pass_busy &= !f.pass_mask; + f.slot_count = 0; + f.pass_mask = 0; + f.ct_count = 0; + } + /// Cancels preparation only. Published units require retire(), abandon() or quarantine(). pub(crate) fn release(&mut self, f: &mut Claims) { for &k in &f.slot[..f.slot_count as usize] { - self.compute_lifetime.cancel(k as usize); - self.pending_compute_retarget[k as usize] = None; - self.pending_compute_owner[k as usize] = None; + self.compute.lifetime.cancel(k as usize); + self.compute.pending_retarget[k as usize] = None; + self.compute.pending_owner[k as usize] = None; + self.compute.pending_arena[k as usize] = None; } for k in 0..2 { if f.pass_mask & (1 << k) != 0 { self.pass_lifetime.cancel(k); self.pending_pass_retarget[k] = None; self.pending_pass_owner[k] = None; + self.pending_pass_arena[k] = None; } } self.pass_busy &= !f.pass_mask; diff --git a/drivers/gpu/drm/asahi/g16/vm.rs b/drivers/gpu/drm/asahi/g16/vm.rs index 5bb4f434454cc..ce75ee6e004fc 100644 --- a/drivers/gpu/drm/asahi/g16/vm.rs +++ b/drivers/gpu/drm/asahi/g16/vm.rs @@ -254,6 +254,11 @@ impl Registry { let index = vms.iter().position(|state| state.id == id); index.and_then(|index| vms.remove(index).ok()) }; + // Arenas still reachable by firmware stay with the slot leases and jobs that hold + // them; the rest (and the spare epochs, which hold the VM) go now. + if let Some(cache) = state.as_ref().and_then(|state| state.resources.as_ref()) { + cache.close(); + } drop(state); } diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 2a18ebbeb5d9c..7118cffde3301 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -203,6 +203,14 @@ pub(crate) struct Destinations(Option bool { + self.0 + .as_deref() + .is_none_or(|destinations| destinations.iter().all(Option::is_none)) + } + /// G16 retains its eight-byte destination alignment and EINVAL contract. #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn resolve_aligned( diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py index b0c92c63fd028..22ff751a2e415 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-command-stream.py @@ -356,8 +356,16 @@ let order=Order::new([4,8],[2,1],[1,NONE],[3,7],RENDER).unwrap(); assert_eq!(order.sequence,7); assert_eq!(order.wait_through,[Some(5),Some(7)]); - assert!(!order.ready([6,6])); - assert!(order.ready([6,7])); + assert!(!order.ready([6,6],[6,6])); + assert!(order.ready([6,7],[6,7])); + // A compute command may follow published compute predecessors (the firmware compute + // queue runs them in order); every cross-engine dependency still needs completion. + let compute=Order::new([0,2],[0,0],[NONE,NONE],[0,2],COMPUTE).unwrap(); + assert!(!compute.ready([0,1],[0,1])); + assert!(compute.ready([0,1],[0,2])); + let render=Order::new([0,2],[0,0],[NONE,NONE],[0,2],RENDER).unwrap(); + assert!(!render.ready([0,1],[0,2])); + assert!(render.ready([0,2],[0,2])); } #[test] fn geometry_capacity_boundary_uses_the_real_encoder() { diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-job-failure.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-job-failure.py index 62c21f5aad575..efe5a60fd2cfd 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-job-failure.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-job-failure.py @@ -43,7 +43,7 @@ def function(text, name): type Result = core::result::Result; const EIO:Error=Error(-5); const ENODEV:Error=Error(-19); const ECANCELED:Error=Error(-125); const ETIMEDOUT:Error=Error(-110);const ENOMEM:Error=Error(-12);const EAGAIN:Error=Error(-11); const EINVAL:Error=Error(-22); -const EOVERFLOW:Error=Error(-75); const NONE:u16=u16::MAX; +const EOVERFLOW:Error=Error(-75); const NONE:u16=u16::MAX; const COMPUTE:usize=1; ORDER_SOURCE pub mod task { pub const MAX_SCHEDULE_TIMEOUT:i64=i64::MAX; } struct InputState { status:AtomicI32,waits:AtomicUsize,interrupted:AtomicI64,gate:Mutex<()>,changed:Condvar } @@ -96,7 +96,7 @@ def function(text, name): } } mod command { pub(crate) use super::Order; } -struct QueueState { completed:[AtomicU64;2],cancelled_at:[AtomicU64;2] } +struct QueueState { completed:[AtomicU64;2],published:[AtomicU64;2],cancelled_at:[AtomicU64;2] } impl QueueState { QUEUE_METHODS } struct Job { id:u64,vm:Vm,_guard:Guard,progress:Progress,started:AtomicBool,accepted:AtomicBool,fence:Fence,commands:Vec<()>,inputs:Vec,orders:Vec,queue:Arc,full_retired:AtomicU32,terminal_retired:AtomicBool } struct Shared { crashed:AtomicBool,submitted:Mutex>,submit_result:Result } @@ -116,7 +116,7 @@ def function(text, name): } fn job_with_inputs(id:u64,status:Arc,inputs:Vec)->Arc { let orders=vec![Order::new([0,0],[0,0],[NONE;2],[0;2],1).unwrap(),Order::new([0,0],[0,1],[NONE;2],[0;2],1).unwrap()]; - job_with_orders(id,status,inputs,orders,Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]})) + job_with_orders(id,status,inputs,orders,Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],published:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]})) } fn job_with_orders(id:u64,status:Arc,inputs:Vec,orders:Vec,queue:Arc)->Arc { Arc::new(Job{id,vm:Vm{status:status.clone()},_guard:Guard(AtomicBool::new(false)), @@ -278,7 +278,7 @@ def function(text, name): } #[test] fn cancellation_checks_both_command_ordinal_and_cross_engine_failed_prefix() { - let queue=Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]}); + let queue=Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],published:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]}); let cutoff=Order::new([0,2],[0,0],[NONE;2],[0;2],1).unwrap();queue.cancel(&[cutoff],0); let earlier=Order::new([0,1],[0,0],[NONE;2],[0;2],1).unwrap(); let independent_render=Order::new([0,3],[0,0],[NONE;2],[0;2],0).unwrap(); @@ -292,7 +292,7 @@ def function(text, name): } #[test] fn exact_prefix_does_not_wait_for_the_aggregate_or_next_ordinal() { - let queue=Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]}); + let queue=Arc::new(QueueState{completed:[const{AtomicU64::new(0)};2],published:[const{AtomicU64::new(0)};2],cancelled_at:[const{AtomicU64::new(0)};2]}); let status=Arc::new(Status::new()); let old=job_with_orders(1,status.clone(),vec![],vec![ Order::new([0,0],[0,0],[NONE;2],[0;2],1).unwrap(), diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-publication.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-publication.py index 6f388b4693fd4..029fee2ee9650 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-publication.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-publication.py @@ -39,7 +39,8 @@ def item(declaration): fn count(&self)->u32{self.count} fn set_pending(&self,v:bool){self.pending.store(v,Ordering::Relaxed)} } -struct Job { id:u32,vm:Vm,queue:Queue,progress:Progress,commands:Vec,dependency:Result,terminal_marked:AtomicBool } +struct Resources;impl Resources{fn compute_units(&self)->Result{Ok(2)}} +struct Job { id:u32,vm:Vm,queue:Queue,progress:Progress,commands:Vec,dependency:Result,terminal_marked:AtomicBool,resources:Resources } impl Job { fn check_command_dependencies(&self,_index:u32)->Result{self.dependency} fn mark_terminal_retired(&self)->Result{assert!(!self.progress.pending.load(Ordering::Relaxed));self.terminal_marked.store(true,Ordering::Relaxed);Ok(())} @@ -64,6 +65,7 @@ def item(declaration): impl Slots{ fn compute_exhausted(&self)->bool{self.compute_exhausted} fn render_exhausted(&self)->bool{self.render_exhausted} + fn compute_exhausted_within(&self,_units:usize)->bool{self.compute_exhausted} fn release(&mut self,_claims:&mut Claims){self.releases+=1;} fn quarantine(&mut self,_claims:&mut Claims){self.quarantines+=1;} } @@ -114,6 +116,9 @@ def item(declaration): } impl Firmware { fn mark_class_unavailable(&mut self,class:usize,error:Error){self.unavailable.push((class,error));} + // Queuing behind in-flight compute is exercised on hardware; this recorder keeps the + // serialized selection and error isolation under test. + fn can_queue_behind(&self,_job:&Arc)->bool{false} fn service_power(&self)->Result{Ok(())} fn drain_rings(&mut self)->Result{Ok(())} fn check_hangs(&mut self,_done:&mut Completions)->Result{Ok(())} @@ -144,7 +149,7 @@ def item(declaration): Arc::new(Job{id,vm:Vm{status:Status(AtomicU32::new(0)),slot:id}, queue:Queue{turn:AtomicU64::new(0),priority:0}, progress:Progress{sent:AtomicU32::new(sent),count:sent+1,pending:AtomicBool::new(true)}, - commands:vec![if render{Command::Render(())}else{Command::Compute(())};sent as usize+1],dependency,terminal_marked:AtomicBool::new(false)}) + commands:vec![if render{Command::Render(())}else{Command::Compute(())};sent as usize+1],dependency,terminal_marked:AtomicBool::new(false),resources:Resources}) } fn firmware(jobs:Vec>,stages:Vec<(Result,bool)>)->Firmware{ Firmware{session:Session{shared:Shared{sleeping:AtomicBool::new(false)}},objects:Checks,recovery:Recovery{error:None,publication_wait_deadline:None}, diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py index 36f4cdeac268e..d3c3df963d607 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-recovery.py @@ -29,21 +29,23 @@ use super::logic::{self, Retirement}; // This is the retirement policy used by poll.rs: an affected fence does not - // erase the independently authenticated owner receipt. No synthetic receipt - // is generated here; the hardware poller remains responsible for that proof. - fn finish(owners: &mut Lifetime, unit: usize, receipt: Retirement, affected: bool) -> bool { + // erase the independently authenticated owner receipt, and an aborted owner is + // abandoned (its job fails, its storage stays retained, its unit serves other + // storage). No synthetic receipt is generated here; the hardware poller remains + // responsible for that proof. + fn finish(owners: &mut Lifetime, unit: usize, receipt: Retirement, affected: bool) -> bool { assert!(receipt.complete()); if receipt.owner_retired() { assert!(owners.retire(unit)); } else { - owners.quarantine(unit); + owners.abandon(unit); } receipt.job_failed(affected) } #[test] fn ten_thousand_affected_full_receipts_preserve_displacement() { - let mut owners = Lifetime::new(); + let mut owners = Lifetime::<2>::new(false); for number in 0..10_000 { let unit = number % 2; assert!(owners.claim(unit)); @@ -58,64 +60,100 @@ #[test] fn actual_abort_never_supplies_a_displacement_receipt() { - let mut owners = Lifetime::new(); + let mut owners = Lifetime::<2>::new(false); assert!(owners.claim(0)); assert!(owners.publish(0)); assert!(!finish(&mut owners, 0, Retirement::Full, false)); assert!(owners.claim(1)); assert!(owners.publish(1)); assert!(finish(&mut owners, 1, Retirement::Aborted, false)); + // The abort does not displace owner 0; only a later full retirement does. for _ in 0..10_000 { assert!(!owners.claim(0)); - assert!(!owners.claim(1)); - assert!(owners.exhausted()); + assert!(!owners.exhausted()); } assert_eq!(owners.counts(), Counts { - retired: 1, quarantined: 1, ..Counts::default() + fresh: 1, retired: 1, reusable: 1, abandoned: 1, ..Counts::default() }); + assert!(owners.claim(1)); + assert!(owners.publish(1)); + assert!(!finish(&mut owners, 1, Retirement::Full, false)); + assert!(owners.claim(0)); } #[test] - fn exhausted_compute_ledger_does_not_poison_render_ledger() { - let mut compute = Lifetime::new(); - let mut render = Lifetime::new(); + fn an_aborted_compute_owner_exhausts_neither_ledger() { + let mut compute = Lifetime::<2>::new(false); + let mut render = Lifetime::<2>::new(false); assert!(compute.claim(0)); assert!(compute.publish(0)); finish(&mut compute, 0, Retirement::Aborted, true); - // The remaining fresh owner can run once, but cannot make the aborted - // one reusable. Class exhaustion must remain explicit after this point. - assert!(compute.claim(1)); - assert!(compute.publish(1)); - finish(&mut compute, 1, Retirement::Full, false); - assert!(compute.exhausted()); for number in 0..10_000 { let unit = number % 2; + assert!(compute.claim(unit)); + assert!(compute.publish(unit)); + assert!(!finish(&mut compute, unit, Retirement::Full, false)); + assert!(!compute.exhausted()); assert!(render.claim(unit)); assert!(render.publish(unit)); assert!(!finish(&mut render, unit, Retirement::Full, false)); assert!(!render.exhausted()); } - assert_eq!(compute.counts().quarantined, 1); - assert_eq!(render.counts().quarantined, 0); + assert_eq!(compute.counts().quarantined, 0); + assert_eq!(compute.counts().abandoned, 1); + assert_eq!(render.counts().abandoned, 0); } #[test] - fn two_aborts_stay_bounded_and_cannot_be_reset_by_preparation_cancel() { - let mut owners = Lifetime::new(); + fn aborts_cannot_be_reset_by_preparation_cancel_and_leave_both_units_in_service() { + let mut owners = Lifetime::<2>::new(false); for unit in 0..2 { assert!(owners.claim(unit)); assert!(owners.publish(unit)); finish(&mut owners, unit, Retirement::Aborted, true); owners.cancel(unit); } - assert!(owners.exhausted()); - assert_eq!(owners.counts().quarantined, 2); - assert_eq!(owners.counts().reusable, 0); + assert!(!owners.exhausted()); + assert_eq!(owners.counts().abandoned, 2); + assert_eq!(owners.counts().reusable, 2); + assert!(owners.claim(0)); + owners.cancel(0); + assert!(owners.claim(0)); + } + + #[test] + fn a_recovery_aborting_every_pipelined_unit_leaves_compute_in_service() { + // Six units in flight together (the compute queue). Every third round a recovery + // aborts every flight; compute stays available, with as many units as before. + let mut owners = Lifetime::<6>::new(true); + let mut aborted = 0; + for round in 0..1_000 { + let mut published = Vec::new(); + for unit in 0..6 { + if owners.claim(unit) { + assert!(owners.publish(unit)); + published.push(unit); + } + } + assert!(published.len() >= 5, "round {round}: {published:?}"); + for &unit in &published { + let receipt = if round % 3 == 0 { + aborted += 1; + Retirement::Aborted + } else { + Retirement::Full + }; + finish(&mut owners, unit, receipt, false); + } + assert!(!owners.exhausted()); + } + assert_eq!(owners.counts().quarantined, 0); + assert_eq!(owners.counts().abandoned, aborted); } #[test] fn repeated_handshakes_and_storm_limit_preserve_owner_accounting() { - let mut owners = Lifetime::new(); + let mut owners = Lifetime::<2>::new(false); let (mut window, mut count) = (1, 0); for number in 0..logic::WINDOW_LIMIT { assert!(logic::admit(100, &mut window, &mut count)); @@ -155,7 +193,7 @@ 'result': 'pass', 'tests': tests, 'scope': 'Production recovery decisions and owner ledger; CPU-only tests, no hardware recovery qualification.', - 'known_limit': 'A genuine aborted owner remains quarantined until reboot; two-owner class capacity is bounded.', + 'known_limit': 'Only an owner whose storage cannot be trusted (corrupt identity, failed retirement) remains quarantined until reboot; an aborted owner is abandoned and its unit serves other storage.', 'source': { name: hashlib.sha256((production / name).read_bytes()).hexdigest() for name in ['recovery/logic.rs', 'pools/lifetime.rs', 'poll.rs', 'recovery.rs'] diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py index a048447b1436a..ced68b4114095 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-resources.py @@ -44,7 +44,11 @@ def module(text, name): pub const ETIMEDOUT:Error=Error(110);pub const ERESTARTSYS:Error=Error(512); pub const GFP_KERNEL:u32=0; pub type Result = std::result::Result; -pub mod prelude {pub use crate::{Result,KVec,GFP_KERNEL,EINVAL,EAGAIN,E2BIG,EBUSY,EOVERFLOW,EIO,ETIMEDOUT,ERESTARTSYS};} +pub mod prelude {pub use crate::{Result,KVec,KBox,GFP_KERNEL,EINVAL,EAGAIN,E2BIG,EBUSY,EOVERFLOW,EIO,ETIMEDOUT,ERESTARTSYS};} +pub struct KBox(Box); +implKBox{pub fn new(x:T,_:u32)->Result{Ok(Self(Box::new(x)))}} +implstd::ops::Deref for KBox{type Target=T;fn deref(&self)->&T{&self.0}} +implstd::ops::DerefMut for KBox{fn deref_mut(&mut self)->&mut T{&mut self.0}} pub struct KVec(Vec); implKVec{pub fn new()->Self{Self(Vec::new())} pub fn with_capacity(n:usize,_:u32)->Result{Ok(Self(Vec::with_capacity(n)))} pub fn push(&mut self,x:T,_:u32)->Result{self.0.push(x);Ok(())}} implstd::ops::Deref for KVec{type Target=Vec;fn deref(&self)->&Vec{&self.0}} @@ -68,7 +72,7 @@ def module(text, name): } pub struct Arc{inner:std::sync::Arc} - implArc{pub fn new(v:T,_:u32)->crate::Result{Ok(Self{inner:std::sync::Arc::new(v)})}pub fn strong_count(v:&Self)->usize{std::sync::Arc::strong_count(&v.inner)}} + implArc{pub fn new(v:T,_:u32)->crate::Result{Ok(Self{inner:std::sync::Arc::new(v)})}pub fn strong_count(v:&Self)->usize{std::sync::Arc::strong_count(&v.inner)}pub fn ptr_eq(a:&Self,b:&Self)->bool{std::sync::Arc::ptr_eq(&a.inner,&b.inner)}} implClone for Arc{fn clone(&self)->Self{Self{inner:self.inner.clone()}}} implstd::ops::Deref for Arc{type Target=T;fn deref(&self)->&T{&self.inner}} pub mod aref {pub struct ARef(T);impl From<&crate::device::Device>for ARef{fn from(v:&crate::device::Device)->Self{Self(v.clone())}}} @@ -143,10 +147,20 @@ def module(text, name): pub mod resources { use core::sync::atomic::{AtomicU64,Ordering};use crate::*; pub struct ComputeArena{pub compute:KVec} + // A job's compute units; here one arena. + pub type Prepared=ComputeArena; + impl ComputeArena{ + pub fn compute_units(&self)->Result{Ok(self.compute.len())} + pub fn compute_unit(&self,k:usize)->Result<&super::slots::ComputeSlot>{self.compute.get(k).ok_or(EINVAL)} + pub fn compute_arena(&self,_k:usize)->Option>{None} + pub fn render_arena(&self)->Option>{None} + } pub struct RenderArena{pub passes:KVec,pub tpc_dirty:[AtomicU64;2],pub tilemap_dirty:[AtomicU64;2]} impl RenderArena{ __SET_DIRTY__ } } pub mod boot {pub fn eproto()->crate::Error{crate::Error(71)}} + // Arenas here have the two base units. + pub mod params {pub fn compute_units()->usize{2}pub fn pb_heap_blocks()->u32{2048}pub fn pb_posted_blocks(heap:u32)->u32{heap}} pub struct Events{pub grow:crate::KVec<[u32;18]>} pub struct Firmware{pub pools:pools::Pools,pub events:Events} pub mod budget{__BUDGET__} @@ -212,7 +226,8 @@ def module(text, name): aliases.push((0x1000000000,0x1000010000)); // Per-VM context store. aliases.sort();for a in aliases.windows(2){assert!(a[0].1<=a[1].0,"overlap {:x?} {:x?}",a[0],a[1]);} for (k,s) in arena.compute.compute.iter().enumerate(){assert_eq!(s.work.lower_gva.unwrap()&0x3fff,0);assert_eq!(s.work.size,0x2000);assert_eq!(s.pool.size,0x2000000);assert_eq!(s.pool_state.size,0x20000);assert_eq!(s.page_list.size,0x4000);assert_eq!(s.context_store.size,0x40000);assert_eq!(s.pool_state.r32(0).unwrap()as u64 |((s.pool_state.r32(4).unwrap()as u64)<<32),s.pool.gva.unwrap());assert_eq!(pools.allocator_refs_for(k).unwrap().parameter_id,k as u32);} - assert_eq!(aliases.len(),43); + // One PB heap per VM, shared by both render allocators. + assert_eq!(aliases.len(),42); } #[test]fn work_has_kernel_lower_alias_without_application_alias(){ let mut o=Objects::new();let slots=Slots::new().unwrap();let arena=arenas(&mut o); @@ -362,10 +377,10 @@ def module(text, name): #[test]fn pb_installations_are_fresh_and_exposed_records_cannot_be_reinitialized(){ let vm=vm(1);let factory=g16::object::Factory;let budget=g16::budget::Budget::new().unwrap(); let a=g16::pools::PbEpoch::new(&factory,&vm,g16::budget::Budget::reserve(&budget,1).unwrap().take().unwrap()).unwrap(); - a.prepare(0,0x70c0000000).unwrap();a.check_assignment(0).unwrap();a.expose(); + a.prepare(0,0x70c0000000,2048).unwrap();a.check_assignment(0).unwrap();a.expose(); a.manager.w64(0xb0,0x123456789abcdef0).unwrap();let saved=a.manager.snapshot(); - assert_eq!(a.prepare(0,0x70c0000000),Err(EINVAL));assert_eq!(saved,a.manager.snapshot()); - let b=g16::pools::PbEpoch::new(&factory,&vm,g16::budget::Budget::reserve(&budget,1).unwrap().take().unwrap()).unwrap();b.prepare(0,0x70c0000000).unwrap(); + assert_eq!(a.prepare(0,0x70c0000000,2048),Err(EINVAL));assert_eq!(saved,a.manager.snapshot()); + let b=g16::pools::PbEpoch::new(&factory,&vm,g16::budget::Budget::reserve(&budget,1).unwrap().take().unwrap()).unwrap();b.prepare(0,0x70c0000000,2048).unwrap(); assert!(!std::ptr::eq(&a.manager.data,&b.manager.data)); assert_eq!(b.manager.class,g16::object::Class::Fw);assert_eq!(b.manager.gva,None); let mut slots=g16::pools::Pools::new(&driver::AsahiDrmDevice(device::Device)); @@ -373,6 +388,28 @@ def module(text, name): b.check_assignment(0).unwrap();b.expose();assert_eq!(sync::Arc::strong_count(&a),2); slots.retire_binding(0,&b).unwrap();assert_eq!(sync::Arc::strong_count(&a),1); } + #[test]fn pb_reuse_needs_the_same_binding_vm_and_a_completed_render_and_holds_nothing(){ + let one=vm(1);let two=vm(2);let factory=g16::object::Factory;let budget=g16::budget::Budget::new().unwrap(); + let a=g16::pools::PbEpoch::new(&factory,&one,g16::budget::Budget::reserve(&budget,1).unwrap().take().unwrap()).unwrap(); + a.prepare(0,0x70c0000000,2048).unwrap();a.check_assignment(0).unwrap();a.expose(); + let mut p=g16::pools::Pools::new(&driver::AsahiDrmDevice(device::Device)); + let n=p.bind(0,false);p.retire_binding(0,&a).unwrap();p.retire_epoch(true,0,n,true); + assert_eq!(sync::Arc::strong_count(&a),2,"only the owner lease"); + let r=p.reusable(true,0,&one).unwrap();assert!(sync::Arc::ptr_eq(&r,&a));drop(r); + assert_eq!(sync::Arc::strong_count(&a),2); + assert!(p.reusable(false,0,&one).is_none(),"g16_pb_reuse=0"); + assert!(p.reusable(true,0,&two).is_none(),"another VM");assert!(p.reusable(true,1,&one).is_none(),"another slot"); + assert_eq!(p.bind(0,true),n);p.retire_epoch(true,0,n,true);assert!(p.reusable(true,0,&one).is_some()); + p.retire_epoch(true,0,n,false);assert!(p.reusable(true,0,&one).is_none(),"failed render"); + p.retire_epoch(true,0,n,true);p.forget(0);assert!(p.reusable(true,0,&one).is_none(),"aborted render"); + let m=p.bind(0,false);assert_ne!(m,n);p.retire_epoch(true,0,n,true); + assert!(p.reusable(true,0,&one).is_none(),"a stale binding after a fresh one"); + p.retire_epoch(true,0,m,true);assert!(p.reusable(true,0,&one).is_some()); + p.invalidate();assert!(p.reusable(true,0,&one).is_none(),"recovery"); + p.retire_epoch(true,0,m,true);assert!(p.reusable(true,0,&one).is_none(),"a render retiring after recovery"); + assert_eq!(p.bind(0,true),m.wrapping_add(1)); + assert_eq!(sync::Arc::strong_count(&a),2); + } } ''' diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-stamp-wrap.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-stamp-wrap.py index fc1682600f23f..788090c6c639e 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-stamp-wrap.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-stamp-wrap.py @@ -43,10 +43,11 @@ def function(text, name): struct Graph{compute:CGraph,render:RGraph} struct Vm{slot:u32}struct Job{vm:Vm,resources:Resources} struct Resources{compute:Vec,passes:Vec,pools:Pools} -impl Resources{fn compute(&self)->Result<&Self>{Ok(self)}fn render(&self)->Result<&Self>{Ok(self)}} +impl Resources{fn compute(&self)->Result<&Self>{Ok(self)}fn render(&self)->Result<&Self>{Ok(self)}fn compute_unit(&self,k:usize)->Result<&CUnit>{self.compute.get(k).ok_or(EINVAL)}} struct Claims{slot:[u32;1],pass:[u32;1]} struct Flight{computes:u32,renders:u32,target:u32,ta_target:u32,fs_target:u32,job:Job,claims:Claims,head:u32,ta_head:u32,fs_head:u32,rec:[u32;3],recovery_suspect:bool,recovery_started_ns:u64,doomed:bool} -struct Engine{flights:Vec} +struct Engine{flights:Vec,target:u32} +fn stamp_reached(observed:u32,own:u32,tail:u32)->bool{logic::stamp_reached(observed,own,tail)} impl Engine{fn oldest(&self)->Option<&Flight>{self.flights.first()}} struct CUnit{counter:Object,its:Object,work:Object} struct RUnit{scene:Object} @@ -58,6 +59,7 @@ def function(text, name): fn rmb(){}fn now_ns()->u64{100_000_000} impl Firmware{ fn queue_receipt(&self,_ch:Channel,_state:&Object,_info:&Object,_head:u32,_rec:u32)->Result{Ok(self.queue_passed)} + fn queue_passed(&self,_ch:Channel,_state:&Object,_info:&Object,_f:&Flight)->Result{Ok(self.queue_passed)} __RETIRED__ __RETIRED_SNAPSHOT__ } @@ -66,7 +68,7 @@ def function(text, name): let mut f=Firmware{ graph:Graph{compute:CGraph{fw_stamp:Object::default(),qstate:Object::default(),qinfo:Object::default(),notifier:Object::default()}, render:RGraph{fw_stamp:Default::default(),qstate:Default::default(),qinfo:Default::default(),notifier:Object::default()}}, - engine:Engine{flights:vec![Flight{computes:u32::from(!render),renders:u32::from(render),target,ta_target:target,fs_target:target,job:Job{vm:Vm{slot:2},resources:Resources{compute:vec![CUnit{counter:Object::default(),its:Object::default(),work:Object::default()}],passes:vec![RUnit{scene:Object::default()}],pools:Pools{full:true}}},claims:Claims{slot:[0],pass:[0]},head:1,ta_head:1,fs_head:1,rec:[1;3],recovery_suspect:false,recovery_started_ns:0,doomed:false}]}, + engine:Engine{flights:vec![Flight{computes:u32::from(!render),renders:u32::from(render),target,ta_target:target,fs_target:target,job:Job{vm:Vm{slot:2},resources:Resources{compute:vec![CUnit{counter:Object::default(),its:Object::default(),work:Object::default()}],passes:vec![RUnit{scene:Object::default()}],pools:Pools{full:true}}},claims:Claims{slot:[0],pass:[0]},head:1,ta_head:1,fs_head:1,rec:[1;3],recovery_suspect:false,recovery_started_ns:0,doomed:false}],target}, slots:Slots{scrub:Scrub{compute:vec![CUnit{counter:Object::default(),its:Object::default(),work:Object::default()}],passes:vec![RUnit{scene:Object::default()}]}}, pools:Pools{full:true},queue_passed:true, }; @@ -113,6 +115,25 @@ def function(text, name): } }} } +#[test]fn queued_compute_flight_retires_on_passed_receipts_in_order(){ + for n in [0x7fffff,0x800000,0xffffff,0x1000000,0x1000001]{ + let mut f=firmware(false,n); + let later=firmware(false,n+1).engine.flights.remove(0); + f.engine.flights.push(later);f.engine.target=encode::stamp(n+1); + // The command behind it finished too: the shared stamp and notifier cells name it. + f.graph.compute.fw_stamp.set(0,u64::from(encode::stamp(n+1))); + f.graph.compute.notifier.set(notifier_page::COMPUTE_STAMP,u64::from(encode::stamp(n+1))); + f.graph.compute.notifier.set(notifier_page::COMPUTE_CONTEXT,9); + assert_eq!(f.retired(),Ok(Retirement::Full)); + f.graph.compute.fw_stamp.set(0,u64::from(encode::stamp(n-1))); + assert_eq!(f.retired(),Ok(Retirement::Pending)); + f.graph.compute.fw_stamp.set(0,u64::from(encode::stamp(n))); + assert_eq!(f.retired(),Ok(Retirement::Full)); + f.queue_passed=false;assert_eq!(f.retired(),Ok(Retirement::Pending)); + f.queue_passed=true;f.engine.flights[0].job.resources.compute[0].work.set(compute::DESCRIPTOR_COMPLETED,n-1); + assert_eq!(f.retired(),Ok(Retirement::Pending)); + } +} #[test]fn changing_packed_receipt_remains_pending_and_stable_retry_can_retire(){ let mut f=firmware(false,1);f.engine.flights[0].job.resources.compute[0].its.unstable=true; assert_eq!(f.retired(),Ok(Retirement::Pending)); diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-timestamps.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-timestamps.py index 9a159e849e86b..2abd69cf194e1 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-timestamps.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-timestamps.py @@ -161,7 +161,7 @@ def function(source, marker): pub struct Scrub{pub compute:Vec,pub passes:Vec} pub struct Slots{pub scrub:Scrub}pub struct Firmware{pub slots:Slots} pub struct Resources{pub compute:Vec,pub passes:Vec} - impl Resources{pub fn compute(&self)->Result<&Self>{Ok(self)}pub fn render(&self)->Result<&Self>{Ok(self)}} + impl Resources{pub fn compute(&self)->Result<&Self>{Ok(self)}pub fn render(&self)->Result<&Self>{Ok(self)}pub fn compute_unit(&self,k:usize)->Result<&Compute>{self.compute.get(k).ok_or(super::EINVAL)}} pub struct Job{pub commands:Vec,pub resources:Resources} pub struct Flight{pub job:Job,pub first:usize,pub count:usize,pub renders:u32} pub mod poll{use super::*;impl Firmware{ __RETIRE__ }} diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py index add96b48bb335..f4c3dbc4a4988 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py @@ -23,7 +23,8 @@ def item(marker): mod module_parameters{pub struct Param;impl Param{pub fn value(&self)->u32{0}}pub static robust_isolation:Param=Param;} mod types{pub struct ScopeGuard(Option);implScopeGuard{pub fn new(f:F)->Self{Self(Some(f))}pub fn dismiss(mut self){self.0.take();}}implDrop for ScopeGuard{fn drop(&mut self){if let Some(f)=self.0.take(){f()}}}} struct Handle;impl Handle{fn is_closed(&self)->bool{false}} -struct VmState{id:u64,vm:Handle,drops:Arc}impl Drop for VmState{fn drop(&mut self){self.drops.fetch_add(1,Ordering::AcqRel);}} +struct Cache{closes:AtomicU32}impl Cache{fn close(&self){self.closes.fetch_add(1,Ordering::AcqRel);}} +struct VmState{id:u64,vm:Handle,drops:Arc,resources:Option>}impl Drop for VmState{fn drop(&mut self){self.drops.fetch_add(1,Ordering::AcqRel);}} __REGISTRY__ struct Reservation<'a>(Option<&'a Registry>,u64); __RESERVATION_DROP__ @@ -46,10 +47,13 @@ def item(marker): } #[test]fn close_releases_registry_owner_without_another_vm_create(){ let r=Registry::new_test();let drops=Arc::new(AtomicU32::new(0)); - let state=Arc::new(VmState{id:1,vm:Handle,drops:drops.clone()});r.vms.lock().push(state.clone(),0).unwrap(); + let cache=Arc::new(Cache{closes:AtomicU32::new(0)}); + let state=Arc::new(VmState{id:1,vm:Handle,drops:drops.clone(),resources:Some(cache.clone())});r.vms.lock().push(state.clone(),0).unwrap(); r.close(1);assert!(r.vms.lock().is_empty());assert_eq!(drops.load(Ordering::Acquire),0); + // The client's close lets the VM's cache drop its arenas while other owners hold the VM. + assert_eq!(cache.closes.load(Ordering::Acquire),1); drop(state);assert_eq!(drops.load(Ordering::Acquire),1); - r.vms.lock().push(Arc::new(VmState{id:2,vm:Handle,drops:drops.clone()}),0).unwrap(); + r.vms.lock().push(Arc::new(VmState{id:2,vm:Handle,drops:drops.clone(),resources:None}),0).unwrap(); r.close(2);assert_eq!(drops.load(Ordering::Acquire),2); } '''