diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index b6e9444ed5449..d49a311c506c2 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -29,6 +29,7 @@ pub(crate) struct Compute { sequence_bytes: KVec, _binding: mmu::VmBind, stats: Region, + stamp_epoch: timeline::StampEpoch, sequence: u64, last_progress_ns: i64, head: u16, @@ -71,7 +72,7 @@ impl Compute { if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);} objects.push(b,GFP_KERNEL)?; } - Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, + Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, timeline::stamp(0xc1000000,1), 0, registers, profile, binding.generation())?; Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0, profile, 0)?; if profile == super::m3_profile::Profile::Extended { @@ -118,10 +119,10 @@ impl Compute { // the invalidation. crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; - Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi}) + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,stamp_epoch:timeline::StampEpoch::default(),sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi}) } fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, - control: crate::m3::agx_compute::Control, sequence: u64, slot: usize, + control: crate::m3::agx_compute::Control, sequence: u64, stamp_value: u32, slot: usize, register_set: command::RegisterSet, profile: super::m3_profile::Profile, generation: u8) -> Result { let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); @@ -136,7 +137,7 @@ impl Compute { if profile == super::m3_profile::Profile::Extended { microsequence::Sequence::EXTENDED_LENGTH as u32 } else { microsequence::LENGTH as u32 }, stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?, - stamp_value: timeline::stamp(0xc1000000,sequence), + stamp_value, timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?], pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false, register_set, @@ -258,8 +259,8 @@ impl Compute { } self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?; self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?; - Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers,self.profile,self._binding.generation())?; let stamp=self.stamp(); + Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,stamp,self.slot,self.registers,self.profile,self._binding.generation())?; Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot,self.profile,self.sequence - 1)?; self.write_counters()?; let address=self.objects[storage::COMMAND].va()+storage::slot_offset_for(self.profile,storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; @@ -326,7 +327,20 @@ impl Compute { } else { microsequence::HAS_ATTACHMENTS },&[u8::from(attachments.count()!=0)])?; agx_memory::publish();Ok(()) } - fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } + fn stamp(&self) -> u32 { self.stamp_epoch.value(0xc1000000,self.sequence) } + pub(crate) fn stamps_need_reseed(&self)->bool { + self.stamp_epoch.needs_reseed(0xc1000000,self.sequence) + } + /// All queues, including cross-engine consumers, must have passed Runtime's + /// final retirement checks. This preserves the pool and command counters. + pub(crate) fn reseed_stamps(&mut self)->Result { + if !self.complete()? {return Err(EBUSY);} + for owner in [storage::STAMP,storage::FW_STAMP] { + self.objects[owner].u32(0,0xc1000000)?; + } + self.stamp_epoch.reseed(self.sequence); + Ok(()) + } pub(crate) fn completion_dependency(&self)->Result { Ok(crate::m3::m3_sync_layout::CompletionDependency { stamp:queue::FirmwareVa::new(self.objects[storage::FW_STAMP].va()).map_err(|_|EINVAL)?, diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs index 9a9f63d15b784..a3ada4a349dfa 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -28,6 +28,7 @@ pub(crate) struct Render { _binding:mmu::VmBind, stats: [Region;2], profile:super::m3_profile::Profile, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, last_progress_ns:i64, + stamp_epoch:timeline::StampEpoch, draw:u64, heads:[u16;2], cached_views:KVec, slot:usize, batch_count:usize, early_count:u32, /// Measurement packets in the pass most recently appended to its queues. @@ -98,7 +99,7 @@ impl Render { tiler_bytes.resize(if profile==super::m3_profile::Profile::Extended {tiler::EXTENDED_SIZE} else {tiler::SIZE},0u8,GFP_KERNEL)?; let mut fragment_bytes=KVec::new(); fragment_bytes.resize(if profile==super::m3_profile::Profile::Extended {fragment::EXTENDED_SIZE} else {fragment::SIZE},0u8,GFP_KERNEL)?; - let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; + let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,stamp_epoch:timeline::StampEpoch::default(),draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; // Lists and the matching manager are built before InitBM publication. job.initialize_parameter_buffer()?; for (index,pool) in UMA_POOLS.iter().enumerate() { @@ -245,11 +246,11 @@ impl Render { value.encode_extended(&mut init).map_err(|_|EINVAL)?; } else {value.encode(&mut init).map_err(|_|EINVAL)?;} self.init_bm.write(0, &init)?; - let ta_value = timeline::stamp(s::STAMP_TA - 256, draw); - let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw); + let ta_value = self.stamp_epoch.value(s::STAMP_TA - 256,draw); + let fs_value = self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,draw); let previous=predecessor.unwrap_or(sync::CompletionDependency { stamp:if early {ta_stamp} else {fs_stamp}, - value:timeline::stamp((if early {s::STAMP_TA} else {s::STAMP_FRAGMENT})-256,draw-1), + value:self.stamp_epoch.value((if early {s::STAMP_TA} else {s::STAMP_FRAGMENT})-256,draw-1), event:u32::from(!early), }); self.write_barrier(P::TilerToFragment, sync::Barrier { @@ -260,7 +261,7 @@ impl Render { event: previous.event, self_value: ta_value, uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass })?; self.write_barrier(P::FragmentDependency, sync::Barrier { - stamp: fs_stamp, wait_value: timeline::stamp(s::STAMP_FRAGMENT - 256, draw - 1), event: 1, self_value: fs_value, + stamp: fs_stamp, wait_value: self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,draw - 1), event: 1, self_value: fs_value, uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass }) } // Prepare each private command after its retired slot is reset. The @@ -286,7 +287,7 @@ impl Render { scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?},gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?, scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?, - stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], + stamp_value:self.stamp_epoch.value(s::STAMP_TA - 256,draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], user_timestamps:[fw(P::TilerUserStart)?,fw(P::TilerUserEnd)?],preemption:[gpu(P::Preemption0)?,gpu(P::Preemption1)?,gpu(P::Preemption2)?], tilemap:gpu(P::Tilemap)?,tpc:gpu(P::Tpc)?,tpc_bytes:g.tpc_bytes,heap:gpu(P::HeapMetadata)?, scene_user:gpu(P::SceneUser)?,initial_scene_entry,geometry:g,page_count:PB_PAGES, @@ -295,7 +296,7 @@ impl Render { }; if self.profile==super::m3_profile::Profile::Extended { value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).ok_or(EOVERFLOW)?, - peer_stamp:timeline::stamp(s::STAMP_FRAGMENT-256,draw),usc_window:usc, + peer_stamp:self.stamp_epoch.value(s::STAMP_FRAGMENT-256,draw),usc_window:usc, firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[0].manager].va()+0x100).map_err(|_|EINVAL)?, scene_scratch:gpu(P::FragmentScratch)?},&mut self.tiler_bytes).map_err(|_|EINVAL)?; } else {value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?;} @@ -317,7 +318,7 @@ impl Render { queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?, manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?, fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?, - context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw), + context:self._binding.slot(),stamp:self.stamp_epoch.value((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256,draw), }; let timed=self.stage_timestamps[usize::from(fs)]; let length=if self.profile==super::m3_profile::Profile::Extended { @@ -380,7 +381,7 @@ impl Render { gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?, heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?, scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?, - fw_stamp:shared(s::FRAGMENT_FW_STAMP)?,stamp_value:timeline::stamp(s::STAMP_FRAGMENT - 256, draw), + fw_stamp:shared(s::FRAGMENT_FW_STAMP)?,stamp_value:self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,draw), timestamps:[fw(P::FragmentStart)?,fw(P::FragmentEnd)?],user_timestamps:[fw(P::FragmentUserStart)?,fw(P::FragmentUserEnd)?], state:fragment::State { geometry,multisample:r.multisample_control,merge_upper:[r.merge_upper_x,r.merge_upper_y], @@ -482,12 +483,27 @@ impl Render { /// across stamp wrap; the runtime bounds outstanding passes to SLOTS. pub(crate) fn complete_through(&mut self,draw:u64)->Result { if draw==0 {return Ok(true);} - for (stamp,fw,value) in [(s::TA_STAMP,s::TA_FW_STAMP,timeline::stamp(s::STAMP_TA-256,draw)), - (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,timeline::stamp(s::STAMP_FRAGMENT-256,draw))] { + for (stamp,fw,value) in [(s::TA_STAMP,s::TA_FW_STAMP,self.stamp_epoch.value(s::STAMP_TA-256,draw)), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,self.stamp_epoch.value(s::STAMP_FRAGMENT-256,draw))] { if !timeline::reached(self.objects[stamp].read_u32(0)?,value) || !timeline::reached(self.objects[fw].read_u32(0)?,value) {return Ok(false);} } Ok(true) } + pub(crate) fn stamps_need_reseed(&self)->bool { + self.stamp_epoch.needs_reseed(s::STAMP_TA-256,self.draw) + } + /// Runtime calls this only after every active batch (including compute and + /// mixed dependencies) has passed the runtime's final retirement checks. + pub(crate) fn reseed_stamps(&mut self)->Result { + if !self.complete()? {return Err(EBUSY);} + for (host,firmware,seed) in [(s::TA_STAMP,s::TA_FW_STAMP,s::STAMP_TA-256), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::STAMP_FRAGMENT-256)] { + self.objects[host].u32(0,seed)?; + self.objects[firmware].u32(0,seed)?; + } + self.stamp_epoch.reseed(self.draw); + Ok(()) + } /// Whether any draw has been published (the first one initializes the buffer manager). pub(crate) fn started(&self)->bool {self.draw>0} pub(crate) fn first(&self)->bool { @@ -609,7 +625,7 @@ impl Render { pub(crate) fn completion_dependency(&self)->Result { Ok(sync::CompletionDependency { stamp:queue::FirmwareVa::new(self.objects[s::FRAGMENT_FW_STAMP].va()).map_err(|_|EINVAL)?, - value:timeline::stamp(s::STAMP_FRAGMENT-256,self.draw),event:1, + value:self.stamp_epoch.value(s::STAMP_FRAGMENT-256,self.draw),event:1, }) } pub(crate) fn append_after(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64, @@ -735,15 +751,15 @@ impl Render { } /// Read the fragment host and firmware stamps without changing retirement. pub(crate) fn fragment_stamp_parts(&mut self)->Result<[bool;2]> { - let value=timeline::stamp(s::STAMP_FRAGMENT - 256,self.draw); + let value=self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,self.draw); Ok([self.objects[s::FRAGMENT_STAMP].read_u32(0)?==value, self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?==value]) } /// Observe each completion condition across both render queues independently. pub(crate) fn completion_parts(&mut self)->Result<[bool;4]> { let mut done=[true;4]; - for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]), - (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] { + for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,self.stamp_epoch.value(s::STAMP_TA - 256,self.draw),self.heads[0]), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,self.draw),self.heads[1])] { done[0]&=self.objects[stamp].read_u32(0)?==value; done[1]&=self.objects[fw].read_u32(0)?==value; done[2]&=self.objects[pointers].read_u32(queue::DONE)?==u32::from(head); @@ -752,8 +768,8 @@ impl Render { Ok(done) } pub(crate) fn complete(&mut self)->Result { - for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]), - (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] { + for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,self.stamp_epoch.value(s::STAMP_TA - 256,self.draw),self.heads[0]), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,self.draw),self.heads[1])] { if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value || self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head) || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index e34236fd57a3f..762c8bc26e75b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -295,11 +295,37 @@ impl Inner { self.config.pipe_free(0)? && self.config.pipe_free(1)?) {return Ok(None);} Ok(Some(room)) } + /// Stop extending the pipeline before any stamp reaches zero. The final + /// active batch must pass poll_active's stamp/queue/event/pipe checks and + /// configured SoC MMIO checks before either epoch can change. + fn reseed_stamp_epochs(&mut self)->Result { + let reseed=self.jobs.iter().any(|job|match job { + NativeJob::Render(j)=>j.stamps_need_reseed(), + NativeJob::Compute(j)=>j.stamps_need_reseed(), + }); + if !reseed {return Ok(true);} + if !self.active.is_empty() {return Ok(false);} + for job in &mut self.jobs { + match job { + NativeJob::Render(j) if j.stamps_need_reseed()=>j.reseed_stamps()?, + NativeJob::Compute(j) if j.stamps_need_reseed()=>j.reseed_stamps()?, + _=>{}, + } + } + crate::m3::agx_memory::publish(); + crate::mem::sync(); + Ok(true) + } /// Publish pending batches while allowed: one when none is active, more behind it with /// asahi.m3_pipeline_depth > 1. Returns whether anything was published or failed. fn publish_next(&mut self)->bool { let mut progress=false; while !self.pending.is_empty() && self.state.healthy() { + match self.reseed_stamp_epochs() { + Ok(true)=>{}, + Ok(false)=>break, + Err(error)=>{self.fail_active(error);return true;}, + } let overlap=!self.active.is_empty(); let (base,max)=match self.pipeline_slot() { Ok(Some(slot))=>slot, diff --git a/drivers/gpu/drm/asahi/m3/m3_timeline.rs b/drivers/gpu/drm/asahi/m3/m3_timeline.rs index 9a9f4dd58c697..12fbe3750d27f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_timeline.rs +++ b/drivers/gpu/drm/asahi/m3/m3_timeline.rs @@ -6,13 +6,39 @@ //! its distance is well below the half-range. Keep the host command ordinal //! separately in u64 so stamp wrap never re-registers a queue or emits InitBM. -/// Nonzero wire stamp after `ordinal` commands, from the retired seed. -/// As on the M4 runtime, do not publish zero as a completion stamp. The -/// 24-bit stamp sequence skips zero; command ordinals/event counts do not. +/// Wire stamp after `ordinal` commands, from the retired seed. +/// Skipping zero would advance by 0x200 and stop the firmware with +/// "DAG: Non Sequential Stamp Updates". Live queues use `StampEpoch` and +/// reseed while idle before reaching zero, rather than relying on zero stamps. pub(crate) const fn stamp(seed: u32, ordinal: u64) -> u32 { - const PERIOD: u64 = 0x00ff_ffff; - let unit = (((seed >> 8) as u64 - 1 + ordinal % PERIOD) % PERIOD) + 1; - (unit as u32) << 8 + seed.wrapping_add((ordinal as u32).wrapping_mul(0x100)) +} + +/// Leave ample room for a complete bounded batch before the earliest wrap, +/// as the G15 runtime does. Admission stops extending the pipeline at this +/// boundary; reseeding requires the runtime's final batch retirement checks. +const RESEED_MARGIN: u64 = 0x10_0000; + +/// Wire-stamp epoch, independent of monotonic command/pool/notifier ordinals. +#[derive(Default)] +pub(crate) struct StampEpoch { + origin: u64, +} + +impl StampEpoch { + pub(crate) fn value(&self, seed: u32, ordinal: u64) -> u32 { + stamp(seed, ordinal - self.origin) + } + + pub(crate) fn needs_reseed(&self, seed: u32, ordinal: u64) -> bool { + ordinal - self.origin >= (1u64 << 24) - (seed >> 8) as u64 - RESEED_MARGIN + } + + /// Caller has drained all active batches and written both stamp copies + /// back to their seeds. No dependency from the old epoch may remain live. + pub(crate) fn reseed(&mut self, retired: u64) { + self.origin = retired; + } } /// Firmware command count, before the command with this host ordinal. diff --git a/tools/asahi/m3-stamp-admission-tests.py b/tools/asahi/m3-stamp-admission-tests.py new file mode 100644 index 0000000000000..613bbcc8db8f5 --- /dev/null +++ b/tools/asahi/m3-stamp-admission-tests.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production stamp-reseed admission body with retired/live queues. + +Jobs model the completion gate and stamp writes; this is not hardware retirement +qualification. The extracted Runtime method must drain every active batch before +changing either engine's epoch, including cross-engine consumers. +""" +import argparse +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def method(text, marker): + start = text.index(marker) + end = text.index('{', start) + 1 + depth = 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +PREFIX = r''' +#![allow(dead_code)] +type Result = std::result::Result; +mod m3 { pub mod agx_memory { pub fn publish() {} } } +mod mem { pub fn sync() {} } +struct Job { due: bool, complete: bool, writes: usize } +impl Job { + fn stamps_need_reseed(&self) -> bool { self.due } + fn reseed_stamps(&mut self) -> Result { + if !self.complete { return Err(-16); } + self.writes += 1; self.due = false; Ok(()) + } +} +enum NativeJob { Render(Job), Compute(Job) } +struct Inner { jobs: Vec, active: Vec<()> } +impl Inner { +''' +TESTS = r''' +} +fn main() { + for render_due in [false, true] { for compute_due in [false, true] { + // Even a due, fully completed producer cannot be reseeded while an + // active batch could still reference it as a cross-engine predecessor. + for active in 0..=3 { + let make = |due| Job { due, complete: true, writes: 0 }; + let mut inner = Inner { + jobs: vec![NativeJob::Render(make(render_due)), NativeJob::Compute(make(compute_due))], + active: vec![(); active], + }; + let due = render_due || compute_due; + assert_eq!(inner.reseed_stamp_epochs(), Ok(!due || active == 0)); + for (job, was_due) in inner.jobs.iter().zip([render_due, compute_due]) { + let j = match job { NativeJob::Render(j) | NativeJob::Compute(j) => j }; + assert_eq!(j.writes, usize::from(was_due && active == 0)); + assert_eq!(j.due, was_due && active != 0); + } + // Draining the final active batch permits exactly one reset per + // due engine; repeated admission must not reset it again. + inner.active.clear(); + assert_eq!(inner.reseed_stamp_epochs(), Ok(true)); + assert_eq!(inner.reseed_stamp_epochs(), Ok(true)); + for (job, was_due) in inner.jobs.iter().zip([render_due, compute_due]) { + let j = match job { NativeJob::Render(j) | NativeJob::Compute(j) => j }; + assert_eq!(j.writes, usize::from(was_due)); + } + } + }} + // An unexpected incomplete queue must propagate failure, preventing publication. + for render in [false, true] { + let j = Job { due: true, complete: false, writes: 0 }; + let mut inner = Inner { jobs: vec![if render { NativeJob::Render(j) } else { NativeJob::Compute(j) }], active: vec![] }; + assert_eq!(inner.reseed_stamp_epochs(), Err(-16)); + } +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--out', type=Path) + parser.add_argument('--mutation', choices=['live-reseed']) + args = parser.parse_args() + runtime = (ROOT / 'drivers/gpu/drm/asahi/m3/m3_runtime.rs').read_text() + body = method(runtime, 'fn reseed_stamp_epochs(') + if args.mutation: + gate = 'if !self.active.is_empty() {return Ok(false);}' + assert gate in body + body = body.replace(gate, '') + out = args.out or Path(tempfile.mkdtemp(prefix='m3-stamp-admission-')) + out.mkdir(parents=True, exist_ok=True) + source = out / 'admission.rs' + source.write_text(PREFIX + body + TESTS) + binary = out / 'admission' + subprocess.run(['rustc', '--edition=2021', str(source), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)]) + if result.returncode: + # Preserve the Rust assertion's exit and diagnostic. A Python traceback + # means a broken harness to run-host-tests, not a killed mutation. + raise SystemExit(result.returncode) + print('PASS: production stamp epoch admission, pipeline drain, mixed engines and completion errors') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/m3-stamp-tests.rs b/tools/asahi/m3-stamp-tests.rs new file mode 100644 index 0000000000000..4fd617d0eaae9 --- /dev/null +++ b/tools/asahi/m3-stamp-tests.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Firmware stamp progression and idle epoch boundaries (#1935). +//! Run: rustc --edition=2021 --test tools/asahi/m3-stamp-tests.rs -o /tmp/m3-stamp-tests +#![allow(dead_code)] +#[path = "../../drivers/gpu/drm/asahi/m3/m3_timeline.rs"] +mod timeline; + +const TA: u32 = 0x7a00_0000; +const FS: u32 = 0x3d00_0000; +const COMPUTE: u32 = 0xc100_0000; + +#[test] +fn reported_fragment_rollovers_must_not_skip_zero() { + // Independent values from both reports: the old period (0xffffff) + // produces the same 0xffffff00 -> 0x100 jump at each of these draws. + let old = |n: u64| (((((FS >> 8) as u64 - 1 + n % 0xffffff) % 0xffffff) + 1) as u32) << 8; + for n in [12_779_520, 29_556_735] { + assert_eq!(old(n - 2), 0xffff_fe00); + assert_eq!(old(n - 1), 0xffff_ff00); + assert_eq!(old(n), 0x100); + assert_eq!(old(n).wrapping_sub(old(n - 1)), 0x200); + } + // Wire arithmetic itself is modulo 2^32. The runtime avoids zero by + // reseeding at idle, rather than removing a step from that sequence. + assert_eq!(timeline::stamp(FS, 12_779_519), 0xffff_ff00); + assert_eq!(timeline::stamp(FS, 12_779_520), 0); + assert_eq!(timeline::stamp(FS, 12_779_521), 0x100); + for seed in [TA, FS, COMPUTE] { + let wrap = (1u64 << 24) - (seed >> 8) as u64; + for n in wrap - 16..=wrap + 16 { + assert_eq!(timeline::stamp(seed, n).wrapping_sub(timeline::stamp(seed, n - 1)), 0x100); + } + } +} + +#[test] +fn reseed_boundary_leaves_room_for_maximum_batch_and_draining_pipeline() { + for seed in [TA, COMPUTE] { + let mut epoch = timeline::StampEpoch::default(); + // Render reseeds before its earlier TA wrap, so FS cannot reach its wrap. + let boundary = if seed == TA { 0x76_0000 } else { 0x2f_0000 }; + assert!(!epoch.needs_reseed(seed, boundary - 1)); + assert!(epoch.needs_reseed(seed, boundary)); + // A publication can cross admission's boundary by at most one batch. + // Existing batches then drain without any further publication. + for passes in 1..=16 { + let retired = boundary - 1 + passes; + assert!(epoch.needs_reseed(seed, retired)); + assert_ne!(epoch.value(seed, retired), 0); + if seed == TA { assert_ne!(epoch.value(FS, retired), 0); } + } + let retired = boundary + 15; + let counters = (timeline::previous(retired + 1), timeline::events(retired + 1, 2)); + epoch.reseed(retired); + assert!(!epoch.needs_reseed(seed, retired)); + assert_eq!(epoch.value(seed, retired), seed); + assert_eq!(epoch.value(seed, retired + 1), seed + 0x100); + if seed == TA { + // The next TA/FS barriers wait on exactly the newly written seeds. + assert_eq!(epoch.value(FS, retired), FS); + assert_eq!(epoch.value(FS, retired + 1), FS + 0x100); + } + assert_eq!((timeline::previous(retired + 1), timeline::events(retired + 1, 2)), counters); + } +} + +#[test] +fn repeated_epochs_keep_host_ordinals_monotonic_past_u32() { + let mut epoch = timeline::StampEpoch::default(); + let mut retired = 0; + for _ in 0..1024 { + retired += 0x76_000f; + assert!(epoch.needs_reseed(TA, retired)); + epoch.reseed(retired); + for n in 1..=16 { + assert_eq!(epoch.value(TA, retired + n), TA + n as u32 * 0x100); + assert_eq!(epoch.value(FS, retired + n), FS + n as u32 * 0x100); + } + } + assert!(retired > u32::MAX as u64); + assert_eq!(timeline::previous(retired + 1), retired as u32); + assert_eq!(timeline::events(retired + 1, 2), ((retired + 1) as u32).wrapping_mul(2)); +} diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 0cfb923dbc72a..0ebf578cbd68e 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -54,6 +54,8 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'm3-stamp-admission-tests.py': [(out(), PASS), + (out() + ['--mutation', 'live-reseed'], KILL)], 'hardware-defaults-tests.py': [([], PASS)], 'j613-mtp-s2idle-tests.py': [([], PASS)], # 11eccbedc3ed writes frames without a transport-wide lock. @@ -267,7 +269,7 @@ def mutations(flag, names, base, expect=KILL): } # Rust unit-test files built with `rustc --test`, and shell drivers. -ASAHI_RUST = ['m3-batching-tests.rs', 'm3-runtime-tests.rs', 'pmp-bootargs-tests.rs', 'pmp-power-tests.rs', +ASAHI_RUST = ['m3-batching-tests.rs', 'm3-stamp-tests.rs', 'm3-runtime-tests.rs', 'pmp-bootargs-tests.rs', 'pmp-power-tests.rs', 'usage-tests.rs'] ASAHI_SHELL = ['g17-ray-query-timeout-tests.sh', 'g17-traversal-admission-tests.sh']