From 188b146eb498d8f55bc362f82153f1725377072b Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Sat, 14 Nov 2026 16:46:54 -0500 Subject: [PATCH] drm/asahi: m3: reseed completion stamps before rollover The M3 stamp helper skips zero, so 0xffffff00 advances to 0x100 rather than by 0x100. The two J613 failures reported by @yuffington in #1935 occur at draws 12779520 and 29556735: exactly successive fragment-stamp rollovers with the old 0xffffff period. Both report the same firmware "DAG: Non Sequential Stamp Updates" error. Use modulo-32-bit stamp arithmetic and avoid reaching zero in live queues. Give render and compute independent stamp epochs. Stop extending the pipeline before the earliest wrap, drain every active batch through the existing retirement checks, then reset host and firmware stamp copies to their seeds. Cross-engine dependencies cannot survive that drain. Preserve command ordinals, pool/notifier counts, queue cursors and InitBM/registration state across reseeds. Cover the reported counts, repeated epochs, boundary-crossing batches, production admission and a live-reseed negative control. Hardware qualification of idle reseeding on the affected M3 Air remains pending. Link: https://github.com/aurora-silicon/linux/issues/1935 Signed-off-by: Diljit Singh Assisted-by: LLM --- drivers/gpu/drm/asahi/m3/m3_compute.rs | 26 ++++-- drivers/gpu/drm/asahi/m3/m3_render.rs | 50 +++++++---- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 26 ++++++ drivers/gpu/drm/asahi/m3/m3_timeline.rs | 38 ++++++-- tools/asahi/m3-stamp-admission-tests.py | 110 ++++++++++++++++++++++++ tools/asahi/m3-stamp-tests.rs | 83 ++++++++++++++++++ tools/asahi/run-host-tests.py | 4 +- 7 files changed, 307 insertions(+), 30 deletions(-) create mode 100644 tools/asahi/m3-stamp-admission-tests.py create mode 100644 tools/asahi/m3-stamp-tests.rs diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index b6e9444ed5449..d49a311c506c2 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -29,6 +29,7 @@ pub(crate) struct Compute { sequence_bytes: KVec, _binding: mmu::VmBind, stats: Region, + stamp_epoch: timeline::StampEpoch, sequence: u64, last_progress_ns: i64, head: u16, @@ -71,7 +72,7 @@ impl Compute { if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);} objects.push(b,GFP_KERNEL)?; } - Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, + Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, timeline::stamp(0xc1000000,1), 0, registers, profile, binding.generation())?; Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0, profile, 0)?; if profile == super::m3_profile::Profile::Extended { @@ -118,10 +119,10 @@ impl Compute { // the invalidation. crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; - Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi}) + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,stamp_epoch:timeline::StampEpoch::default(),sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi}) } fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, - control: crate::m3::agx_compute::Control, sequence: u64, slot: usize, + control: crate::m3::agx_compute::Control, sequence: u64, stamp_value: u32, slot: usize, register_set: command::RegisterSet, profile: super::m3_profile::Profile, generation: u8) -> Result { let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); @@ -136,7 +137,7 @@ impl Compute { if profile == super::m3_profile::Profile::Extended { microsequence::Sequence::EXTENDED_LENGTH as u32 } else { microsequence::LENGTH as u32 }, stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?, - stamp_value: timeline::stamp(0xc1000000,sequence), + stamp_value, timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?], pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false, register_set, @@ -258,8 +259,8 @@ impl Compute { } self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?; self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?; - Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers,self.profile,self._binding.generation())?; let stamp=self.stamp(); + Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,stamp,self.slot,self.registers,self.profile,self._binding.generation())?; Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot,self.profile,self.sequence - 1)?; self.write_counters()?; let address=self.objects[storage::COMMAND].va()+storage::slot_offset_for(self.profile,storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; @@ -326,7 +327,20 @@ impl Compute { } else { microsequence::HAS_ATTACHMENTS },&[u8::from(attachments.count()!=0)])?; agx_memory::publish();Ok(()) } - fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } + fn stamp(&self) -> u32 { self.stamp_epoch.value(0xc1000000,self.sequence) } + pub(crate) fn stamps_need_reseed(&self)->bool { + self.stamp_epoch.needs_reseed(0xc1000000,self.sequence) + } + /// All queues, including cross-engine consumers, must have passed Runtime's + /// final retirement checks. This preserves the pool and command counters. + pub(crate) fn reseed_stamps(&mut self)->Result { + if !self.complete()? {return Err(EBUSY);} + for owner in [storage::STAMP,storage::FW_STAMP] { + self.objects[owner].u32(0,0xc1000000)?; + } + self.stamp_epoch.reseed(self.sequence); + Ok(()) + } pub(crate) fn completion_dependency(&self)->Result { Ok(crate::m3::m3_sync_layout::CompletionDependency { stamp:queue::FirmwareVa::new(self.objects[storage::FW_STAMP].va()).map_err(|_|EINVAL)?, diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs index 9a9f63d15b784..a3ada4a349dfa 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -28,6 +28,7 @@ pub(crate) struct Render { _binding:mmu::VmBind, stats: [Region;2], profile:super::m3_profile::Profile, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, last_progress_ns:i64, + stamp_epoch:timeline::StampEpoch, draw:u64, heads:[u16;2], cached_views:KVec, slot:usize, batch_count:usize, early_count:u32, /// Measurement packets in the pass most recently appended to its queues. @@ -98,7 +99,7 @@ impl Render { tiler_bytes.resize(if profile==super::m3_profile::Profile::Extended {tiler::EXTENDED_SIZE} else {tiler::SIZE},0u8,GFP_KERNEL)?; let mut fragment_bytes=KVec::new(); fragment_bytes.resize(if profile==super::m3_profile::Profile::Extended {fragment::EXTENDED_SIZE} else {fragment::SIZE},0u8,GFP_KERNEL)?; - let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; + let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,stamp_epoch:timeline::StampEpoch::default(),draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; // Lists and the matching manager are built before InitBM publication. job.initialize_parameter_buffer()?; for (index,pool) in UMA_POOLS.iter().enumerate() { @@ -245,11 +246,11 @@ impl Render { value.encode_extended(&mut init).map_err(|_|EINVAL)?; } else {value.encode(&mut init).map_err(|_|EINVAL)?;} self.init_bm.write(0, &init)?; - let ta_value = timeline::stamp(s::STAMP_TA - 256, draw); - let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw); + let ta_value = self.stamp_epoch.value(s::STAMP_TA - 256,draw); + let fs_value = self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,draw); let previous=predecessor.unwrap_or(sync::CompletionDependency { stamp:if early {ta_stamp} else {fs_stamp}, - value:timeline::stamp((if early {s::STAMP_TA} else {s::STAMP_FRAGMENT})-256,draw-1), + value:self.stamp_epoch.value((if early {s::STAMP_TA} else {s::STAMP_FRAGMENT})-256,draw-1), event:u32::from(!early), }); self.write_barrier(P::TilerToFragment, sync::Barrier { @@ -260,7 +261,7 @@ impl Render { event: previous.event, self_value: ta_value, uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass })?; self.write_barrier(P::FragmentDependency, sync::Barrier { - stamp: fs_stamp, wait_value: timeline::stamp(s::STAMP_FRAGMENT - 256, draw - 1), event: 1, self_value: fs_value, + stamp: fs_stamp, wait_value: self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,draw - 1), event: 1, self_value: fs_value, uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass }) } // Prepare each private command after its retired slot is reset. The @@ -286,7 +287,7 @@ impl Render { scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?},gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?, scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?, - stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], + stamp_value:self.stamp_epoch.value(s::STAMP_TA - 256,draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], user_timestamps:[fw(P::TilerUserStart)?,fw(P::TilerUserEnd)?],preemption:[gpu(P::Preemption0)?,gpu(P::Preemption1)?,gpu(P::Preemption2)?], tilemap:gpu(P::Tilemap)?,tpc:gpu(P::Tpc)?,tpc_bytes:g.tpc_bytes,heap:gpu(P::HeapMetadata)?, scene_user:gpu(P::SceneUser)?,initial_scene_entry,geometry:g,page_count:PB_PAGES, @@ -295,7 +296,7 @@ impl Render { }; if self.profile==super::m3_profile::Profile::Extended { value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).ok_or(EOVERFLOW)?, - peer_stamp:timeline::stamp(s::STAMP_FRAGMENT-256,draw),usc_window:usc, + peer_stamp:self.stamp_epoch.value(s::STAMP_FRAGMENT-256,draw),usc_window:usc, firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[0].manager].va()+0x100).map_err(|_|EINVAL)?, scene_scratch:gpu(P::FragmentScratch)?},&mut self.tiler_bytes).map_err(|_|EINVAL)?; } else {value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?;} @@ -317,7 +318,7 @@ impl Render { queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?, manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?, fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?, - context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw), + context:self._binding.slot(),stamp:self.stamp_epoch.value((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256,draw), }; let timed=self.stage_timestamps[usize::from(fs)]; let length=if self.profile==super::m3_profile::Profile::Extended { @@ -380,7 +381,7 @@ impl Render { gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?, heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?, scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?, - fw_stamp:shared(s::FRAGMENT_FW_STAMP)?,stamp_value:timeline::stamp(s::STAMP_FRAGMENT - 256, draw), + fw_stamp:shared(s::FRAGMENT_FW_STAMP)?,stamp_value:self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,draw), timestamps:[fw(P::FragmentStart)?,fw(P::FragmentEnd)?],user_timestamps:[fw(P::FragmentUserStart)?,fw(P::FragmentUserEnd)?], state:fragment::State { geometry,multisample:r.multisample_control,merge_upper:[r.merge_upper_x,r.merge_upper_y], @@ -482,12 +483,27 @@ impl Render { /// across stamp wrap; the runtime bounds outstanding passes to SLOTS. pub(crate) fn complete_through(&mut self,draw:u64)->Result { if draw==0 {return Ok(true);} - for (stamp,fw,value) in [(s::TA_STAMP,s::TA_FW_STAMP,timeline::stamp(s::STAMP_TA-256,draw)), - (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,timeline::stamp(s::STAMP_FRAGMENT-256,draw))] { + for (stamp,fw,value) in [(s::TA_STAMP,s::TA_FW_STAMP,self.stamp_epoch.value(s::STAMP_TA-256,draw)), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,self.stamp_epoch.value(s::STAMP_FRAGMENT-256,draw))] { if !timeline::reached(self.objects[stamp].read_u32(0)?,value) || !timeline::reached(self.objects[fw].read_u32(0)?,value) {return Ok(false);} } Ok(true) } + pub(crate) fn stamps_need_reseed(&self)->bool { + self.stamp_epoch.needs_reseed(s::STAMP_TA-256,self.draw) + } + /// Runtime calls this only after every active batch (including compute and + /// mixed dependencies) has passed the runtime's final retirement checks. + pub(crate) fn reseed_stamps(&mut self)->Result { + if !self.complete()? {return Err(EBUSY);} + for (host,firmware,seed) in [(s::TA_STAMP,s::TA_FW_STAMP,s::STAMP_TA-256), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::STAMP_FRAGMENT-256)] { + self.objects[host].u32(0,seed)?; + self.objects[firmware].u32(0,seed)?; + } + self.stamp_epoch.reseed(self.draw); + Ok(()) + } /// Whether any draw has been published (the first one initializes the buffer manager). pub(crate) fn started(&self)->bool {self.draw>0} pub(crate) fn first(&self)->bool { @@ -609,7 +625,7 @@ impl Render { pub(crate) fn completion_dependency(&self)->Result { Ok(sync::CompletionDependency { stamp:queue::FirmwareVa::new(self.objects[s::FRAGMENT_FW_STAMP].va()).map_err(|_|EINVAL)?, - value:timeline::stamp(s::STAMP_FRAGMENT-256,self.draw),event:1, + value:self.stamp_epoch.value(s::STAMP_FRAGMENT-256,self.draw),event:1, }) } pub(crate) fn append_after(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64, @@ -735,15 +751,15 @@ impl Render { } /// Read the fragment host and firmware stamps without changing retirement. pub(crate) fn fragment_stamp_parts(&mut self)->Result<[bool;2]> { - let value=timeline::stamp(s::STAMP_FRAGMENT - 256,self.draw); + let value=self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,self.draw); Ok([self.objects[s::FRAGMENT_STAMP].read_u32(0)?==value, self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?==value]) } /// Observe each completion condition across both render queues independently. pub(crate) fn completion_parts(&mut self)->Result<[bool;4]> { let mut done=[true;4]; - for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]), - (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] { + for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,self.stamp_epoch.value(s::STAMP_TA - 256,self.draw),self.heads[0]), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,self.draw),self.heads[1])] { done[0]&=self.objects[stamp].read_u32(0)?==value; done[1]&=self.objects[fw].read_u32(0)?==value; done[2]&=self.objects[pointers].read_u32(queue::DONE)?==u32::from(head); @@ -752,8 +768,8 @@ impl Render { Ok(done) } pub(crate) fn complete(&mut self)->Result { - for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]), - (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] { + for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,self.stamp_epoch.value(s::STAMP_TA - 256,self.draw),self.heads[0]), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,self.draw),self.heads[1])] { if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value || self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head) || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index e34236fd57a3f..762c8bc26e75b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -295,11 +295,37 @@ impl Inner { self.config.pipe_free(0)? && self.config.pipe_free(1)?) {return Ok(None);} Ok(Some(room)) } + /// Stop extending the pipeline before any stamp reaches zero. The final + /// active batch must pass poll_active's stamp/queue/event/pipe checks and + /// configured SoC MMIO checks before either epoch can change. + fn reseed_stamp_epochs(&mut self)->Result { + let reseed=self.jobs.iter().any(|job|match job { + NativeJob::Render(j)=>j.stamps_need_reseed(), + NativeJob::Compute(j)=>j.stamps_need_reseed(), + }); + if !reseed {return Ok(true);} + if !self.active.is_empty() {return Ok(false);} + for job in &mut self.jobs { + match job { + NativeJob::Render(j) if j.stamps_need_reseed()=>j.reseed_stamps()?, + NativeJob::Compute(j) if j.stamps_need_reseed()=>j.reseed_stamps()?, + _=>{}, + } + } + crate::m3::agx_memory::publish(); + crate::mem::sync(); + Ok(true) + } /// Publish pending batches while allowed: one when none is active, more behind it with /// asahi.m3_pipeline_depth > 1. Returns whether anything was published or failed. fn publish_next(&mut self)->bool { let mut progress=false; while !self.pending.is_empty() && self.state.healthy() { + match self.reseed_stamp_epochs() { + Ok(true)=>{}, + Ok(false)=>break, + Err(error)=>{self.fail_active(error);return true;}, + } let overlap=!self.active.is_empty(); let (base,max)=match self.pipeline_slot() { Ok(Some(slot))=>slot, diff --git a/drivers/gpu/drm/asahi/m3/m3_timeline.rs b/drivers/gpu/drm/asahi/m3/m3_timeline.rs index 9a9f4dd58c697..12fbe3750d27f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_timeline.rs +++ b/drivers/gpu/drm/asahi/m3/m3_timeline.rs @@ -6,13 +6,39 @@ //! its distance is well below the half-range. Keep the host command ordinal //! separately in u64 so stamp wrap never re-registers a queue or emits InitBM. -/// Nonzero wire stamp after `ordinal` commands, from the retired seed. -/// As on the M4 runtime, do not publish zero as a completion stamp. The -/// 24-bit stamp sequence skips zero; command ordinals/event counts do not. +/// Wire stamp after `ordinal` commands, from the retired seed. +/// Skipping zero would advance by 0x200 and stop the firmware with +/// "DAG: Non Sequential Stamp Updates". Live queues use `StampEpoch` and +/// reseed while idle before reaching zero, rather than relying on zero stamps. pub(crate) const fn stamp(seed: u32, ordinal: u64) -> u32 { - const PERIOD: u64 = 0x00ff_ffff; - let unit = (((seed >> 8) as u64 - 1 + ordinal % PERIOD) % PERIOD) + 1; - (unit as u32) << 8 + seed.wrapping_add((ordinal as u32).wrapping_mul(0x100)) +} + +/// Leave ample room for a complete bounded batch before the earliest wrap, +/// as the G15 runtime does. Admission stops extending the pipeline at this +/// boundary; reseeding requires the runtime's final batch retirement checks. +const RESEED_MARGIN: u64 = 0x10_0000; + +/// Wire-stamp epoch, independent of monotonic command/pool/notifier ordinals. +#[derive(Default)] +pub(crate) struct StampEpoch { + origin: u64, +} + +impl StampEpoch { + pub(crate) fn value(&self, seed: u32, ordinal: u64) -> u32 { + stamp(seed, ordinal - self.origin) + } + + pub(crate) fn needs_reseed(&self, seed: u32, ordinal: u64) -> bool { + ordinal - self.origin >= (1u64 << 24) - (seed >> 8) as u64 - RESEED_MARGIN + } + + /// Caller has drained all active batches and written both stamp copies + /// back to their seeds. No dependency from the old epoch may remain live. + pub(crate) fn reseed(&mut self, retired: u64) { + self.origin = retired; + } } /// Firmware command count, before the command with this host ordinal. diff --git a/tools/asahi/m3-stamp-admission-tests.py b/tools/asahi/m3-stamp-admission-tests.py new file mode 100644 index 0000000000000..613bbcc8db8f5 --- /dev/null +++ b/tools/asahi/m3-stamp-admission-tests.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production stamp-reseed admission body with retired/live queues. + +Jobs model the completion gate and stamp writes; this is not hardware retirement +qualification. The extracted Runtime method must drain every active batch before +changing either engine's epoch, including cross-engine consumers. +""" +import argparse +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def method(text, marker): + start = text.index(marker) + end = text.index('{', start) + 1 + depth = 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +PREFIX = r''' +#![allow(dead_code)] +type Result = std::result::Result; +mod m3 { pub mod agx_memory { pub fn publish() {} } } +mod mem { pub fn sync() {} } +struct Job { due: bool, complete: bool, writes: usize } +impl Job { + fn stamps_need_reseed(&self) -> bool { self.due } + fn reseed_stamps(&mut self) -> Result { + if !self.complete { return Err(-16); } + self.writes += 1; self.due = false; Ok(()) + } +} +enum NativeJob { Render(Job), Compute(Job) } +struct Inner { jobs: Vec, active: Vec<()> } +impl Inner { +''' +TESTS = r''' +} +fn main() { + for render_due in [false, true] { for compute_due in [false, true] { + // Even a due, fully completed producer cannot be reseeded while an + // active batch could still reference it as a cross-engine predecessor. + for active in 0..=3 { + let make = |due| Job { due, complete: true, writes: 0 }; + let mut inner = Inner { + jobs: vec![NativeJob::Render(make(render_due)), NativeJob::Compute(make(compute_due))], + active: vec![(); active], + }; + let due = render_due || compute_due; + assert_eq!(inner.reseed_stamp_epochs(), Ok(!due || active == 0)); + for (job, was_due) in inner.jobs.iter().zip([render_due, compute_due]) { + let j = match job { NativeJob::Render(j) | NativeJob::Compute(j) => j }; + assert_eq!(j.writes, usize::from(was_due && active == 0)); + assert_eq!(j.due, was_due && active != 0); + } + // Draining the final active batch permits exactly one reset per + // due engine; repeated admission must not reset it again. + inner.active.clear(); + assert_eq!(inner.reseed_stamp_epochs(), Ok(true)); + assert_eq!(inner.reseed_stamp_epochs(), Ok(true)); + for (job, was_due) in inner.jobs.iter().zip([render_due, compute_due]) { + let j = match job { NativeJob::Render(j) | NativeJob::Compute(j) => j }; + assert_eq!(j.writes, usize::from(was_due)); + } + } + }} + // An unexpected incomplete queue must propagate failure, preventing publication. + for render in [false, true] { + let j = Job { due: true, complete: false, writes: 0 }; + let mut inner = Inner { jobs: vec![if render { NativeJob::Render(j) } else { NativeJob::Compute(j) }], active: vec![] }; + assert_eq!(inner.reseed_stamp_epochs(), Err(-16)); + } +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--out', type=Path) + parser.add_argument('--mutation', choices=['live-reseed']) + args = parser.parse_args() + runtime = (ROOT / 'drivers/gpu/drm/asahi/m3/m3_runtime.rs').read_text() + body = method(runtime, 'fn reseed_stamp_epochs(') + if args.mutation: + gate = 'if !self.active.is_empty() {return Ok(false);}' + assert gate in body + body = body.replace(gate, '') + out = args.out or Path(tempfile.mkdtemp(prefix='m3-stamp-admission-')) + out.mkdir(parents=True, exist_ok=True) + source = out / 'admission.rs' + source.write_text(PREFIX + body + TESTS) + binary = out / 'admission' + subprocess.run(['rustc', '--edition=2021', str(source), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)]) + if result.returncode: + # Preserve the Rust assertion's exit and diagnostic. A Python traceback + # means a broken harness to run-host-tests, not a killed mutation. + raise SystemExit(result.returncode) + print('PASS: production stamp epoch admission, pipeline drain, mixed engines and completion errors') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/m3-stamp-tests.rs b/tools/asahi/m3-stamp-tests.rs new file mode 100644 index 0000000000000..4fd617d0eaae9 --- /dev/null +++ b/tools/asahi/m3-stamp-tests.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Firmware stamp progression and idle epoch boundaries (#1935). +//! Run: rustc --edition=2021 --test tools/asahi/m3-stamp-tests.rs -o /tmp/m3-stamp-tests +#![allow(dead_code)] +#[path = "../../drivers/gpu/drm/asahi/m3/m3_timeline.rs"] +mod timeline; + +const TA: u32 = 0x7a00_0000; +const FS: u32 = 0x3d00_0000; +const COMPUTE: u32 = 0xc100_0000; + +#[test] +fn reported_fragment_rollovers_must_not_skip_zero() { + // Independent values from both reports: the old period (0xffffff) + // produces the same 0xffffff00 -> 0x100 jump at each of these draws. + let old = |n: u64| (((((FS >> 8) as u64 - 1 + n % 0xffffff) % 0xffffff) + 1) as u32) << 8; + for n in [12_779_520, 29_556_735] { + assert_eq!(old(n - 2), 0xffff_fe00); + assert_eq!(old(n - 1), 0xffff_ff00); + assert_eq!(old(n), 0x100); + assert_eq!(old(n).wrapping_sub(old(n - 1)), 0x200); + } + // Wire arithmetic itself is modulo 2^32. The runtime avoids zero by + // reseeding at idle, rather than removing a step from that sequence. + assert_eq!(timeline::stamp(FS, 12_779_519), 0xffff_ff00); + assert_eq!(timeline::stamp(FS, 12_779_520), 0); + assert_eq!(timeline::stamp(FS, 12_779_521), 0x100); + for seed in [TA, FS, COMPUTE] { + let wrap = (1u64 << 24) - (seed >> 8) as u64; + for n in wrap - 16..=wrap + 16 { + assert_eq!(timeline::stamp(seed, n).wrapping_sub(timeline::stamp(seed, n - 1)), 0x100); + } + } +} + +#[test] +fn reseed_boundary_leaves_room_for_maximum_batch_and_draining_pipeline() { + for seed in [TA, COMPUTE] { + let mut epoch = timeline::StampEpoch::default(); + // Render reseeds before its earlier TA wrap, so FS cannot reach its wrap. + let boundary = if seed == TA { 0x76_0000 } else { 0x2f_0000 }; + assert!(!epoch.needs_reseed(seed, boundary - 1)); + assert!(epoch.needs_reseed(seed, boundary)); + // A publication can cross admission's boundary by at most one batch. + // Existing batches then drain without any further publication. + for passes in 1..=16 { + let retired = boundary - 1 + passes; + assert!(epoch.needs_reseed(seed, retired)); + assert_ne!(epoch.value(seed, retired), 0); + if seed == TA { assert_ne!(epoch.value(FS, retired), 0); } + } + let retired = boundary + 15; + let counters = (timeline::previous(retired + 1), timeline::events(retired + 1, 2)); + epoch.reseed(retired); + assert!(!epoch.needs_reseed(seed, retired)); + assert_eq!(epoch.value(seed, retired), seed); + assert_eq!(epoch.value(seed, retired + 1), seed + 0x100); + if seed == TA { + // The next TA/FS barriers wait on exactly the newly written seeds. + assert_eq!(epoch.value(FS, retired), FS); + assert_eq!(epoch.value(FS, retired + 1), FS + 0x100); + } + assert_eq!((timeline::previous(retired + 1), timeline::events(retired + 1, 2)), counters); + } +} + +#[test] +fn repeated_epochs_keep_host_ordinals_monotonic_past_u32() { + let mut epoch = timeline::StampEpoch::default(); + let mut retired = 0; + for _ in 0..1024 { + retired += 0x76_000f; + assert!(epoch.needs_reseed(TA, retired)); + epoch.reseed(retired); + for n in 1..=16 { + assert_eq!(epoch.value(TA, retired + n), TA + n as u32 * 0x100); + assert_eq!(epoch.value(FS, retired + n), FS + n as u32 * 0x100); + } + } + assert!(retired > u32::MAX as u64); + assert_eq!(timeline::previous(retired + 1), retired as u32); + assert_eq!(timeline::events(retired + 1, 2), ((retired + 1) as u32).wrapping_mul(2)); +} diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index a01579d348f6b..55bd875f8a36f 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -54,6 +54,8 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'm3-stamp-admission-tests.py': [(out(), PASS), + (out() + ['--mutation', 'live-reseed'], KILL)], 'j613-mtp-s2idle-tests.py': [([], PASS)], # 11eccbedc3ed writes frames without a transport-wide lock. 'mtp-tx-frame-tests.py': [(out(), PASS), @@ -237,7 +239,7 @@ def mutations(flag, names, base, expect=KILL): } # Rust unit-test files built with `rustc --test`, and shell drivers. -ASAHI_RUST = ['m3-batching-tests.rs', 'm3-runtime-tests.rs', 'pmp-bootargs-tests.rs', 'pmp-power-tests.rs', +ASAHI_RUST = ['m3-batching-tests.rs', 'm3-stamp-tests.rs', 'm3-runtime-tests.rs', 'pmp-bootargs-tests.rs', 'pmp-power-tests.rs', 'usage-tests.rs'] ASAHI_SHELL = ['g17-ray-query-timeout-tests.sh', 'g17-traversal-admission-tests.sh']