Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions drivers/gpu/drm/asahi/m3/m3_compute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ pub(crate) struct Compute {
sequence_bytes: KVec<u8>,
_binding: mmu::VmBind,
stats: Region,
stamp_epoch: timeline::StampEpoch,
sequence: u64,
last_progress_ns: i64,
head: u16,
Expand Down Expand Up @@ -71,7 +72,7 @@ impl Compute {
if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);}
objects.push(b,GFP_KERNEL)?;
}
Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0,
Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, timeline::stamp(0xc1000000,1), 0,
registers, profile, binding.generation())?;
Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0, profile, 0)?;
if profile == super::m3_profile::Profile::Extended {
Expand Down Expand Up @@ -118,10 +119,10 @@ impl Compute {
// the invalidation.
crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();
unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))};
Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi})
Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,stamp_epoch:timeline::StampEpoch::default(),sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi})
}
fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32,
control: crate::m3::agx_compute::Control, sequence: u64, slot: usize,
control: crate::m3::agx_compute::Control, sequence: u64, stamp_value: u32, slot: usize,
register_set: command::RegisterSet, profile: super::m3_profile::Profile, generation: u8) -> Result {
let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL);
let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL);
Expand All @@ -136,7 +137,7 @@ impl Compute {
if profile == super::m3_profile::Profile::Extended { microsequence::Sequence::EXTENDED_LENGTH as u32 }
else { microsequence::LENGTH as u32 },
stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?,
stamp_value: timeline::stamp(0xc1000000,sequence),
stamp_value,
timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?],
pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false,
register_set,
Expand Down Expand Up @@ -258,8 +259,8 @@ impl Compute {
}
self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?;
self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?;
Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers,self.profile,self._binding.generation())?;
let stamp=self.stamp();
Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,stamp,self.slot,self.registers,self.profile,self._binding.generation())?;
Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot,self.profile,self.sequence - 1)?;
self.write_counters()?;
let address=self.objects[storage::COMMAND].va()+storage::slot_offset_for(self.profile,storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64;
Expand Down Expand Up @@ -326,7 +327,20 @@ impl Compute {
} else { microsequence::HAS_ATTACHMENTS },&[u8::from(attachments.count()!=0)])?;
agx_memory::publish();Ok(())
}
fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) }
fn stamp(&self) -> u32 { self.stamp_epoch.value(0xc1000000,self.sequence) }
pub(crate) fn stamps_need_reseed(&self)->bool {
self.stamp_epoch.needs_reseed(0xc1000000,self.sequence)
}
/// All queues, including cross-engine consumers, must have passed Runtime's
/// final retirement checks. This preserves the pool and command counters.
pub(crate) fn reseed_stamps(&mut self)->Result {
if !self.complete()? {return Err(EBUSY);}
for owner in [storage::STAMP,storage::FW_STAMP] {
self.objects[owner].u32(0,0xc1000000)?;
}
self.stamp_epoch.reseed(self.sequence);
Ok(())
}
pub(crate) fn completion_dependency(&self)->Result<crate::m3::m3_sync_layout::CompletionDependency> {
Ok(crate::m3::m3_sync_layout::CompletionDependency {
stamp:queue::FirmwareVa::new(self.objects[storage::FW_STAMP].va()).map_err(|_|EINVAL)?,
Expand Down
50 changes: 33 additions & 17 deletions drivers/gpu/drm/asahi/m3/m3_render.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ pub(crate) struct Render {
_binding:mmu::VmBind,
stats: [Region;2], profile:super::m3_profile::Profile, sequence_bytes: KVec<u8>, tiler_bytes: KVec<u8>, fragment_bytes: KVec<u8>,
last_progress_ns:i64,
stamp_epoch:timeline::StampEpoch,
draw:u64, heads:[u16;2], cached_views:KVec<CachedViews>,
slot:usize, batch_count:usize, early_count:u32,
/// Measurement packets in the pass most recently appended to its queues.
Expand Down Expand Up @@ -98,7 +99,7 @@ impl Render {
tiler_bytes.resize(if profile==super::m3_profile::Profile::Extended {tiler::EXTENDED_SIZE} else {tiler::SIZE},0u8,GFP_KERNEL)?;
let mut fragment_bytes=KVec::new();
fragment_bytes.resize(if profile==super::m3_profile::Profile::Extended {fragment::EXTENDED_SIZE} else {fragment::SIZE},0u8,GFP_KERNEL)?;
let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi};
let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,stamp_epoch:timeline::StampEpoch::default(),draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi};
// Lists and the matching manager are built before InitBM publication.
job.initialize_parameter_buffer()?;
for (index,pool) in UMA_POOLS.iter().enumerate() {
Expand Down Expand Up @@ -245,11 +246,11 @@ impl Render {
value.encode_extended(&mut init).map_err(|_|EINVAL)?;
} else {value.encode(&mut init).map_err(|_|EINVAL)?;}
self.init_bm.write(0, &init)?;
let ta_value = timeline::stamp(s::STAMP_TA - 256, draw);
let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw);
let ta_value = self.stamp_epoch.value(s::STAMP_TA - 256,draw);
let fs_value = self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,draw);
let previous=predecessor.unwrap_or(sync::CompletionDependency {
stamp:if early {ta_stamp} else {fs_stamp},
value:timeline::stamp((if early {s::STAMP_TA} else {s::STAMP_FRAGMENT})-256,draw-1),
value:self.stamp_epoch.value((if early {s::STAMP_TA} else {s::STAMP_FRAGMENT})-256,draw-1),
event:u32::from(!early),
});
self.write_barrier(P::TilerToFragment, sync::Barrier {
Expand All @@ -260,7 +261,7 @@ impl Render {
event: previous.event, self_value: ta_value,
uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass })?;
self.write_barrier(P::FragmentDependency, sync::Barrier {
stamp: fs_stamp, wait_value: timeline::stamp(s::STAMP_FRAGMENT - 256, draw - 1), event: 1, self_value: fs_value,
stamp: fs_stamp, wait_value: self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,draw - 1), event: 1, self_value: fs_value,
uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass })
}
// Prepare each private command after its retired slot is reset. The
Expand All @@ -286,7 +287,7 @@ impl Render {
scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?},gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,
sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?,
scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?,
stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?],
stamp_value:self.stamp_epoch.value(s::STAMP_TA - 256,draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?],
user_timestamps:[fw(P::TilerUserStart)?,fw(P::TilerUserEnd)?],preemption:[gpu(P::Preemption0)?,gpu(P::Preemption1)?,gpu(P::Preemption2)?],
tilemap:gpu(P::Tilemap)?,tpc:gpu(P::Tpc)?,tpc_bytes:g.tpc_bytes,heap:gpu(P::HeapMetadata)?,
scene_user:gpu(P::SceneUser)?,initial_scene_entry,geometry:g,page_count:PB_PAGES,
Expand All @@ -295,7 +296,7 @@ impl Render {
};
if self.profile==super::m3_profile::Profile::Extended {
value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).ok_or(EOVERFLOW)?,
peer_stamp:timeline::stamp(s::STAMP_FRAGMENT-256,draw),usc_window:usc,
peer_stamp:self.stamp_epoch.value(s::STAMP_FRAGMENT-256,draw),usc_window:usc,
firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[0].manager].va()+0x100).map_err(|_|EINVAL)?,
scene_scratch:gpu(P::FragmentScratch)?},&mut self.tiler_bytes).map_err(|_|EINVAL)?;
} else {value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?;}
Expand All @@ -317,7 +318,7 @@ impl Render {
queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?,
manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?,
fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?,
context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw),
context:self._binding.slot(),stamp:self.stamp_epoch.value((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256,draw),
};
let timed=self.stage_timestamps[usize::from(fs)];
let length=if self.profile==super::m3_profile::Profile::Extended {
Expand Down Expand Up @@ -380,7 +381,7 @@ impl Render {
gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?,
heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?,
scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?,
fw_stamp:shared(s::FRAGMENT_FW_STAMP)?,stamp_value:timeline::stamp(s::STAMP_FRAGMENT - 256, draw),
fw_stamp:shared(s::FRAGMENT_FW_STAMP)?,stamp_value:self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,draw),
timestamps:[fw(P::FragmentStart)?,fw(P::FragmentEnd)?],user_timestamps:[fw(P::FragmentUserStart)?,fw(P::FragmentUserEnd)?],
state:fragment::State {
geometry,multisample:r.multisample_control,merge_upper:[r.merge_upper_x,r.merge_upper_y],
Expand Down Expand Up @@ -482,12 +483,27 @@ impl Render {
/// across stamp wrap; the runtime bounds outstanding passes to SLOTS.
pub(crate) fn complete_through(&mut self,draw:u64)->Result<bool> {
if draw==0 {return Ok(true);}
for (stamp,fw,value) in [(s::TA_STAMP,s::TA_FW_STAMP,timeline::stamp(s::STAMP_TA-256,draw)),
(s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,timeline::stamp(s::STAMP_FRAGMENT-256,draw))] {
for (stamp,fw,value) in [(s::TA_STAMP,s::TA_FW_STAMP,self.stamp_epoch.value(s::STAMP_TA-256,draw)),
(s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,self.stamp_epoch.value(s::STAMP_FRAGMENT-256,draw))] {
if !timeline::reached(self.objects[stamp].read_u32(0)?,value) || !timeline::reached(self.objects[fw].read_u32(0)?,value) {return Ok(false);}
}
Ok(true)
}
pub(crate) fn stamps_need_reseed(&self)->bool {
self.stamp_epoch.needs_reseed(s::STAMP_TA-256,self.draw)
}
/// Runtime calls this only after every active batch (including compute and
/// mixed dependencies) has passed the runtime's final retirement checks.
pub(crate) fn reseed_stamps(&mut self)->Result {
if !self.complete()? {return Err(EBUSY);}
for (host,firmware,seed) in [(s::TA_STAMP,s::TA_FW_STAMP,s::STAMP_TA-256),
(s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::STAMP_FRAGMENT-256)] {
self.objects[host].u32(0,seed)?;
self.objects[firmware].u32(0,seed)?;
}
self.stamp_epoch.reseed(self.draw);
Ok(())
}
/// Whether any draw has been published (the first one initializes the buffer manager).
pub(crate) fn started(&self)->bool {self.draw>0}
pub(crate) fn first(&self)->bool {
Expand Down Expand Up @@ -609,7 +625,7 @@ impl Render {
pub(crate) fn completion_dependency(&self)->Result<sync::CompletionDependency> {
Ok(sync::CompletionDependency {
stamp:queue::FirmwareVa::new(self.objects[s::FRAGMENT_FW_STAMP].va()).map_err(|_|EINVAL)?,
value:timeline::stamp(s::STAMP_FRAGMENT-256,self.draw),event:1,
value:self.stamp_epoch.value(s::STAMP_FRAGMENT-256,self.draw),event:1,
})
}
pub(crate) fn append_after(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,
Expand Down Expand Up @@ -735,15 +751,15 @@ impl Render {
}
/// Read the fragment host and firmware stamps without changing retirement.
pub(crate) fn fragment_stamp_parts(&mut self)->Result<[bool;2]> {
let value=timeline::stamp(s::STAMP_FRAGMENT - 256,self.draw);
let value=self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,self.draw);
Ok([self.objects[s::FRAGMENT_STAMP].read_u32(0)?==value,
self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?==value])
}
/// Observe each completion condition across both render queues independently.
pub(crate) fn completion_parts(&mut self)->Result<[bool;4]> {
let mut done=[true;4];
for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]),
(s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] {
for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,self.stamp_epoch.value(s::STAMP_TA - 256,self.draw),self.heads[0]),
(s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,self.draw),self.heads[1])] {
done[0]&=self.objects[stamp].read_u32(0)?==value;
done[1]&=self.objects[fw].read_u32(0)?==value;
done[2]&=self.objects[pointers].read_u32(queue::DONE)?==u32::from(head);
Expand All @@ -752,8 +768,8 @@ impl Render {
Ok(done)
}
pub(crate) fn complete(&mut self)->Result<bool> {
for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]),
(s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] {
for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,self.stamp_epoch.value(s::STAMP_TA - 256,self.draw),self.heads[0]),
(s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,self.stamp_epoch.value(s::STAMP_FRAGMENT - 256,self.draw),self.heads[1])] {
if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value
|| self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head)
|| self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);}
Expand Down
26 changes: 26 additions & 0 deletions drivers/gpu/drm/asahi/m3/m3_runtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -295,11 +295,37 @@ impl Inner {
self.config.pipe_free(0)? && self.config.pipe_free(1)?) {return Ok(None);}
Ok(Some(room))
}
/// Stop extending the pipeline before any stamp reaches zero. The final
/// active batch must pass poll_active's stamp/queue/event/pipe checks and
/// configured SoC MMIO checks before either epoch can change.
fn reseed_stamp_epochs(&mut self)->Result<bool> {
let reseed=self.jobs.iter().any(|job|match job {
NativeJob::Render(j)=>j.stamps_need_reseed(),
NativeJob::Compute(j)=>j.stamps_need_reseed(),
});
if !reseed {return Ok(true);}
if !self.active.is_empty() {return Ok(false);}
for job in &mut self.jobs {
match job {
NativeJob::Render(j) if j.stamps_need_reseed()=>j.reseed_stamps()?,
NativeJob::Compute(j) if j.stamps_need_reseed()=>j.reseed_stamps()?,
_=>{},
}
}
crate::m3::agx_memory::publish();
crate::mem::sync();
Ok(true)
}
/// Publish pending batches while allowed: one when none is active, more behind it with
/// asahi.m3_pipeline_depth > 1. Returns whether anything was published or failed.
fn publish_next(&mut self)->bool {
let mut progress=false;
while !self.pending.is_empty() && self.state.healthy() {
match self.reseed_stamp_epochs() {
Ok(true)=>{},
Ok(false)=>break,
Err(error)=>{self.fail_active(error);return true;},
}
let overlap=!self.active.is_empty();
let (base,max)=match self.pipeline_slot() {
Ok(Some(slot))=>slot,
Expand Down
38 changes: 32 additions & 6 deletions drivers/gpu/drm/asahi/m3/m3_timeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,39 @@
//! its distance is well below the half-range. Keep the host command ordinal
//! separately in u64 so stamp wrap never re-registers a queue or emits InitBM.

/// Nonzero wire stamp after `ordinal` commands, from the retired seed.
/// As on the M4 runtime, do not publish zero as a completion stamp. The
/// 24-bit stamp sequence skips zero; command ordinals/event counts do not.
/// Wire stamp after `ordinal` commands, from the retired seed.
/// Skipping zero would advance by 0x200 and stop the firmware with
/// "DAG: Non Sequential Stamp Updates". Live queues use `StampEpoch` and
/// reseed while idle before reaching zero, rather than relying on zero stamps.
pub(crate) const fn stamp(seed: u32, ordinal: u64) -> u32 {
const PERIOD: u64 = 0x00ff_ffff;
let unit = (((seed >> 8) as u64 - 1 + ordinal % PERIOD) % PERIOD) + 1;
(unit as u32) << 8
seed.wrapping_add((ordinal as u32).wrapping_mul(0x100))
}

/// Leave ample room for a complete bounded batch before the earliest wrap,
/// as the G15 runtime does. Admission stops extending the pipeline at this
/// boundary; reseeding requires the runtime's final batch retirement checks.
const RESEED_MARGIN: u64 = 0x10_0000;

/// Wire-stamp epoch, independent of monotonic command/pool/notifier ordinals.
#[derive(Default)]
pub(crate) struct StampEpoch {
origin: u64,
}

impl StampEpoch {
pub(crate) fn value(&self, seed: u32, ordinal: u64) -> u32 {
stamp(seed, ordinal - self.origin)
}

pub(crate) fn needs_reseed(&self, seed: u32, ordinal: u64) -> bool {
ordinal - self.origin >= (1u64 << 24) - (seed >> 8) as u64 - RESEED_MARGIN
}

/// Caller has drained all active batches and written both stamp copies
/// back to their seeds. No dependency from the old epoch may remain live.
pub(crate) fn reseed(&mut self, retired: u64) {
self.origin = retired;
}
}

/// Firmware command count, before the command with this host ordinal.
Expand Down
Loading
Loading