Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions drivers/gpu/drm/asahi/g18/boot/recovery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,22 @@ const JOB_WATCHDOG_MS: u32 = 10_000;
const RECOVERED_RETIRE_MS: u32 = 2000;

/// Poll interval for a wait that has lasted `us`: fine while a job is
/// likely to be about to retire, coarse once it has clearly not.
/// likely to be about to retire, coarse once it has clearly not. Full-panel
/// jobs take 2-10 ms, so the fine interval covers them.
pub(super) const fn poll_interval_us(us: i64) -> i64 {
if us < 2_000 { 20 } else if us < 50_000 { 200 } else { 1_000 }
if us < 20_000 { 20 } else if us < 50_000 { 200 } else { 1_000 }
}

/// Set by a job's retirement check once its stages have finished (the
/// scheduler's completed count or the stage stamps): the conditions that
/// remain follow within 50-150 us. One job is in flight at a time, under the
/// session lock; wait_job clears this before its first poll.
pub(super) static RETIRE_NEAR: AtomicBool = AtomicBool::new(false);
/// How long wait_job re-polls without sleeping after RETIRE_NEAR, and the
/// delay between those polls.
const RETIRE_NEAR_SPIN_US: i64 = 200;
const RETIRE_NEAR_STEP_US: i64 = 2;

/// Upper bounds, in microseconds, of the latency buckets; the last bucket is
/// open-ended.
const LATENCY_BOUNDS_US: [i64; 6] = [100, 300, 1_000, 3_000, 10_000, 100_000];
Expand Down Expand Up @@ -392,6 +403,8 @@ impl G18Session {
let mut recovery: Option<G18Recovery> = None;
let mut since = Instant::<Monotonic>::now();
let mut reported = 0u32;
let mut near_since: Option<i64> = None;
RETIRE_NEAR.store(false, Ordering::Relaxed);
loop {
if let Some(serviced) = self.poll_faults(label)? {
if recovery.is_none() {
Expand Down Expand Up @@ -426,6 +439,15 @@ impl G18Session {
since = Instant::<Monotonic>::now();
reported = 0;
}
// Stages finished and no recovery: the last conditions are a few
// tens of microseconds away, less than one sleeping poll.
if recovery.is_none() && RETIRE_NEAR.load(Ordering::Relaxed) {
let first = *near_since.get_or_insert(us);
if us - first < RETIRE_NEAR_SPIN_US {
kernel::time::delay::udelay(Delta::from_micros(RETIRE_NEAR_STEP_US));
continue;
}
}
fsleep(Delta::from_micros(poll_interval_us(us)));
}
}
Expand Down
41 changes: 33 additions & 8 deletions drivers/gpu/drm/asahi/g18/boot/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ impl ApplicationOwner {
let session=self.session.clone();let failed=self.failed.clone();
let quarantined=self.quarantined.clone();
let ticket=Arc::new(AtomicI32::new(TICKET_RUNNING),GFP_KERNEL)?;let done=ticket.clone();
// The caller sleeps on `gate`; the worker's guard completes it when
// the closure ends, or is dropped unrun.
let gate=Arc::pin_init(kernel::sync::Completion::new(),GFP_KERNEL)?;
let signal=TicketSignal(gate.clone());
kernel::workqueue::system_long().try_spawn(GFP_KERNEL,move || {
let mut owner=session.lock();
let result=if failed.load(Ordering::Acquire) {Err(Failure::Fatal(EIO))} else {
Expand Down Expand Up @@ -75,16 +79,23 @@ impl ApplicationOwner {
Err(Failure::Job(JobFailure::Recovered(_) | JobFailure::Unbound(..) | JobFailure::Released))=>JobOutcome::Recovered as i32,
Err(Failure::Job(JobFailure::Stranded(..) | JobFailure::Retained))=>JobOutcome::Stranded as i32,
},Ordering::Release);
// The outcome is stored. Release the session before waking the
// caller: it, or the next client's worker, takes that lock next.
drop(owner);
drop(signal);
})?;
let waited=Instant::<Monotonic>::now();
loop {
match ticket.load(Ordering::Acquire) {
TICKET_RUNNING => fsleep(Delta::from_micros(recovery::poll_interval_us(waited.elapsed().as_micros_ceil()))),
0 => {recovery::SUBMIT_LATENCY.record(waited.elapsed().as_micros_ceil());return Ok(())}
code if code==JobOutcome::Recovered as i32 => return Err(Failure::Job(JobFailure::Released)),
code if code==JobOutcome::Stranded as i32 => return Err(Failure::Job(JobFailure::Retained)),
errno => return Err(Failure::Fatal(Error::from_errno(errno))),
}
// Sleep in slices: queued behind other clients and recovery, a wait
// can outlast the hung-task timeout, which a single uninterruptible
// sleep would trip.
while !gate.wait_for_completion_timeout(kernel::time::msecs_to_jiffies(1000)) {}
match ticket.load(Ordering::Acquire) {
// The worker closure was dropped without running.
TICKET_RUNNING => Err(Failure::Fatal(EIO)),
0 => {recovery::SUBMIT_LATENCY.record(waited.elapsed().as_micros_ceil());Ok(())}
code if code==JobOutcome::Recovered as i32 => Err(Failure::Job(JobFailure::Released)),
code if code==JobOutcome::Stranded as i32 => Err(Failure::Job(JobFailure::Retained)),
errno => Err(Failure::Fatal(Error::from_errno(errno))),
}
}

Expand All @@ -101,6 +112,15 @@ impl ApplicationOwner {
}
}

/// Wakes the thread waiting in [`ApplicationOwner::work`] when dropped: at the
/// end of the worker closure, or when the closure is dropped without running.
struct TicketSignal(Arc<kernel::sync::Completion>);
impl Drop for TicketSignal {
fn drop(&mut self) {
self.0.complete_all();
}
}

/// Worker ticket values besides 0 and negative errnos.
const TICKET_RUNNING: i32 = 1;
#[repr(i32)]
Expand Down Expand Up @@ -239,6 +259,11 @@ impl G18Session {
&& scheduler.submitted==frame.submitted() && scheduler.completed==frame.submitted()
&& scheduler.linked==0 && list==[0,job.job.queues().job_list()];
let completed=stamps==[frame.stamp();2] && masks[0]&0x600==0x600;
// The remaining conditions follow within 50-150 us: wait_job
// re-polls without sleeping for a bounded time.
if !recovered && (scheduler.completed==frame.submitted() || stamps==[frame.stamp();2]) {
recovery::RETIRE_NEAR.store(true,Ordering::Relaxed);
}
recovery::note_phases(&mut first,&kicked,&[outer==[outer_after;3],inner==[[frame.inner_after();3];2],
scheduler.submitted==frame.submitted(),scheduler.completed==frame.submitted(),scheduler.linked==0,
list==[0,job.job.queues().job_list()],stamps==[frame.stamp();2],masks[0]&0x600==0x600]);
Expand Down
4 changes: 3 additions & 1 deletion drivers/gpu/drm/asahi/g18/initdata/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -501,7 +501,9 @@ impl G18FirmwareMemory {
// bit9 (global 0x1093e5 bit1), the native stall diagnostic category.
// audit-g18-graphics-stall.py executes that original predicate.
let graphics_stall_trace = crate::g18::params::g18_graphics_stall_trace.value() == 1;
let trace_mask = if graphics_stall_trace { 0x203 } else { 3 };
// No per-job trace records unless asked for (g18_fw_trace).
let trace_mask = if graphics_stall_trace { 0x203 }
else if crate::g18::params::g18_fw_trace.value() != 0 { 3 } else { 0 };
self.private.write_u32(trace, trace_mask)?;
// b000 refresh 0x352c/0x3688 copies region-C+0x9ac into
// global 0x109488. Its compute nibble (bits8..11) selects native
Expand Down
2 changes: 2 additions & 0 deletions drivers/gpu/drm/asahi/g18/params.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ g18_param!(g18_sma_minimum: u32 = 1, PARAM_OPS_U32,
"Stage9/profile4 or stage10/profile1, variant0: 1 probes fixed native minimum TA/fragment resource words with unchanged owned backing. Default 1");
g18_param!(g18_graphics_stall_trace: u32 = 0, PARAM_OPS_U32,
"Stage9/profile4 or stage10/profile1, variant0: 1 enables firmware category9 diagnostics and the native TA mode2 progress sampler. No timeout changes. Default 0");
g18_param!(g18_fw_trace: u32 = 0, PARAM_OPS_U32,
"G18 firmware trace: 0 (default) writes trace mask 0, so render jobs produce no firmware trace records and G18_APP_FAIL dumps have none; 1 writes mask 3, the earlier default, which restores them. g18_graphics_stall_trace=1 still selects 0x203");
g18_param!(g18_kick_log: u32 = 0, PARAM_OPS_U32,
"Stage10 application DRM log volume: 0 (default) omits the per-kick, frame-pass and per-job resource lines (G18_APP_KICK, G18_APP_FRAME_PASS, G18_APP_COMPUTE_KNOBS, G18_COMPUTE_*, G18_SHARED_COMPUTE_*, G18 UMA/compute-context lines, G18_POOL_DESCRIPTOR_CURSOR, role-0 type-1 G18_EVENT) once G18_DRM_READY is logged; 1 prints them as before. Startup, fault, recovery, refusal, wait, stall, admission and sampled summary lines are always printed");
g18_param!(g18_vdm_high_alias: u32 = 0, PARAM_OPS_U32,
Expand Down
Loading