diff --git a/drivers/gpu/drm/asahi/g18/boot/recovery.rs b/drivers/gpu/drm/asahi/g18/boot/recovery.rs index d60912f869a6f..01849cc5abf7a 100644 --- a/drivers/gpu/drm/asahi/g18/boot/recovery.rs +++ b/drivers/gpu/drm/asahi/g18/boot/recovery.rs @@ -43,11 +43,22 @@ const JOB_WATCHDOG_MS: u32 = 10_000; const RECOVERED_RETIRE_MS: u32 = 2000; /// Poll interval for a wait that has lasted `us`: fine while a job is -/// likely to be about to retire, coarse once it has clearly not. +/// likely to be about to retire, coarse once it has clearly not. Full-panel +/// jobs take 2-10 ms, so the fine interval covers them. pub(super) const fn poll_interval_us(us: i64) -> i64 { - if us < 2_000 { 20 } else if us < 50_000 { 200 } else { 1_000 } + if us < 20_000 { 20 } else if us < 50_000 { 200 } else { 1_000 } } +/// Set by a job's retirement check once its stages have finished (the +/// scheduler's completed count or the stage stamps): the conditions that +/// remain follow within 50-150 us. One job is in flight at a time, under the +/// session lock; wait_job clears this before its first poll. +pub(super) static RETIRE_NEAR: AtomicBool = AtomicBool::new(false); +/// How long wait_job re-polls without sleeping after RETIRE_NEAR, and the +/// delay between those polls. +const RETIRE_NEAR_SPIN_US: i64 = 200; +const RETIRE_NEAR_STEP_US: i64 = 2; + /// Upper bounds, in microseconds, of the latency buckets; the last bucket is /// open-ended. const LATENCY_BOUNDS_US: [i64; 6] = [100, 300, 1_000, 3_000, 10_000, 100_000]; @@ -392,6 +403,8 @@ impl G18Session { let mut recovery: Option = None; let mut since = Instant::::now(); let mut reported = 0u32; + let mut near_since: Option = None; + RETIRE_NEAR.store(false, Ordering::Relaxed); loop { if let Some(serviced) = self.poll_faults(label)? { if recovery.is_none() { @@ -426,6 +439,15 @@ impl G18Session { since = Instant::::now(); reported = 0; } + // Stages finished and no recovery: the last conditions are a few + // tens of microseconds away, less than one sleeping poll. + if recovery.is_none() && RETIRE_NEAR.load(Ordering::Relaxed) { + let first = *near_since.get_or_insert(us); + if us - first < RETIRE_NEAR_SPIN_US { + kernel::time::delay::udelay(Delta::from_micros(RETIRE_NEAR_STEP_US)); + continue; + } + } fsleep(Delta::from_micros(poll_interval_us(us))); } } diff --git a/drivers/gpu/drm/asahi/g18/boot/session.rs b/drivers/gpu/drm/asahi/g18/boot/session.rs index a44fe7f872f22..7daa933896924 100644 --- a/drivers/gpu/drm/asahi/g18/boot/session.rs +++ b/drivers/gpu/drm/asahi/g18/boot/session.rs @@ -37,6 +37,10 @@ impl ApplicationOwner { let session=self.session.clone();let failed=self.failed.clone(); let quarantined=self.quarantined.clone(); let ticket=Arc::new(AtomicI32::new(TICKET_RUNNING),GFP_KERNEL)?;let done=ticket.clone(); + // The caller sleeps on `gate`; the worker's guard completes it when + // the closure ends, or is dropped unrun. + let gate=Arc::pin_init(kernel::sync::Completion::new(),GFP_KERNEL)?; + let signal=TicketSignal(gate.clone()); kernel::workqueue::system_long().try_spawn(GFP_KERNEL,move || { let mut owner=session.lock(); let result=if failed.load(Ordering::Acquire) {Err(Failure::Fatal(EIO))} else { @@ -75,16 +79,23 @@ impl ApplicationOwner { Err(Failure::Job(JobFailure::Recovered(_) | JobFailure::Unbound(..) | JobFailure::Released))=>JobOutcome::Recovered as i32, Err(Failure::Job(JobFailure::Stranded(..) | JobFailure::Retained))=>JobOutcome::Stranded as i32, },Ordering::Release); + // The outcome is stored. Release the session before waking the + // caller: it, or the next client's worker, takes that lock next. + drop(owner); + drop(signal); })?; let waited=Instant::::now(); - loop { - match ticket.load(Ordering::Acquire) { - TICKET_RUNNING => fsleep(Delta::from_micros(recovery::poll_interval_us(waited.elapsed().as_micros_ceil()))), - 0 => {recovery::SUBMIT_LATENCY.record(waited.elapsed().as_micros_ceil());return Ok(())} - code if code==JobOutcome::Recovered as i32 => return Err(Failure::Job(JobFailure::Released)), - code if code==JobOutcome::Stranded as i32 => return Err(Failure::Job(JobFailure::Retained)), - errno => return Err(Failure::Fatal(Error::from_errno(errno))), - } + // Sleep in slices: queued behind other clients and recovery, a wait + // can outlast the hung-task timeout, which a single uninterruptible + // sleep would trip. + while !gate.wait_for_completion_timeout(kernel::time::msecs_to_jiffies(1000)) {} + match ticket.load(Ordering::Acquire) { + // The worker closure was dropped without running. + TICKET_RUNNING => Err(Failure::Fatal(EIO)), + 0 => {recovery::SUBMIT_LATENCY.record(waited.elapsed().as_micros_ceil());Ok(())} + code if code==JobOutcome::Recovered as i32 => Err(Failure::Job(JobFailure::Released)), + code if code==JobOutcome::Stranded as i32 => Err(Failure::Job(JobFailure::Retained)), + errno => Err(Failure::Fatal(Error::from_errno(errno))), } } @@ -101,6 +112,15 @@ impl ApplicationOwner { } } +/// Wakes the thread waiting in [`ApplicationOwner::work`] when dropped: at the +/// end of the worker closure, or when the closure is dropped without running. +struct TicketSignal(Arc); +impl Drop for TicketSignal { + fn drop(&mut self) { + self.0.complete_all(); + } +} + /// Worker ticket values besides 0 and negative errnos. const TICKET_RUNNING: i32 = 1; #[repr(i32)] @@ -239,6 +259,11 @@ impl G18Session { && scheduler.submitted==frame.submitted() && scheduler.completed==frame.submitted() && scheduler.linked==0 && list==[0,job.job.queues().job_list()]; let completed=stamps==[frame.stamp();2] && masks[0]&0x600==0x600; + // The remaining conditions follow within 50-150 us: wait_job + // re-polls without sleeping for a bounded time. + if !recovered && (scheduler.completed==frame.submitted() || stamps==[frame.stamp();2]) { + recovery::RETIRE_NEAR.store(true,Ordering::Relaxed); + } recovery::note_phases(&mut first,&kicked,&[outer==[outer_after;3],inner==[[frame.inner_after();3];2], scheduler.submitted==frame.submitted(),scheduler.completed==frame.submitted(),scheduler.linked==0, list==[0,job.job.queues().job_list()],stamps==[frame.stamp();2],masks[0]&0x600==0x600]); diff --git a/drivers/gpu/drm/asahi/g18/initdata/mod.rs b/drivers/gpu/drm/asahi/g18/initdata/mod.rs index 50b8901e252a4..d098bbe185917 100644 --- a/drivers/gpu/drm/asahi/g18/initdata/mod.rs +++ b/drivers/gpu/drm/asahi/g18/initdata/mod.rs @@ -501,7 +501,9 @@ impl G18FirmwareMemory { // bit9 (global 0x1093e5 bit1), the native stall diagnostic category. // audit-g18-graphics-stall.py executes that original predicate. let graphics_stall_trace = crate::g18::params::g18_graphics_stall_trace.value() == 1; - let trace_mask = if graphics_stall_trace { 0x203 } else { 3 }; + // No per-job trace records unless asked for (g18_fw_trace). + let trace_mask = if graphics_stall_trace { 0x203 } + else if crate::g18::params::g18_fw_trace.value() != 0 { 3 } else { 0 }; self.private.write_u32(trace, trace_mask)?; // b000 refresh 0x352c/0x3688 copies region-C+0x9ac into // global 0x109488. Its compute nibble (bits8..11) selects native diff --git a/drivers/gpu/drm/asahi/g18/params.rs b/drivers/gpu/drm/asahi/g18/params.rs index 555c9959ead2f..745938e9c0bbc 100644 --- a/drivers/gpu/drm/asahi/g18/params.rs +++ b/drivers/gpu/drm/asahi/g18/params.rs @@ -99,6 +99,8 @@ g18_param!(g18_sma_minimum: u32 = 1, PARAM_OPS_U32, "Stage9/profile4 or stage10/profile1, variant0: 1 probes fixed native minimum TA/fragment resource words with unchanged owned backing. Default 1"); g18_param!(g18_graphics_stall_trace: u32 = 0, PARAM_OPS_U32, "Stage9/profile4 or stage10/profile1, variant0: 1 enables firmware category9 diagnostics and the native TA mode2 progress sampler. No timeout changes. Default 0"); +g18_param!(g18_fw_trace: u32 = 0, PARAM_OPS_U32, + "G18 firmware trace: 0 (default) writes trace mask 0, so render jobs produce no firmware trace records and G18_APP_FAIL dumps have none; 1 writes mask 3, the earlier default, which restores them. g18_graphics_stall_trace=1 still selects 0x203"); g18_param!(g18_kick_log: u32 = 0, PARAM_OPS_U32, "Stage10 application DRM log volume: 0 (default) omits the per-kick, frame-pass and per-job resource lines (G18_APP_KICK, G18_APP_FRAME_PASS, G18_APP_COMPUTE_KNOBS, G18_COMPUTE_*, G18_SHARED_COMPUTE_*, G18 UMA/compute-context lines, G18_POOL_DESCRIPTOR_CURSOR, role-0 type-1 G18_EVENT) once G18_DRM_READY is logged; 1 prints them as before. Startup, fault, recovery, refusal, wait, stall, admission and sampled summary lines are always printed"); g18_param!(g18_vdm_high_alias: u32 = 0, PARAM_OPS_U32,