diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index fe47291dff0dc..0b9e855a98677 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -268,6 +268,60 @@ patternProperties: The SMC, whose package power reading the low-power platform profile holds the CPU clusters to. + apple,cpu-memory-states: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + description: + The DCS state (first row) and fabric state (second row) a busy CPU + cluster asks for below 25, 50, 75 and 100 percent of its frequency, + as measured on this SoC's memory. Without it, the states of the + compatible's own table. + minItems: 2 + maxItems: 2 + items: + minItems: 4 + maxItems: 4 + items: + maximum: 255 + + apple,cpu-level-mhz: + description: + The CPU frequency whose quarters index apple,cpu-memory-states for + every cluster, so that clusters of different speed share them. + Without it, each cluster's own maximum. + maxItems: 1 + + apple,gpu-busy-memory-states: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + description: + The DCS state (first row) and fabric state (second row) for GPU work + the GPU driver reports, by the busy share it reports, below each of + the three apple,gpu-busy-permille shares and at or above the last. + Without it, such work gets the top of the CPU states. + minItems: 2 + maxItems: 2 + items: + minItems: 4 + maxItems: 4 + items: + maximum: 255 + + apple,gpu-busy-permille: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The GPU busy shares, in permille of the time the GPU spends + executing, from which reported GPU work gets the second, third and + top column of apple,gpu-busy-memory-states. + minItems: 3 + maxItems: 3 + items: + maximum: 1000 + + dependentRequired: + apple,gpu-busy-memory-states: + - apple,gpu-busy-permille + apple,gpu-busy-permille: + - apple,gpu-busy-memory-states + required: - compatible - reg diff --git a/arch/arm64/boot/dts/apple/t6041.dtsi b/arch/arm64/boot/dts/apple/t6041.dtsi index be2b09cc1dfd5..896b73c478f17 100644 --- a/arch/arm64/boot/dts/apple/t6041.dtsi +++ b/arch/arm64/boot/dts/apple/t6041.dtsi @@ -340,6 +340,27 @@ apple,static-voters = <1 3>; }; +/* + * The CLPC's memory states, measured on a J616c. A busy CPU cluster asks + * for DCS 1, 1, 3 or 4 and fabric 0, by quarters of 4512 MHz, so the E + * cluster at its top speed (57 %) gets the third column. With a 3 GB/s + * copy: DCS 0 / fabric 0 5.46 W at 309 ns DRAM latency, DCS 3 6.22 W at + * 166 ns, DCS 4 6.92 W at 114 ns, DCS 5 7.04 W at 113 ns; fabric 1 and 2 + * add 0.3 and 1.4 W with less gain than a DCS step. + * + * GPU work takes DCS 1, 2, 4 or 5 and fabric 0, 0, 0 or 2 by the busy + * share the GPU driver reports, split at 160, 500 and 800 permille, and the + * top states with no share reported. Scrolling a browser went from 24.9 to + * 22.1 W on AC with these states and the G17 light effort, against the + * M4 Pro's. + */ +&clpc { + apple,cpu-memory-states = <1 1 3 4>, <0 0 0 0>; + apple,cpu-level-mhz = <4512>; + apple,gpu-busy-memory-states = <1 2 4 5>, <0 0 0 2>; + apple,gpu-busy-permille = <160 500 800>; +}; + / { /* * The loader fills the operating points, the CS and AFR tables and the diff --git a/drivers/gpu/drm/asahi/g17/clpc.c b/drivers/gpu/drm/asahi/g17/clpc.c index 58adaba19910f..f4d8d36c8ea8c 100644 --- a/drivers/gpu/drm/asahi/g17/clpc.c +++ b/drivers/gpu/drm/asahi/g17/clpc.c @@ -10,6 +10,10 @@ void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) { apple_clpc_gpu_set_active(gpu, active); } +void asahi_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille) +{ + apple_clpc_gpu_set_busy(gpu, permille); +} void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu) { apple_clpc_gpu_put(gpu); diff --git a/drivers/gpu/drm/asahi/g17/clpc.h b/drivers/gpu/drm/asahi/g17/clpc.h index 372820ade7eff..fa36e30ef8896 100644 --- a/drivers/gpu/drm/asahi/g17/clpc.h +++ b/drivers/gpu/drm/asahi/g17/clpc.h @@ -6,4 +6,5 @@ struct apple_clpc_gpu; int asahi_clpc_gpu_get(void *dev, struct apple_clpc_gpu **out); void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu); +void asahi_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille); #endif diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index b2c0aa5e0c4b2..f2ee9d1789275 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -254,6 +254,10 @@ pub(crate) struct Config { pub(crate) gpu_generation: hw::GpuGen, /// The firmware builds this SoC runs. pub(crate) firmware: fw::variant::Firmware, + /// Lower firmware effort for quiet GPU work: [`power::LIGHT_DEFAULT`] on every G17. `None` + /// runs every submission at full effort, for a SoC whose hardware needs that; its entry says + /// why. + pub(crate) light_effort: Option, /// Layouts of the command buffers that follow the GPU's core layout. pub(crate) core_layout: CoreLayout, } @@ -954,7 +958,7 @@ impl Firmware { self.primary.state.shared.note_gpu_activity(); self.effort.submit(); self.effort.note_waiters(now_ns(), fence::waiters()); - self.apply_effort(power::FULL_EFFORT) + self.apply_effort(power::submission_effort()) } /// One governor step with the hotspot temperature reading; returns whether to run again and @@ -1606,6 +1610,7 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + power::configure_light(cfg.light_effort.as_ref()); let calibration = if cfg.chip_id == crate::hw::t6040::CHIP_ID { Some(calibration::Calibration::read(dev).inspect_err(|_| { dev_err!( diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index f9bd31f13bc7d..2c8805b4fac6d 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -529,7 +529,9 @@ impl Port { pub(super) fn note_submission(&self) -> bool { self.submissions.fetch_add(1, Ordering::SeqCst); self.note_gpu_activity(); + // Take the slow path when the governor wants a different effort. self.effort_full.load(Ordering::SeqCst) + && super::power::submission_effort() == super::power::FULL_EFFORT } pub(super) fn submissions(&self) -> u64 { @@ -564,6 +566,7 @@ impl Port { self.last_submission_ns .store(super::now_ns(), Ordering::Relaxed); self.demand.set_active(true); + self.demand.report_busy(super::now_ns()); } pub(super) fn write_power(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 8be3bae04afb0..a4caca498f3c2 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -3,7 +3,7 @@ //! Work-based firmware performance effort. Idle power gating is managed separately by the //! device-control ring; this input never keeps an otherwise idle GPU powered. -use core::sync::atomic::AtomicU32; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; pub(super) const FULL_EFFORT: u32 = 0x10000; pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; @@ -351,6 +351,97 @@ unsafe extern "C" { fn asahi_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int; fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void); + fn asahi_clpc_gpu_set_busy(gpu: *mut core::ffi::c_void, permille: core::ffi::c_uint); +} + +/// Busy-share window: long enough to span frames, short enough to follow a burst. +const BUSY_WINDOW_NS: u64 = 200_000_000; +/// A longer gap between reports was idle: start over rather than average it in. +const BUSY_WINDOW_MAX_NS: u64 = 2_000_000_000; +static BUSY_WINDOW_START: AtomicU64 = AtomicU64::new(0); +static BUSY_WINDOW_TOTAL: AtomicU64 = AtomicU64::new(0); + +/// A window under this share (permille) with no long job is quiet. +/// Consecutive quiet windows needed to go light. +const LIGHT_WINDOWS: u32 = 4; +static LIGHT: AtomicBool = AtomicBool::new(false); +static QUIET_WINDOWS: AtomicU32 = AtomicU32::new(0); + +/// A SoC's light effort for quiet GPU work. Light effort runs the same work about twice as +/// long, so the way back to full effort sits at about twice the way in, or work that fits +/// light effort sticks at full. Shares in permille of a busy window. +pub(crate) struct LightEffort { + /// Firmware performance effort, of 65536. + pub(crate) effort: u32, + /// A window below this share with no job longer than `job_enter_us` is quiet. + pub(crate) enter_permille: u32, + /// Back to full effort from this share, or from a job of `job_exit_us`. + pub(crate) exit_permille: u32, + pub(crate) job_enter_us: u32, + pub(crate) job_exit_us: u32, +} + +/// The light effort of every G17. Nothing in the quiet-window logic depends on the SoC: the +/// values were measured on the M4 Max (scrolling 24.9 -> 22.1 W on AC with its CLPC memory +/// states) and apply wherever the firmware takes an effort input. +pub(crate) const LIGHT_DEFAULT: LightEffort = LightEffort { + effort: 0x8000, + enter_permille: 350, + exit_permille: 750, + job_enter_us: 3000, + job_exit_us: 7000, +}; + +static LIGHT_EFFORT: AtomicU32 = AtomicU32::new(0); +static LIGHT_ENTER: AtomicU32 = AtomicU32::new(0); +static LIGHT_EXIT: AtomicU32 = AtomicU32::new(0); +static LIGHT_JOB_ENTER_US: AtomicU32 = AtomicU32::new(0); +static LIGHT_JOB_EXIT_US: AtomicU32 = AtomicU32::new(0); + +/// Takes the SoC's light effort from its configuration; `None` keeps full effort. +pub(super) fn configure_light(light: Option<&LightEffort>) { + let (effort, enter, exit, job_enter, job_exit) = light.map_or((0, 0, 0, 0, 0), |l| { + (l.effort, l.enter_permille, l.exit_permille, l.job_enter_us, l.job_exit_us) + }); + LIGHT_ENTER.store(enter, Ordering::Relaxed); + LIGHT_EXIT.store(exit, Ordering::Relaxed); + LIGHT_JOB_ENTER_US.store(job_enter, Ordering::Relaxed); + LIGHT_JOB_EXIT_US.store(job_exit, Ordering::Relaxed); + LIGHT.store(false, Ordering::Relaxed); + QUIET_WINDOWS.store(0, Ordering::Relaxed); + LIGHT_EFFORT.store(effort, Ordering::Relaxed); +} + +/// The effort for the next submission: light once the GPU has been quiet for a while. +pub(super) fn submission_effort() -> u32 { + let light = LIGHT_EFFORT.load(Ordering::Relaxed); + if LIGHT.load(Ordering::Relaxed) && light != 0 { + light + } else { + FULL_EFFORT + } +} + +fn light_param(param: &AtomicU32) -> u64 { + u64::from(param.load(Ordering::Relaxed)) +} + +fn light_step(permille: u64, longest_ns: u64) { + if LIGHT_EFFORT.load(Ordering::Relaxed) == 0 { + return; + } + if LIGHT.load(Ordering::Relaxed) { + if permille >= light_param(&LIGHT_EXIT) || longest_ns >= light_param(&LIGHT_JOB_EXIT_US) * 1000 { + LIGHT.store(false, Ordering::Relaxed); + QUIET_WINDOWS.store(0, Ordering::Relaxed); + } + } else if permille < light_param(&LIGHT_ENTER) && longest_ns < light_param(&LIGHT_JOB_ENTER_US) * 1000 { + if QUIET_WINDOWS.fetch_add(1, Ordering::Relaxed) + 1 >= LIGHT_WINDOWS { + LIGHT.store(true, Ordering::Relaxed); + } + } else { + QUIET_WINDOWS.store(0, Ordering::Relaxed); + } } /// DT-selected memory-demand provider. Its independent allocation outlives a @@ -378,6 +469,33 @@ impl Demand { // SAFETY: the retained reference is live; NULL denotes no DT provider. unsafe { asahi_clpc_gpu_set_active(self.0, active) }; } + + /// Reports the share of the last window the GPU spent executing, once a window has run. + /// Of submitters racing at the end of a window, only the one that moves its start reports. + pub(super) fn report_busy(&self, now_ns: u64) { + let start = BUSY_WINDOW_START.load(Ordering::Relaxed); + let elapsed = now_ns.saturating_sub(start); + if elapsed < BUSY_WINDOW_NS + || BUSY_WINDOW_START + .compare_exchange(start, now_ns, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return; + } + let total = crate::usage::TOTAL_NS.load(Ordering::Relaxed); + let busy = total.saturating_sub(BUSY_WINDOW_TOTAL.swap(total, Ordering::Relaxed)); + if start == 0 || elapsed > BUSY_WINDOW_MAX_NS { + return; + } + let permille = busy.saturating_mul(1000) / elapsed; + light_step(permille, crate::usage::MAX_NS.swap(0, Ordering::Relaxed)); + // Where light effort is configured, full effort does the same work in about half the + // time; report the share in light effort's terms so that one memory ladder fits both. + let light = LIGHT_EFFORT.load(Ordering::Relaxed) != 0; + let scaled = if !light || LIGHT.load(Ordering::Relaxed) { permille } else { permille * 2 }; + // SAFETY: the retained reference is live; NULL denotes no DT provider. + unsafe { asahi_clpc_gpu_set_busy(self.0, scaled.min(1000) as core::ffi::c_uint) }; + } } impl Drop for Demand { diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index cfb369da993e9..54e2fcd52e48b 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -110,6 +110,7 @@ pub(crate) const CONFIG: Config = Config { variant: &fw::variant::V25G83, config: None, }]), + light_effort: Some(crate::g17::power::LIGHT_DEFAULT), // Measured on the 20-core M4 Pro; the alternative below remains experimental. core_layout: CoreLayout { cores: Some((2, 20)), diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 5dc8020b89f49..ac02d23291bda 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -54,6 +54,7 @@ pub(super) const BASE: Config = Config { recovery_slot_keys: None, gpu_generation: GpuGen::G16, firmware: fw::variant::Firmware::Identified(&[]), + light_effort: Some(crate::g17::power::LIGHT_DEFAULT), core_layout: CoreLayout { cores: Some((4, 40)), alternate_core_masks: None, diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index 4ea3862efd4cb..b76af3f52743f 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -70,6 +70,7 @@ pub(crate) const CONFIG: Config = Config { variant: &crate::g17::fw::variant::V26A428_M5, config: None, }]), + light_effort: Some(crate::g17::power::LIGHT_DEFAULT), // Two clusters of ten cores. The buffers are sized for every core slot of the board, so the // board's cluster and slot counts admit the GPU, whichever cores are fused off. core_layout: CoreLayout { @@ -141,6 +142,7 @@ pub(crate) const CONFIG_MAX: Config = Config { variant: &crate::g17::fw::variant::V26A428_M5, config: None, }]), + light_effort: Some(crate::g17::power::LIGHT_DEFAULT), // Four clusters of ten cores. The buffers are sized for every core slot of the board, so the // board's cluster and slot counts admit the GPU, whichever cores are fused off. core_layout: CoreLayout { @@ -226,6 +228,7 @@ pub(crate) const CONFIG_ULTRA: Config = Config { variant: &crate::g17::fw::variant::V26A428_M5, config: None, }]), + light_effort: Some(crate::g17::power::LIGHT_DEFAULT), // Eight clusters of ten cores. M5 renders lay out their tiler context store themselves // (`fw::m5::render_registers`), so the deflake layout is the M5 Pro's. As on the M5 Pro, the // buffers are sized for all 80 core slots and the board admits the GPU by clusters and slots: diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 0165140565567..962be96e9fcde 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -59,6 +59,7 @@ pub(crate) const CONFIG: Config = Config { fabric_bridge: true, sleep_restart: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), + light_effort: Some(crate::g17::power::LIGHT_DEFAULT), // The layouts the A18 Pro has always run with. core_layout: crate::g17::CoreLayout { cores: None, diff --git a/drivers/gpu/drm/asahi/hw/t8142.rs b/drivers/gpu/drm/asahi/hw/t8142.rs index 26fe2e49abcdb..cbcb51c26ed72 100644 --- a/drivers/gpu/drm/asahi/hw/t8142.rs +++ b/drivers/gpu/drm/asahi/hw/t8142.rs @@ -48,6 +48,7 @@ pub(crate) const PROBE: Config = Config { recovery_slot_keys: None, gpu_generation: GpuGen::G17, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428_M5), + light_effort: Some(crate::g17::power::LIGHT_DEFAULT), core_layout: CoreLayout { cores: None, alternate_core_masks: None, diff --git a/drivers/gpu/drm/asahi/usage.rs b/drivers/gpu/drm/asahi/usage.rs index aa817d2d41768..9aebd85b8be9b 100644 --- a/drivers/gpu/drm/asahi/usage.rs +++ b/drivers/gpu/drm/asahi/usage.rs @@ -11,6 +11,11 @@ use core::sync::atomic::{AtomicU64, Ordering}; +/// Engine time charged across all clients, in ns, for the GPU busy share. +pub(crate) static TOTAL_NS: AtomicU64 = AtomicU64::new(0); +/// Longest single stage charged since the governor last read it, in ns. +pub(crate) static MAX_NS: AtomicU64 = AtomicU64::new(0); + #[derive(Clone, Copy)] pub(crate) enum Engine { Vertex, @@ -38,6 +43,8 @@ impl Client { return; } let ns = ticks_to_ns(end - start, frequency_hz); + TOTAL_NS.fetch_add(ns, Ordering::Relaxed); + MAX_NS.fetch_max(ns, Ordering::Relaxed); let counter = &self.ns[engine as usize]; let mut old = counter.load(Ordering::Relaxed); loop { diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index bdf5331d3ff3a..9e867b8628c9e 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -77,13 +77,17 @@ enum apple_clpc_rail { #define APPLE_CLPC_REASON_HOLD BIT(4) #define APPLE_CLPC_REASON_ANE BIT(5) +/* A lower GPU busy level must last this long; a higher one applies at once. */ +#define APPLE_CLPC_GPU_DOWN_MS 400 + /** * struct apple_clpc_hw - per-SoC CLPC parameters * @rail: PMC rail of the DCS and of the fabric * @sustain: consecutive busy samples before a cluster requests a state; * 0 or 1 for the first * @map: state of each rail for a busy cluster below 25, 50, 75 and 100 - * percent of its maximum frequency + * percent of its maximum frequency; the node's apple,cpu-memory-states + * where it has them (apple_clpc_dt_hw()) * @gpu_demand: streaming GPU work requests @gpu * @gpu: state of each rail for streaming GPU work * @gpu_sustain_ms: time graphics stays powered without a break before it @@ -101,7 +105,7 @@ enum apple_clpc_rail { * measured on; SoCs sharing this CLPC without it get no cluster floor * @gpu_input: the GPU driver reports its work in flight through * apple_clpc_gpu_get(); while there is some, both rails go to the top - * of @map + * of @map, or to @gpu_busy if the driver reports a busy share * @idle_on_time: sample every period even with no vote held, instead of * from a deferrable timer * @low_power_khz: CPU frequency ceiling of the low-power platform profile, @@ -110,6 +114,20 @@ enum apple_clpc_rail { * sharing this CLPC without it get no low-power profile. NULL for any * @low_power_mw: package power (the SMC's PHPC key) the low-power profile * holds the CPU clusters to; 0 for none + * @gpu_busy: state of each rail for reported GPU work below @gpu_busy_mid, + * @gpu_busy_high and @gpu_busy_top and at or above it; all zero to always + * use the top of @map. From apple,gpu-busy-memory-states and + * apple,gpu-busy-permille where the node has them + * @gpu_busy_mid: GPU busy share in permille (apple_clpc_gpu_set_busy()) + * from which reported GPU work gets the second column of @gpu_busy + * @gpu_busy_high: busy share from which it gets the third column + * @gpu_busy_top: busy share from which it gets the top column, as does a + * GPU that reports no share. The share is time spent executing at + * whatever clock the GPU runs, not bandwidth: a 120 fps game reads + * about a third, a scrolling browser a few percent. + * @level_khz: frequency @map's quarters are taken from, so that clusters of + * different speed share them; 0 for each cluster's own maximum. From + * apple,cpu-level-mhz where the node has it * @high_power_machine: the machine compatible on which the performance * profile tells the SMC that macOS's High Power mode is on (the zETM * record below); NULL for none @@ -125,6 +143,11 @@ struct apple_clpc_hw { unsigned int pmp_agent; unsigned int gpu_dense; bool gpu_input; + u8 gpu_busy[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; + unsigned int gpu_busy_mid; + unsigned int gpu_busy_high; + unsigned int gpu_busy_top; + unsigned int level_khz; u8 ane[APPLE_CLPC_RAILS]; unsigned int ane_cluster; unsigned int ane_cluster_khz; @@ -160,6 +183,11 @@ struct apple_clpc_gpu { bool active; u64 updated_ns; u64 sequence; + /* Busy share of the last window in permille, if the GPU reports one */ + unsigned int busy; + bool busy_valid; + unsigned int level; + unsigned long level_since; struct delayed_work *wake; }; @@ -271,12 +299,35 @@ void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) if (active && !gpu->active && gpu->wake) mod_delayed_work(system_freezable_power_efficient_wq, gpu->wake, 0); gpu->active = active; + /* A new burst after idle starts from the top, not from an old share. */ + if (!active) + gpu->busy_valid = false; gpu->updated_ns = ktime_get_mono_fast_ns(); gpu->sequence++; spin_unlock_irqrestore(&gpu->lock, flags); } EXPORT_SYMBOL_GPL(apple_clpc_gpu_set_active); +/** + * apple_clpc_gpu_set_busy() - Report the GPU busy share of the last window + * @gpu: the GPU input, from apple_clpc_gpu_get() + * @permille: share of the window the GPU spent executing work, 0..1000 + * + * Scales the floor that reported GPU work gets, see &struct apple_clpc_hw.gpu_busy. + */ +void apple_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille) +{ + unsigned long flags; + + if (!gpu) + return; + spin_lock_irqsave(&gpu->lock, flags); + gpu->busy = min(permille, 1000U); + gpu->busy_valid = true; + spin_unlock_irqrestore(&gpu->lock, flags); +} +EXPORT_SYMBOL_GPL(apple_clpc_gpu_set_busy); + void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu) { if (gpu) { @@ -344,6 +395,36 @@ int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) } EXPORT_SYMBOL_GPL(apple_clpc_gpu_get); +/* + * The column of @hw->gpu_busy for reported GPU work, from its busy share. + * Called once per CLPC sample: a higher column applies at once, a lower one + * at the first sample at least APPLE_CLPC_GPU_DOWN_MS after the last one + * that asked for the current column or a higher one. + */ +static unsigned int apple_clpc_gpu_level(const struct apple_clpc_hw *hw, + struct apple_clpc_gpu *gpu) +{ + unsigned long flags; + unsigned int level; + + spin_lock_irqsave(&gpu->lock, flags); + if (!gpu->busy_valid || gpu->busy >= hw->gpu_busy_top) + level = APPLE_CLPC_LEVELS - 1; + else if (gpu->busy >= hw->gpu_busy_high) + level = 2; + else + level = gpu->busy >= hw->gpu_busy_mid ? 1 : 0; + if (level >= gpu->level || + time_after_eq(jiffies, gpu->level_since + + msecs_to_jiffies(APPLE_CLPC_GPU_DOWN_MS))) { + gpu->level = level; + gpu->level_since = jiffies; + } + level = gpu->level; + spin_unlock_irqrestore(&gpu->lock, flags); + return level; +} + static bool apple_clpc_gpu_active(struct apple_clpc_gpu *gpu, u64 *updated, u64 *sequence) { unsigned long flags; @@ -550,7 +631,8 @@ static void apple_clpc_sample_cluster(struct apple_clpc *clpc, policy = cpufreq_cpu_get(cl->policy_cpu); if (policy) { if (policy->cpuinfo.max_freq) - cl->level = policy->cur * 100 / policy->cpuinfo.max_freq; + cl->level = policy->cur * 100 / + (clpc->hw->level_khz ?: policy->cpuinfo.max_freq); cpufreq_cpu_put(policy); } @@ -727,22 +809,26 @@ static void apple_clpc_gpu_floor_count(struct apple_clpc *clpc, bool released) /* * The GPU driver's input, where it drives demand instead of the GFX power - * state: every rail at the top of @map, or at @gpu where that is higher, - * while the driver reports work. + * state: while the driver reports work, every rail at its top state, or at + * the gpu_busy state for the busy share the driver reports. */ static bool apple_clpc_gpu_input_demand(const struct apple_clpc_hw *hw, struct apple_clpc_gpu *gpu, unsigned int *demand) { u64 updated, sequence; - unsigned int rail; + unsigned int level, rail; if (!gpu || hw->gpu_demand || !apple_clpc_gpu_active(gpu, &updated, &sequence)) return false; + level = apple_clpc_gpu_level(hw, gpu); for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - demand[rail] = max(demand[rail], - max(hw->map[rail][APPLE_CLPC_LEVELS - 1], hw->gpu[rail])); + demand[rail] = max_t(unsigned int, demand[rail], + max_t(unsigned int, hw->gpu[rail], + hw->gpu_busy[APPLE_CLPC_DCS][APPLE_CLPC_LEVELS - 1] ? + hw->gpu_busy[rail][level] : + hw->map[rail][APPLE_CLPC_LEVELS - 1])); return true; } @@ -1035,9 +1121,21 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) READ_ONCE(clpc->last_floor), READ_ONCE(clpc->last_reason), READ_ONCE(clpc->last_vote_ns)); if (clpc->gpu) { + struct apple_clpc_gpu *gpu = clpc->gpu; + unsigned int busy, level; + unsigned long flags; u64 updated, sequence; - bool active = apple_clpc_gpu_active(clpc->gpu, &updated, &sequence); - + bool active = apple_clpc_gpu_active(gpu, &updated, &sequence); + bool busy_valid; + + /* Show the level the last sample chose; reading must not move it. */ + spin_lock_irqsave(&gpu->lock, flags); + busy = gpu->busy; + busy_valid = gpu->busy_valid; + level = gpu->level; + spin_unlock_irqrestore(&gpu->lock, flags); + seq_printf(s, "host_gpu busy: %u permille (%s) level %u\n", + busy, busy_valid ? "reported" : "none", level); seq_printf(s, "host_gpu: active %u request_ns %llu sequence %llu\n", active, updated, sequence); } @@ -1349,6 +1447,96 @@ static const struct platform_profile_ops apple_clpc_profile_ops = { .profile_set = apple_clpc_profile_set, }; +/* Fills @map from a DT matrix: APPLE_CLPC_RAILS rows of APPLE_CLPC_LEVELS states. */ +static int apple_clpc_dt_map(u8 map[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS], + const u32 *cells, int n) +{ + unsigned int rail, level; + + if (n != APPLE_CLPC_RAILS * APPLE_CLPC_LEVELS) + return -EINVAL; + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) { + for (level = 0; level < APPLE_CLPC_LEVELS; level++) { + u32 state = cells[rail * APPLE_CLPC_LEVELS + level]; + + if (state > U8_MAX) + return -EINVAL; + map[rail][level] = state; + } + } + return 0; +} + +/* The busy shares from which reported GPU work gets the 2nd, 3rd and top column. */ +static int apple_clpc_dt_busy(struct apple_clpc_hw *hw, const u32 *cells, int n) +{ + if (n != 3 || cells[0] > cells[1] || cells[1] > cells[2] || cells[2] > 1000) + return -EINVAL; + hw->gpu_busy_mid = cells[0]; + hw->gpu_busy_high = cells[1]; + hw->gpu_busy_top = cells[2]; + return 0; +} + +/* Reads up to @max cells of @name; returns how many, or an error. */ +static int apple_clpc_dt_cells(struct device_node *np, const char *name, + u32 *cells, int max) +{ + int n = of_property_count_u32_elems(np, name); + + if (n < 0) + return n; + if (n > max) + return -EOVERFLOW; + return of_property_read_u32_array(np, name, cells, n) ?: n; +} + +/* + * The compatible's table, with the memory states the node gives in place of + * its own (apple,pmc.yaml). The states are measured on a SoC's memory, so + * the DT says which a board has: the M4 Max's come from t6041.dtsi, and the + * M4 Pro, which shares its compatible, keeps the table's. + */ +static const struct apple_clpc_hw *apple_clpc_dt_hw(struct device *dev, + const struct apple_clpc_hw *soc) +{ + u32 cells[APPLE_CLPC_RAILS * APPLE_CLPC_LEVELS]; + struct device_node *np = dev->of_node; + struct apple_clpc_hw *hw; + bool cpu, gpu; + u32 mhz; + int n; + + cpu = of_property_present(np, "apple,cpu-memory-states"); + gpu = of_property_present(np, "apple,gpu-busy-memory-states") || + of_property_present(np, "apple,gpu-busy-permille"); + if (!cpu && !gpu && !of_property_present(np, "apple,cpu-level-mhz")) + return soc; + hw = devm_kmemdup(dev, soc, sizeof(*soc), GFP_KERNEL); + if (!hw) + return ERR_PTR(-ENOMEM); + if (cpu) { + n = apple_clpc_dt_cells(np, "apple,cpu-memory-states", cells, ARRAY_SIZE(cells)); + n = n < 0 ? n : apple_clpc_dt_map(hw->map, cells, n); + if (n) + return ERR_PTR(n); + } + if (gpu) { + n = apple_clpc_dt_cells(np, "apple,gpu-busy-memory-states", cells, + ARRAY_SIZE(cells)); + n = n < 0 ? n : apple_clpc_dt_map(hw->gpu_busy, cells, n); + if (n) + return ERR_PTR(n); + n = apple_clpc_dt_cells(np, "apple,gpu-busy-permille", cells, ARRAY_SIZE(cells)); + n = n < 0 ? n : apple_clpc_dt_busy(hw, cells, n); + if (n) + return ERR_PTR(n); + } + if (!of_property_read_u32(np, "apple,cpu-level-mhz", &mhz)) + hw->level_khz = mhz * 1000; + return hw; +} + static int apple_clpc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1363,7 +1551,10 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->dev = dev; clpc->pmc = dev->parent; - clpc->hw = of_device_get_match_data(dev); + clpc->hw = apple_clpc_dt_hw(dev, of_device_get_match_data(dev)); + if (IS_ERR(clpc->hw)) + return dev_err_probe(dev, PTR_ERR(clpc->hw), + "invalid memory states in the device tree\n"); /* T6040 firmware keeps GFX powered for its whole lifetime. A separate * host publication/idle input, not that supplier state, drives demand; * the T8132 and M5 (T8142) GPU drivers report through the same input. diff --git a/include/linux/soc/apple/clpc.h b/include/linux/soc/apple/clpc.h index 0a4c4f439ac56..5f83b2aedc609 100644 --- a/include/linux/soc/apple/clpc.h +++ b/include/linux/soc/apple/clpc.h @@ -12,6 +12,7 @@ struct apple_clpc_ane; #if IS_REACHABLE(CONFIG_APPLE_CLPC) int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out); void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); +void apple_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille); void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu); /* Optional, single-publisher input: success with NULL means no provider. */ int apple_clpc_ane_get(struct device *dev, struct apple_clpc_ane **out); @@ -24,6 +25,7 @@ static inline int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu * return device_property_present(dev, "apple,clpc") ? -ENODEV : 0; } static inline void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) {} +static inline void apple_clpc_gpu_set_busy(struct apple_clpc_gpu *gpu, unsigned int permille) {} static inline void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu) {} /* ANE remains usable when the optional floor provider is not reachable. */ diff --git a/tools/asahi/clpc-gpu-demand-tests.py b/tools/asahi/clpc-gpu-demand-tests.py index 2e852e6f91643..4a8f9282ac1bd 100644 --- a/tools/asahi/clpc-gpu-demand-tests.py +++ b/tools/asahi/clpc-gpu-demand-tests.py @@ -26,10 +26,26 @@ def main(): rail_shift = re.search(r'^#define APPLE_PMC_RAIL_SHIFT\(rail\).*$', header, re.M).group(0) dense_samples = re.search(r'^#define APPLE_CLPC_GPU_DENSE_SAMPLES\s.*$', source, re.M).group(0) reasons = '\n'.join(re.findall(r'^#define APPLE_CLPC_REASON_\w+\s.*$', source, re.M)) + gpu_down = re.search(r'^#define APPLE_CLPC_GPU_DOWN_MS\s.*$', source, re.M).group(0) types = '\n'.join(item(source, s) + ';' for s in [ 'struct apple_clpc_hw {', 'struct apple_clpc_vote {']) tables = '\n'.join(item(source, 'static const struct apple_clpc_hw ' + n) + ';' for n in ['apple_clpc_hw_t8132', 'apple_clpc_hw_t8140', 'apple_clpc_hw_t6040']) + # T6041 is the T6040 table with the memory states its device tree gives. + dts = (ROOT / 'arch/arm64/boot/dts/apple/t6041.dtsi').read_text() + clpc_node = item(dts, '&clpc {') + + def cells(name): + value = re.search(r'\s' + re.escape(name) + r'\s*=\s*([^;]*);', clpc_node).group(1) + return [int(x, 0) for x in re.findall(r'0x[0-9a-fA-F]+|\d+', value)] + + def c_array(name): + return '{' + ', '.join(str(x) for x in cells(name)) + '}' + t6041_dt = ( + 'static const u32 t6041_cpu_states[] = ' + c_array('apple,cpu-memory-states') + ';\n' + 'static const u32 t6041_gpu_states[] = ' + c_array('apple,gpu-busy-memory-states') + ';\n' + 'static const u32 t6041_gpu_busy[] = ' + c_array('apple,gpu-busy-permille') + ';\n' + 'static const u32 t6041_level_mhz = ' + str(cells('apple,cpu-level-mhz')[0]) + ';\n') methods = '\n'.join(item(source, s) for s in [ 'static unsigned int apple_clpc_vote(', 'static u32 apple_clpc_floor(', 'static bool apple_clpc_gpu_streaming(', @@ -37,6 +53,9 @@ def main(): 'static bool apple_clpc_gpu_dense(', 'static bool apple_clpc_gpu_limited(', 'static void apple_clpc_gpu_floor_count(', + 'static unsigned int apple_clpc_gpu_level(', + 'static int apple_clpc_dt_map(', + 'static int apple_clpc_dt_busy(', 'static bool apple_clpc_gpu_input_demand(', 'static bool apple_clpc_ane_input_demand(', 'static void apple_clpc_ane_cluster(', @@ -56,6 +75,9 @@ def main(): #define APPLE_CLPC_GPU_BACKOFF_MS 100 #define HZ 1000 #define max(a,b) ((a) > (b) ? (a) : (b)) +#define max_t(t,a,b) ((t)(a) > (t)(b) ? (t)(a) : (t)(b)) +#define U8_MAX 255 +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) #define GENMASK(h,l) (((1u << ((h)-(l)+1))-1) << (l)) #define BIT(n) (1u << (n)) #define APPLE_PMC_RAIL_SHIFT(r) ((r)*8) @@ -67,7 +89,10 @@ def main(): #define dev_warn_once(d, ...) ((void)(d), warned++) enum apple_clpc_rail { APPLE_CLPC_DCS, APPLE_CLPC_SOC, APPLE_CLPC_RAILS }; ''' + types + r''' -struct apple_clpc_gpu { bool active; u64 sequence; }; +struct apple_clpc_gpu { + int lock; bool active; u64 sequence; + unsigned int busy; bool busy_valid; unsigned int level; unsigned long level_since; +}; struct apple_clpc_ane { int lock; bool active, pending; }; #define spin_lock_irqsave(l, f) ((void)(l), (f) = 0) #define spin_unlock_irqrestore(l, f) ((void)(l), (void)(f)) @@ -134,7 +159,30 @@ def main(): static void apple_clpc_queue(struct apple_clpc *c, bool idle) { (void)c; queued_idle=idle; } #define READ_ONCE(x) (x) static void apple_clpc_profile_limit(struct apple_clpc *c) { (void)c; } -''' + dense_samples + '\n' + reasons + '\n' + tables + '\n' + methods + r''' +''' + dense_samples + '\n' + reasons + '\n' + gpu_down + '\n' + tables + '\n' + methods + '\n' + t6041_dt + r''' +// T6041's table: T6040's, with the states from t6041.dtsi through the driver's DT parsers. +// They must be the J616c measurements the T6041 floors were tuned with, and bad cells are refused. +static struct apple_clpc_hw apple_clpc_hw_t6041; +static void t6041_from_dt(void) { + static const u8 cpu[2][4] = {{1, 1, 3, 4}, {0, 0, 0, 0}}, gpu[2][4] = {{1, 2, 4, 5}, {0, 0, 0, 2}}; + struct apple_clpc_hw bad = apple_clpc_hw_t6040; + u32 cells[8] = {1, 1, 3, 4, 0, 0, 0, 256}, busy[3] = {500, 160, 800}; + apple_clpc_hw_t6041 = apple_clpc_hw_t6040; + assert(!apple_clpc_dt_map(apple_clpc_hw_t6041.map, t6041_cpu_states, ARRAY_SIZE(t6041_cpu_states))); + assert(!apple_clpc_dt_map(apple_clpc_hw_t6041.gpu_busy, t6041_gpu_states, ARRAY_SIZE(t6041_gpu_states))); + assert(!apple_clpc_dt_busy(&apple_clpc_hw_t6041, t6041_gpu_busy, ARRAY_SIZE(t6041_gpu_busy))); + apple_clpc_hw_t6041.level_khz = t6041_level_mhz * 1000; + for (int r = 0; r < 2; r++) + for (int l = 0; l < 4; l++) + assert(apple_clpc_hw_t6041.map[r][l] == cpu[r][l] && apple_clpc_hw_t6041.gpu_busy[r][l] == gpu[r][l]); + assert(apple_clpc_hw_t6041.level_khz == 4512000); + assert(apple_clpc_dt_map(bad.map, cells, 8) == -EINVAL); + assert(apple_clpc_dt_map(bad.map, cells, 7) == -EINVAL); + assert(apple_clpc_dt_busy(&bad, busy, 3) == -EINVAL); + busy[0] = 160; busy[1] = 500; busy[2] = 1001; + assert(apple_clpc_dt_busy(&bad, busy, 3) == -EINVAL); + assert(apple_clpc_dt_busy(&bad, busy, 2) == -EINVAL); +} // T8140 ANE work holds its DRAM and fabric states while it runs, a request // shorter than one sample is still seen once, and the hold then expires. static void check_t8140_ane(void) { @@ -171,6 +219,46 @@ def main(): ane.active=true; jiffies=2040; apple_clpc_sample(&c); assert(qos_adds==1 && qos_min==3756000); } +// T6041 GPU work takes its floor from the busy share the driver reports: +// DCS 1, 2, 4 or 5 and fabric 0, 0, 0 or 2, split at 160, 500 and 800 +// permille. A higher level applies at once, a lower one at the first sample +// at least 400 ms after the last one that asked for the current level or more. +static void check_t6041_busy(void) { + struct apple_clpc_gpu gpu={0}; + struct apple_clpc c={.hw=&apple_clpc_hw_t6041,.agent=1,.mask=3,.gpu=&gpu}; + assert(c.hw->gpu_busy_mid==160 && c.hw->gpu_busy_high==500 && c.hw->gpu_busy_top==800); + read_error=vote_error=0; pmgr_status=0xf0; pmp_floor=0; published=~0u; + // No share reported: the top of the table, as before the busy share. + gpu.active=true; jiffies=3000; apple_clpc_sample(&c); + assert(published==0x25 && c.last_reason==2 && gpu.level==3); + gpu.active=false; jiffies=3100; apple_clpc_sample(&c); assert(published==0); + // Below gpu_busy_mid: the lowest GPU state (after a long idle, at once). + gpu.active=true; gpu.busy_valid=true; gpu.busy=100; jiffies=3500; + apple_clpc_sample(&c); assert(published==0x01 && c.last_reason==2 && gpu.level==0); + // Between the thresholds, going up: each level applies at the next sample. + gpu.busy=160; jiffies=3504; apple_clpc_sample(&c); assert(published==0x02 && gpu.level==1); + gpu.busy=499; jiffies=3508; apple_clpc_sample(&c); assert(published==0x02 && gpu.level==1); + gpu.busy=500; jiffies=3512; apple_clpc_sample(&c); assert(published==0x04 && gpu.level==2); + gpu.busy=799; jiffies=3516; apple_clpc_sample(&c); assert(published==0x04 && gpu.level==2); + gpu.busy=800; jiffies=3520; apple_clpc_sample(&c); assert(published==0x25 && gpu.level==3); + // Going down waits 400 ms from the last sample at the top. + gpu.busy=100; + for (jiffies=3524; jiffies<3920; jiffies+=4) { + apple_clpc_sample(&c); assert(published==0x25 && gpu.level==3); + } + jiffies=3919; apple_clpc_sample(&c); assert(published==0x25 && gpu.level==3); + // At 400 ms the DCS drops at once; the fabric state keeps HOLD_MS. + jiffies=3920; apple_clpc_sample(&c); assert(published==0x21 && gpu.level==0); + jiffies=3952; apple_clpc_sample(&c); assert(published==0x01 && c.last_reason==2); + // A sample that asks for the current level again restarts the 400 ms. + gpu.busy=600; jiffies=4000; apple_clpc_sample(&c); assert(published==0x04 && gpu.level==2); + gpu.busy=100; jiffies=4200; apple_clpc_sample(&c); assert(gpu.level==2); + gpu.busy=550; jiffies=4300; apple_clpc_sample(&c); assert(gpu.level==2); + gpu.busy=100; jiffies=4699; apple_clpc_sample(&c); assert(published==0x04 && gpu.level==2); + jiffies=4700; apple_clpc_sample(&c); assert(published==0x01 && gpu.level==0); + gpu.active=false; jiffies=4800; apple_clpc_sample(&c); assert(published==0 && queued_idle); +} + // T8132's G16 firmware keeps graphics powered while the GPU idles: the // M5/F3 floor follows the work the GPU driver reports, as on T6040. static void check_t8132(void) { @@ -268,13 +356,15 @@ def main(): assert(published==0x12 && c.last_reason==1); c.clusters[0].request[0]=c.clusters[0].request[1]=0; jiffies=264; apple_clpc_sample(&c); assert(published==0); + t6041_from_dt(); + check_t6041_busy(); check_t8132(); check_read_error(); check_t8140(); check_t8140_dense(); check_t8140_ane(); check_t6040_ane(); - puts("PASS: T6040 and T8132 host demand; T8140 ANE, streaming, dense work, read errors and PMP backoff; idle release"); + puts("PASS: T6040 and T8132 host demand; T6041 memory states from its DT, GPU busy share and down-step hold; T8140 ANE, streaming, dense work, read errors and PMP backoff; idle release"); } ''' code = code.replace('#define APPLE_PMC_RAIL_SHIFT(r) ((r)*8)', rail_shift) @@ -299,7 +389,14 @@ def main(): ('omit-pmp-backoff', code.replace('bool released = apple_clpc_gpu_limited(clpc, now);', 'bool released = (apple_clpc_gpu_limited(clpc, now), false);')), ('release-too-early', code.replace('msecs_to_jiffies(APPLE_CLPC_HOLD_MS)', '0')), + ('gpu-down-at-once', code.replace('msecs_to_jiffies(APPLE_CLPC_GPU_DOWN_MS)', '0')), + ('gpu-busy-ignored', code.replace('if (!gpu->busy_valid || gpu->busy >= hw->gpu_busy_top)', + 'if (true || gpu->busy >= hw->gpu_busy_top)')), + ('gpu-no-share-uses-table', code.replace('if (!gpu->busy_valid || gpu->busy >= hw->gpu_busy_top)', + 'if (gpu->busy >= hw->gpu_busy_top)')), ('retain-at-idle', code.replace('vote->state = 0;', '')), + ('dt-state-unchecked', code.replace('if (state > U8_MAX)', 'if (false)')), + ('dt-busy-unordered', code.replace('cells[0] > cells[1] || ', '')), ]: # A mutation that no longer matches the driver tests nothing. assert name == 'fixed' or text != code, 'stale mutation: ' + name