diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index f6b5decff53da..f383d15a104cf 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -139,8 +139,17 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) */ wait_for_completion_timeout(&cpu_running, msecs_to_jiffies(5000)); - if (cpu_online(cpu)) + if (cpu_online(cpu)) { + /* + * The CPU has taken its task. Cleared, as kernels before 5.x did, so a core that + * reaches __secondary_switched without a bring-up of its own waits in + * __secondary_too_slow instead of running as this CPU a second time. + */ + WRITE_ONCE(secondary_data.task, NULL); + dcache_clean_inval_poc((unsigned long)&secondary_data, + (unsigned long)&secondary_data + sizeof(secondary_data)); return 0; + } pr_crit("CPU%u: failed to come online\n", cpu); secondary_data.task = NULL; diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index e66806dbbf9c5..0d0f200e3de6a 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -222,6 +222,9 @@ int spin_table_hold_parked(void) int us, ret; unsigned int cpu; + /* No CPU restarted from here to the wake may pass the pen without its own bring-up. */ + write_pen_release(INVALID_HWID); + for_each_present_cpu(cpu) WRITE_ONCE(spin_table_hold_error[cpu], 0); WRITE_ONCE(spin_table_hold_generation, READ_ONCE(spin_table_hold_generation) + 1); @@ -331,6 +334,8 @@ void spin_table_stop_finished(void) /* Before any CPU is started again: the pen must let them boot. */ if (secondary_pen_deep) spin_table_pen_deep_set(0); + /* ...but only the one its bring-up releases (cores started early wait in the pen). */ + write_pen_release(INVALID_HWID); WRITE_ONCE(spin_table_hold_requested, false); WRITE_ONCE(spin_table_stop_requested, false); WRITE_ONCE(spin_table_stop_released, false); @@ -473,6 +478,14 @@ static int smp_rvbar_spin_table_cpu_kill(unsigned int cpu) static void smp_rvbar_spin_table_postboot(void) { + /* + * Close the pen behind this CPU. Left at its MPIDR, the next restart at the reset vector + * (after a sleep's commit, or a platform that starts every core at once at the wake) + * lets it straight through the pen into secondary_startup ahead of its own bring-up, + * with the secondary_data of its last boot: it then runs as that CPU with no STARTING + * callbacks run, e.g. without a tick device. + */ + write_pen_release(INVALID_HWID); cpumask_clear_cpu(smp_processor_id(), &spin_table_cpus_off); if (READ_ONCE(spin_table_power) && spin_table_power->cpu_restore) spin_table_power->cpu_restore(read_cpuid_mpidr() & MPIDR_HWID_BITMASK); diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index ce1bcee755574..ec06d5602c08f 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -813,6 +813,7 @@ struct apple_sleep { void __iomem *ps; void __iomem *pwrgate; void __iomem *cpu_start; + unsigned int cpu_burst_n; /* cores the last wake's burst started (cpu_burst) */ void __iomem *cllt; void __iomem *cluster; void __iomem *commit; @@ -2234,9 +2235,43 @@ MODULE_PARM_DESC(mem_restore, "T8142: put the memory side (AMCC, SoC NI, DCS) ba static bool sleep_clk_restore = true; module_param_named(clk_restore, sleep_clk_restore, bool, 0644); MODULE_PARM_DESC(clk_restore, "T8142: put the 126 PMGR clkgen slots and the memory clock word back at the wake as XNU does"); -static bool sleep_perf_down = true; +/* + * Default N: a traced macOS 27.2 run on J813 (boot, a 40 s sleep, the wake and three more entries) + * never touches the perf-state page 0x380024000, though it was hooked. Y keeps the earlier + * behaviour. + */ +static bool sleep_perf_down; module_param_named(perf_down, sleep_perf_down, bool, 0644); -MODULE_PARM_DESC(perf_down, "T8142: SoC/DCS perf states down before the memory stop (macOS on J813 never writes them; A/B)"); +MODULE_PARM_DESC(perf_down, + "T8142: SoC/DCS perf states down before the memory stop (macOS never writes them; default N)"); + +/* + * The clusters' words at the commit as XNU writes them. In the same trace the first sleep after + * boot reads 0xf and writes 0; every commit after a wake (three of three) reads 0x2f and writes + * 0x20: only the low nibble cleared, bit 5 kept. Y (default): that read-modify-write; N: write 0, + * as before. + */ +static bool sleep_cluster_rmw = true; +module_param_named(cluster_rmw, sleep_cluster_rmw, bool, 0644); +MODULE_PARM_DESC(cluster_rmw, + "T8142: clear only the clusters' on bits at the commit, keep the rest, as XNU (default Y; N = write 0)"); + +/* + * The cores' start after the wake as XNU does it. In the same trace, 16.5 ms after the wake, right + * after the memory side and the WD2 stop, the boot core starts all nine other cores in one 0.23 ms + * burst, three words each: +4 the core's bit, +8 the E core's bit or 0, +0xc the P core's bit or + * 0. Linux started each core from its own bring-up, later and one by one, with two words. Y + * (default): the burst at the end of the wake, for the cores that were online when the sleep + * began; they come up through the boot object into the kernel's holding pen and wait there, and + * each bring-up only releases its core. N: the per-core start. + */ +static bool sleep_cpu_burst = true; +module_param_named(cpu_burst, sleep_cpu_burst, bool, 0644); +MODULE_PARM_DESC(cpu_burst, + "T8142: start the other cores in one burst at the end of the wake, as XNU (default Y; N = per core at bring-up)"); +static struct cpumask sleep_cpus_burst; +/* The CPUs online at .begin: the ones the PM core brings back after the wake. */ +static struct cpumask sleep_cpus_awake; static u64 t8142_boot_mpidr = ~0ULL; extern char t8142_el1_wfi_stub[], t8142_el1_wfi_stub_end[]; @@ -2497,6 +2532,15 @@ static int sleep_cpu_on(u64 mpidr) u64 flag0 = 0, flag1 = 0; unsigned int us = 0; + /* Started by the wake's burst (apple_sleep.cpu_burst): it waits in the pen; release it. */ + if (cpu >= 0 && cpumask_test_and_clear_cpu(cpu, &sleep_cpus_burst)) { + sleep_cpu_start_words(s, after); + pr_info("apple-sleep: cpu%d start %#llx: started in the wake burst (%u cores), released from the pen; stop/sys/e/p now %x/%x/%x/%x; aura slot flag %llu\n", + cpu, mpidr, s->cpu_burst_n, after[0], after[1], after[2], after[3], + slot ? sleep_cpu_slot_read(slot, 8) : 0ULL); + return 0; + } + if (trace) { sleep_cpu_start_words(s, before); if (slot) @@ -6338,6 +6382,8 @@ static int apple_sleep_begin(suspend_state_t state) /* The PM core also calls .begin for s2idle when no s2idle ops are installed. */ if (state != PM_SUSPEND_MEM) return 0; + /* Before the PM core takes the other CPUs down: the wake's burst starts only these. */ + cpumask_copy(&sleep_cpus_awake, cpu_online_mask); ret = apple_sleep_profile_check(apple_sleep); if (ret) @@ -8960,8 +9006,11 @@ static void t8142_last_mile(struct apple_sleep *s) t8142_commit_note(s); - for (i = 0; i < soc->ncluster_cpu; i++) - writel(0, s->cluster + soc->cluster_cpu[i]); + for (i = 0; i < soc->ncluster_cpu; i++) { + void __iomem *reg = s->cluster + soc->cluster_cpu[i]; + + writel(READ_ONCE(sleep_cluster_rmw) ? readl(reg) & ~CLUSTER_ON : 0, reg); + } s->quiesce_still_on = 0; for (i = 0; i < soc->quiesce.count; i++) @@ -9089,6 +9138,35 @@ static int t8142_cluster_on(struct apple_sleep *s, u32 word, void __iomem *acc) * word, map 0x4e's ten words) are gated by flags its constructor clears and nothing sets. * Then T8132's: ATC0's words, SEP, WD2, ANS's PCIe block and the radio's link. */ +/* apple_sleep.cpu_burst: the other cores started at once, three words each, as XNU. */ +static void t8142_cpu_burst(struct apple_sleep *s) +{ + unsigned int cpu, n = 0, self = smp_processor_id(); + + cpumask_clear(&sleep_cpus_burst); + s->cpu_burst_n = 0; + /* T8142's data states no cpu_clusters (0): E = MPIDR cluster 0, P = 1, from the MPIDR. */ + if (!READ_ONCE(sleep_cpu_burst) || !s->cpu_start) + return; + for_each_cpu(cpu, &sleep_cpus_awake) { + u64 mpidr = cpu_logical_map(cpu); + unsigned int cluster = sleep_cpu_cluster(s, mpidr), core = sleep_cpu_core(mpidr); + + if (cpu == self || cluster > 1) + continue; + writel(sleep_cpu_start_bit(s, mpidr), s->cpu_start + CPU_START_SYS); + writel(cluster == 0 ? BIT(core) : 0, s->cpu_start + CPU_START_CLUSTER(0)); + writel(cluster == 1 ? BIT(core) : 0, s->cpu_start + CPU_START_CLUSTER(1)); + cpumask_set_cpu(cpu, &sleep_cpus_burst); + n++; + } + s->cpu_burst_n = n; + pr_info("apple-sleep: wake: %u cores started in one burst (cpu_burst=Y, XNU's +4/+8/+0xc); stop/sys/e/p now %x/%x/%x/%x\n", + n, readl(s->cpu_start + CPU_START_STOP), readl(s->cpu_start + CPU_START_SYS), + readl(s->cpu_start + CPU_START_CLUSTER(0)), + readl(s->cpu_start + CPU_START_CLUSTER(1))); +} + static void t8142_wake(struct apple_sleep *s) { if (t8142_alarm_ok) @@ -9143,6 +9221,7 @@ static void t8142_wake(struct apple_sleep *s) s->aon_wdt_marked = false; } apple_sleep_set_wd2(s, WDT_SUSPEND_SECS); + t8142_cpu_burst(s); if (s->pcie_ans_stopped) s->pcie_ans_on_ret = apple_sleep_raise_apcie(s);