Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion arch/arm64/kernel/smp.c
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,17 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
*/
wait_for_completion_timeout(&cpu_running,
msecs_to_jiffies(5000));
if (cpu_online(cpu))
if (cpu_online(cpu)) {
/*
* The CPU has taken its task. Cleared, as kernels before 5.x did, so a core that
* reaches __secondary_switched without a bring-up of its own waits in
* __secondary_too_slow instead of running as this CPU a second time.
*/
WRITE_ONCE(secondary_data.task, NULL);
dcache_clean_inval_poc((unsigned long)&secondary_data,
(unsigned long)&secondary_data + sizeof(secondary_data));
return 0;
}

pr_crit("CPU%u: failed to come online\n", cpu);
secondary_data.task = NULL;
Expand Down
13 changes: 13 additions & 0 deletions arch/arm64/kernel/smp_spin_table.c
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,9 @@ int spin_table_hold_parked(void)
int us, ret;
unsigned int cpu;

/* No CPU restarted from here to the wake may pass the pen without its own bring-up. */
write_pen_release(INVALID_HWID);

for_each_present_cpu(cpu)
WRITE_ONCE(spin_table_hold_error[cpu], 0);
WRITE_ONCE(spin_table_hold_generation, READ_ONCE(spin_table_hold_generation) + 1);
Expand Down Expand Up @@ -331,6 +334,8 @@ void spin_table_stop_finished(void)
/* Before any CPU is started again: the pen must let them boot. */
if (secondary_pen_deep)
spin_table_pen_deep_set(0);
/* ...but only the one its bring-up releases (cores started early wait in the pen). */
write_pen_release(INVALID_HWID);
WRITE_ONCE(spin_table_hold_requested, false);
WRITE_ONCE(spin_table_stop_requested, false);
WRITE_ONCE(spin_table_stop_released, false);
Expand Down Expand Up @@ -473,6 +478,14 @@ static int smp_rvbar_spin_table_cpu_kill(unsigned int cpu)

static void smp_rvbar_spin_table_postboot(void)
{
/*
* Close the pen behind this CPU. Left at its MPIDR, the next restart at the reset vector
* (after a sleep's commit, or a platform that starts every core at once at the wake)
* lets it straight through the pen into secondary_startup ahead of its own bring-up,
* with the secondary_data of its last boot: it then runs as that CPU with no STARTING
* callbacks run, e.g. without a tick device.
*/
write_pen_release(INVALID_HWID);
cpumask_clear_cpu(smp_processor_id(), &spin_table_cpus_off);
if (READ_ONCE(spin_table_power) && spin_table_power->cpu_restore)
spin_table_power->cpu_restore(read_cpuid_mpidr() & MPIDR_HWID_BITMASK);
Expand Down
87 changes: 83 additions & 4 deletions drivers/soc/apple/sleep.c
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,7 @@ struct apple_sleep {
void __iomem *ps;
void __iomem *pwrgate;
void __iomem *cpu_start;
unsigned int cpu_burst_n; /* cores the last wake's burst started (cpu_burst) */
void __iomem *cllt;
void __iomem *cluster;
void __iomem *commit;
Expand Down Expand Up @@ -2234,9 +2235,43 @@ MODULE_PARM_DESC(mem_restore, "T8142: put the memory side (AMCC, SoC NI, DCS) ba
static bool sleep_clk_restore = true;
module_param_named(clk_restore, sleep_clk_restore, bool, 0644);
MODULE_PARM_DESC(clk_restore, "T8142: put the 126 PMGR clkgen slots and the memory clock word back at the wake as XNU does");
static bool sleep_perf_down = true;
/*
* Default N: a traced macOS 27.2 run on J813 (boot, a 40 s sleep, the wake and three more entries)
* never touches the perf-state page 0x380024000, though it was hooked. Y keeps the earlier
* behaviour.
*/
static bool sleep_perf_down;
module_param_named(perf_down, sleep_perf_down, bool, 0644);
MODULE_PARM_DESC(perf_down, "T8142: SoC/DCS perf states down before the memory stop (macOS on J813 never writes them; A/B)");
MODULE_PARM_DESC(perf_down,
"T8142: SoC/DCS perf states down before the memory stop (macOS never writes them; default N)");

/*
* The clusters' words at the commit as XNU writes them. In the same trace the first sleep after
* boot reads 0xf and writes 0; every commit after a wake (three of three) reads 0x2f and writes
* 0x20: only the low nibble cleared, bit 5 kept. Y (default): that read-modify-write; N: write 0,
* as before.
*/
static bool sleep_cluster_rmw = true;
module_param_named(cluster_rmw, sleep_cluster_rmw, bool, 0644);
MODULE_PARM_DESC(cluster_rmw,
"T8142: clear only the clusters' on bits at the commit, keep the rest, as XNU (default Y; N = write 0)");

/*
* The cores' start after the wake as XNU does it. In the same trace, 16.5 ms after the wake, right
* after the memory side and the WD2 stop, the boot core starts all nine other cores in one 0.23 ms
* burst, three words each: +4 the core's bit, +8 the E core's bit or 0, +0xc the P core's bit or
* 0. Linux started each core from its own bring-up, later and one by one, with two words. Y
* (default): the burst at the end of the wake, for the cores that were online when the sleep
* began; they come up through the boot object into the kernel's holding pen and wait there, and
* each bring-up only releases its core. N: the per-core start.
*/
static bool sleep_cpu_burst = true;
module_param_named(cpu_burst, sleep_cpu_burst, bool, 0644);
MODULE_PARM_DESC(cpu_burst,
"T8142: start the other cores in one burst at the end of the wake, as XNU (default Y; N = per core at bring-up)");
static struct cpumask sleep_cpus_burst;
/* The CPUs online at .begin: the ones the PM core brings back after the wake. */
static struct cpumask sleep_cpus_awake;
static u64 t8142_boot_mpidr = ~0ULL;

extern char t8142_el1_wfi_stub[], t8142_el1_wfi_stub_end[];
Expand Down Expand Up @@ -2497,6 +2532,15 @@ static int sleep_cpu_on(u64 mpidr)
u64 flag0 = 0, flag1 = 0;
unsigned int us = 0;

/* Started by the wake's burst (apple_sleep.cpu_burst): it waits in the pen; release it. */
if (cpu >= 0 && cpumask_test_and_clear_cpu(cpu, &sleep_cpus_burst)) {
sleep_cpu_start_words(s, after);
pr_info("apple-sleep: cpu%d start %#llx: started in the wake burst (%u cores), released from the pen; stop/sys/e/p now %x/%x/%x/%x; aura slot flag %llu\n",
cpu, mpidr, s->cpu_burst_n, after[0], after[1], after[2], after[3],
slot ? sleep_cpu_slot_read(slot, 8) : 0ULL);
return 0;
}

if (trace) {
sleep_cpu_start_words(s, before);
if (slot)
Expand Down Expand Up @@ -6338,6 +6382,8 @@ static int apple_sleep_begin(suspend_state_t state)
/* The PM core also calls .begin for s2idle when no s2idle ops are installed. */
if (state != PM_SUSPEND_MEM)
return 0;
/* Before the PM core takes the other CPUs down: the wake's burst starts only these. */
cpumask_copy(&sleep_cpus_awake, cpu_online_mask);

ret = apple_sleep_profile_check(apple_sleep);
if (ret)
Expand Down Expand Up @@ -8960,8 +9006,11 @@ static void t8142_last_mile(struct apple_sleep *s)

t8142_commit_note(s);

for (i = 0; i < soc->ncluster_cpu; i++)
writel(0, s->cluster + soc->cluster_cpu[i]);
for (i = 0; i < soc->ncluster_cpu; i++) {
void __iomem *reg = s->cluster + soc->cluster_cpu[i];

writel(READ_ONCE(sleep_cluster_rmw) ? readl(reg) & ~CLUSTER_ON : 0, reg);
}

s->quiesce_still_on = 0;
for (i = 0; i < soc->quiesce.count; i++)
Expand Down Expand Up @@ -9089,6 +9138,35 @@ static int t8142_cluster_on(struct apple_sleep *s, u32 word, void __iomem *acc)
* word, map 0x4e's ten words) are gated by flags its constructor clears and nothing sets.
* Then T8132's: ATC0's words, SEP, WD2, ANS's PCIe block and the radio's link.
*/
/* apple_sleep.cpu_burst: the other cores started at once, three words each, as XNU. */
static void t8142_cpu_burst(struct apple_sleep *s)
{
unsigned int cpu, n = 0, self = smp_processor_id();

cpumask_clear(&sleep_cpus_burst);
s->cpu_burst_n = 0;
/* T8142's data states no cpu_clusters (0): E = MPIDR cluster 0, P = 1, from the MPIDR. */
if (!READ_ONCE(sleep_cpu_burst) || !s->cpu_start)
return;
for_each_cpu(cpu, &sleep_cpus_awake) {
u64 mpidr = cpu_logical_map(cpu);
unsigned int cluster = sleep_cpu_cluster(s, mpidr), core = sleep_cpu_core(mpidr);

if (cpu == self || cluster > 1)
continue;
writel(sleep_cpu_start_bit(s, mpidr), s->cpu_start + CPU_START_SYS);
writel(cluster == 0 ? BIT(core) : 0, s->cpu_start + CPU_START_CLUSTER(0));
writel(cluster == 1 ? BIT(core) : 0, s->cpu_start + CPU_START_CLUSTER(1));
cpumask_set_cpu(cpu, &sleep_cpus_burst);
n++;
}
s->cpu_burst_n = n;
pr_info("apple-sleep: wake: %u cores started in one burst (cpu_burst=Y, XNU's +4/+8/+0xc); stop/sys/e/p now %x/%x/%x/%x\n",
n, readl(s->cpu_start + CPU_START_STOP), readl(s->cpu_start + CPU_START_SYS),
readl(s->cpu_start + CPU_START_CLUSTER(0)),
readl(s->cpu_start + CPU_START_CLUSTER(1)));
}

static void t8142_wake(struct apple_sleep *s)
{
if (t8142_alarm_ok)
Expand Down Expand Up @@ -9143,6 +9221,7 @@ static void t8142_wake(struct apple_sleep *s)
s->aon_wdt_marked = false;
}
apple_sleep_set_wd2(s, WDT_SUSPEND_SECS);
t8142_cpu_burst(s);

if (s->pcie_ans_stopped)
s->pcie_ans_on_ret = apple_sleep_raise_apcie(s);
Expand Down
Loading