Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions arch/arm64/include/asm/kexec.h
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,12 @@ int machine_kexec_post_load(struct kimage *image);
#define machine_kexec_post_load machine_kexec_post_load
#endif

#if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_ARM64_CTXR_EXECMEM)
void arm64_crash_handoff_wait(void);
#else
static inline void arm64_crash_handoff_wait(void) {}
#endif

#define ARCH_HAS_KIMAGE_ARCH

struct kimage_arch {
Expand Down
109 changes: 106 additions & 3 deletions arch/arm64/kernel/machine_kexec.c
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
* Copyright (C) Huawei Futurewei Technologies.
*/

#include <linux/delay.h>
#include <linux/execmem.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/kernel.h>
#include <linux/kexec.h>
#include <linux/kprobes.h>
#include <linux/page-flags.h>
#include <linux/reboot.h>
#include <linux/set_memory.h>
Expand Down Expand Up @@ -56,10 +58,12 @@ void machine_kexec_cleanup(struct kimage *kimage)
int machine_kexec_prepare(struct kimage *kimage)
{
/*
* The relocation code runs from an ordinary page, outside the window,
* and a crash dump can start on a core enforcing it.
* A new kernel runs from ordinary memory, outside the window, so the
* confined secondaries could never run it. Only a crash kernel can
* start: it runs on the boot CPU alone, which the window doesn't
* confine (see crash_handoff()).
*/
if (execmem_confine_active()) {
if (execmem_confine_active() && kimage->type != KEXEC_TYPE_CRASH) {
pr_err("Can't kexec: the kernel's code is confined to a CTXR window.\n");
return -EOPNOTSUPP;
}
Expand Down Expand Up @@ -165,6 +169,100 @@ int machine_kexec_post_load(struct kimage *kimage)
return 0;
}

#if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_ARM64_CTXR_EXECMEM)
/*
* Under a CTXR window every core but the boot CPU fetches code only from the
* window, and the crash kernel sits in ordinary memory. A crash handled on a
* confined core therefore offers the jump to CPU 0, which waits for it in its
* stop handler. The state makes taking the offer and withdrawing it after a
* timeout exclusive, so exactly one of the two CPUs goes on.
*/
enum {
CRASH_HANDOFF_IDLE,
CRASH_HANDOFF_OFFERED,
CRASH_HANDOFF_TAKEN,
CRASH_HANDOFF_WITHDRAWN,
};

static int crash_handoff_state;
static struct kimage *crash_handoff_image;

static bool crash_needs_handoff(void)
{
return execmem_confine_active() && smp_processor_id() != 0;
}

/*
* Called by every CPU told to stop, with interrupts masked. On CPU 0, with a
* window and a crash kernel loaded, wait for a confined core to hand the crash
* kernel over instead of parking. Waiting is parking until that happens.
*/
void arm64_crash_handoff_wait(void)
{
if (!execmem_confine_active() || smp_processor_id() != 0 ||
!kexec_crash_loaded())
return;

for (;;) {
if (cmpxchg(&crash_handoff_state, CRASH_HANDOFF_OFFERED,
CRASH_HANDOFF_TAKEN) == CRASH_HANDOFF_OFFERED)
machine_kexec(READ_ONCE(crash_handoff_image));
wfe();
}
}
NOKPROBE_SYMBOL(arm64_crash_handoff_wait);

/*
* Offer the crash kernel to CPU 0 and park. Returns only when CPU 0 can't take
* it: it never acknowledged the stop, or didn't answer within a second. The
* panic then goes on to its normal restart.
*/
static void crash_handoff(struct kimage *kimage)
{
unsigned int cpu = smp_processor_id();
unsigned long timeout = USEC_PER_SEC;

if (cpu_online(0)) {
pr_err("CPU 0 did not stop, so the crash kernel can't start from confined CPU %u\n",
cpu);
return;
}

pr_info("CPU %u is confined to the CTXR window, CPU 0 starts the crash kernel\n",
cpu);
WRITE_ONCE(crash_handoff_image, kimage);
/* CPU 0 then sees no other CPU online, as after any crash stop. */
set_cpu_online(cpu, false);
if (cmpxchg(&crash_handoff_state, CRASH_HANDOFF_IDLE,
CRASH_HANDOFF_OFFERED) != CRASH_HANDOFF_IDLE)
cpu_park_loop();
dsb(ish);
sev();

while (timeout--) {
if (READ_ONCE(crash_handoff_state) == CRASH_HANDOFF_TAKEN)
cpu_park_loop();
udelay(1);
}

if (cmpxchg(&crash_handoff_state, CRASH_HANDOFF_OFFERED,
CRASH_HANDOFF_WITHDRAWN) == CRASH_HANDOFF_TAKEN)
cpu_park_loop();

set_cpu_online(cpu, true);
pr_err("CPU 0 did not take the crash kernel\n");
}
#else
static bool crash_needs_handoff(void)
{
return false;
}

static void crash_handoff(struct kimage *kimage)
{
}
#endif

/**
* machine_kexec - Do the kexec reboot.
*
Expand All @@ -175,6 +273,11 @@ void machine_kexec(struct kimage *kimage)
bool in_kexec_crash = (kimage == kexec_crash_image);
bool stuck_cpus = cpus_are_stuck_in_kernel();

if (in_kexec_crash && crash_needs_handoff()) {
crash_handoff(kimage);
return;
}

/*
* New cpus may have become stuck_in_kernel after we loaded the image.
*/
Expand Down
3 changes: 3 additions & 0 deletions arch/arm64/kernel/smp.c
Original file line number Diff line number Diff line change
Expand Up @@ -924,6 +924,9 @@ void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash)

sdei_mask_local_cpu();

/* Under a CTXR window, CPU 0 may have to start the crash kernel. */
arm64_crash_handoff_wait();

if (crash && die_on_crash)
__cpu_try_die(cpu);

Expand Down
11 changes: 10 additions & 1 deletion arch/arm64/mm/ctxr_execmem.c
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

#define pr_fmt(fmt) "ctxr-execmem: " fmt

#include <linux/crash_dump.h>
#include <linux/efi.h>
#include <linux/execmem.h>
#include <linux/genalloc.h>
Expand Down Expand Up @@ -255,7 +256,15 @@ void __init ctxr_exec_reserve(void)
* core. So a command-line window can only replace the device-tree one
* when it is identical; a different one (a sub-window, say) is ignored.
*/
if (dt < 0) {
if (is_kdump_kernel() && (dt || cmd)) {
/*
* A crash kernel runs on the boot CPU only, from ordinary memory.
* The confined secondaries stay parked in the crashed kernel,
* whose window holds that kernel's code, not this one's.
*/
ctxr_hold_cpus = true;
pr_info("crash kernel: running on the boot CPU, outside the window\n");
} else if (dt < 0) {
ctxr_hold_secondaries("malformed aurora,ctxr-exec-window");
} else if (dt || cmd) {
if (!is_kernel_in_hyp_mode()) {
Expand Down
13 changes: 3 additions & 10 deletions arch/arm64/mm/init.c
Original file line number Diff line number Diff line change
Expand Up @@ -109,17 +109,10 @@ static void __init arch_reserve_crashkernel(void)
return;

/*
* A CTXR window refuses kexec (the relocation code runs outside it), so
* a crash kernel can never run. Don't reserve and waste its memory.
* Under a CTXR window the crash kernel runs on the boot CPU, which the
* window doesn't confine. The window is already reserved, so the crash
* kernel lands outside it.
*/
if (ctxr_exec_window_reserved()) {
if (parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
&crash_size, &crash_base, &low_size,
&cma_size, &high) == 0 && crash_size)
pr_warn("ctxr-execmem: ignoring crashkernel=, kexec is unavailable under a window\n");
return;
}

ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
&crash_size, &crash_base,
&low_size, &cma_size, &high);
Expand Down
63 changes: 63 additions & 0 deletions drivers/nvme/host/apple.c
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include <linux/blk-crypto-profile.h>
#include <linux/blkdev.h>
#include <linux/blk-mq.h>
#include <linux/crash_dump.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
Expand Down Expand Up @@ -1612,6 +1613,38 @@ static void apple_nvme_wait_freeze(struct apple_nvme *anv)
}
}

/*
* Stop the pre-M4 ANS session a crashed kernel left running, so that the
* reset path restarts the coprocessor from scratch. Each step is best effort:
* if the old session does not answer, the PMGR reset still stops it.
*/
static void apple_nvme_stop_crashed_session(struct apple_nvme *anv)
{
int ret;

anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP);
anv->ctrl.ctrl_config = readl(anv->mmio_nvme + NVME_REG_CC);
dev_info(anv->dev,
"crash kernel: ANS still running (CC=%#x CSTS=%#x), restarting it\n",
anv->ctrl.ctrl_config, readl(anv->mmio_nvme + NVME_REG_CSTS));

ret = apple_rtkit_adopt_running(anv->rtk);
if (!ret && (anv->ctrl.ctrl_config & NVME_CC_ENABLE))
ret = nvme_disable_ctrl(&anv->ctrl, false);
if (!ret)
ret = apple_rtkit_shutdown(anv->rtk);
if (ret) {
dev_warn(anv->dev,
"crash kernel: old ANS session did not stop (%d), resetting it\n",
ret);
/* Forget the old session; the reset below discards it. */
apple_rtkit_reinit(anv->rtk);
}

writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL);
anv->ctrl.ctrl_config = 0;
}

static void apple_nvme_reset_work(struct work_struct *work)
{
unsigned int nr_io_queues = 1;
Expand All @@ -1621,6 +1654,7 @@ static void apple_nvme_reset_work(struct work_struct *work)
struct apple_nvme *anv =
container_of(work, struct apple_nvme, ctrl.reset_work);
enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl);
bool inherited = !anv->owns_rtkit;

if (anv->hw->needs_ioq_registers) {
mutex_lock(&anv->disable_lock);
Expand Down Expand Up @@ -1677,6 +1711,17 @@ static void apple_nvme_reset_work(struct work_struct *work)
goto rtkit_ready;
}

/*
* A crashed kernel never stops ANS, so on pre-M4 the crash kernel finds
* the coprocessor still running the old kernel's RTKit session. That
* session does not repeat HELLO and EPMAP, so a wake cannot work. Take
* the session over, stop it and restart ANS through the cold path below.
*/
if (inherited && !anv->hw->needs_ioq_registers && is_kdump_kernel() &&
(readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) &
APPLE_ANS_COPROC_CPU_CONTROL_RUN))
apple_nvme_stop_crashed_session(anv);

/*
* RTKit must be shut down cleanly for the (soft)-reset to work, except
* on post-M4: that firmware stays live and only its queue DMA is
Expand Down Expand Up @@ -1772,6 +1817,24 @@ static void apple_nvme_reset_work(struct work_struct *work)

dev_dbg(anv->dev, "ANS booted successfully.");

/*
* Pre-M4 ANS keeps running across a kexec, and a crashed kernel hands
* it to the crash kernel with the controller still enabled on its own
* queues. Woken this way, it would ignore the queue registers below,
* which may only change while CC.EN is clear. Disable it first.
*/
if (inherited && !anv->hw->needs_ioq_registers) {
anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP);
anv->ctrl.ctrl_config = readl(anv->mmio_nvme + NVME_REG_CC);
if (anv->ctrl.ctrl_config & NVME_CC_ENABLE) {
dev_info(anv->dev,
"controller handed over enabled, disabling it\n");
ret = nvme_disable_ctrl(&anv->ctrl, false);
if (ret)
goto out;
}
}

/*
* Limit the max command size to prevent iod->sg allocations going
* over a single page.
Expand Down
16 changes: 15 additions & 1 deletion drivers/watchdog/apple_wdt.c
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

#include <linux/bits.h>
#include <linux/clk.h>
#include <linux/crash_dump.h>
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/kernel.h>
Expand Down Expand Up @@ -185,12 +186,25 @@ static int apple_wdt_probe(struct platform_device *pdev)
wdt->wdd.timeout = APPLE_WDT_TIMEOUT_DEFAULT;

wdt_ctrl = readl_relaxed(wdt->regs + APPLE_WDT_WD1_CTRL);
if (wdt_ctrl & APPLE_WDT_CTRL_RESET_EN)
if ((wdt_ctrl & APPLE_WDT_CTRL_RESET_EN) && !is_kdump_kernel())
set_bit(WDOG_HW_RUNNING, &wdt->wdd.status);

watchdog_set_nowayout(&wdt->wdd, WATCHDOG_NOWAYOUT);
watchdog_init_timeout(&wdt->wdd, 0, dev);
apple_wdt_set_timeout(&wdt->wdd, wdt->wdd.timeout);

/*
* A crash kernel only saves the dump and restarts. If it stalls, the
* machine must still restart rather than hang: arm the watchdog at its
* longest timeout and leave it unpinged, unless userspace opens it.
*/
if (is_kdump_kernel()) {
apple_wdt_set_timeout(&wdt->wdd,
wdt->wdd.max_hw_heartbeat_ms / 1000);
apple_wdt_start(&wdt->wdd);
dev_info(dev, "crash kernel: restarting in %u s unless done\n",
wdt->wdd.timeout);
}
watchdog_stop_on_unregister(&wdt->wdd);
watchdog_set_restart_priority(&wdt->wdd, 128);

Expand Down
3 changes: 3 additions & 0 deletions kernel/crash_core.c
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,9 @@ void __noclone __crash_kexec(struct pt_regs *regs)
} else {
struct pt_regs fixed_regs;

if (kexec_blocked())
pr_warn("crash kernel starts while devices keep DMA to retained memory; first reason: %s\n",
kexec_block_reason());
crash_setup_regs(&fixed_regs, regs);
crash_save_vmcoreinfo();
machine_crash_shutdown(&fixed_regs);
Expand Down
3 changes: 2 additions & 1 deletion kernel/kexec.c
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,8 @@ int do_kexec_load(unsigned long entry, unsigned long nr_segments,
*/
if (!kexec_trylock())
return -EBUSY;
if (nr_segments && kexec_blocked()) {
if (nr_segments && ((flags & KEXEC_ON_CRASH) ? kexec_crash_blocked() :
kexec_blocked())) {
pr_warn_ratelimited("kexec load refused by active interlock; first reason: %s\n",
kexec_block_reason());
ret = -EOPNOTSUPP;
Expand Down
Loading
Loading