diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h index 892e5bebda957..3f4bac22eb7d7 100644 --- a/arch/arm64/include/asm/kexec.h +++ b/arch/arm64/include/asm/kexec.h @@ -104,6 +104,12 @@ int machine_kexec_post_load(struct kimage *image); #define machine_kexec_post_load machine_kexec_post_load #endif +#if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_ARM64_CTXR_EXECMEM) +void arm64_crash_handoff_wait(void); +#else +static inline void arm64_crash_handoff_wait(void) {} +#endif + #define ARCH_HAS_KIMAGE_ARCH struct kimage_arch { diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 9c4a407222ea1..b8e6b6c8aa653 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -6,11 +6,13 @@ * Copyright (C) Huawei Futurewei Technologies. */ +#include #include #include #include #include #include +#include #include #include #include @@ -56,10 +58,12 @@ void machine_kexec_cleanup(struct kimage *kimage) int machine_kexec_prepare(struct kimage *kimage) { /* - * The relocation code runs from an ordinary page, outside the window, - * and a crash dump can start on a core enforcing it. + * A new kernel runs from ordinary memory, outside the window, so the + * confined secondaries could never run it. Only a crash kernel can + * start: it runs on the boot CPU alone, which the window doesn't + * confine (see crash_handoff()). */ - if (execmem_confine_active()) { + if (execmem_confine_active() && kimage->type != KEXEC_TYPE_CRASH) { pr_err("Can't kexec: the kernel's code is confined to a CTXR window.\n"); return -EOPNOTSUPP; } @@ -165,6 +169,100 @@ int machine_kexec_post_load(struct kimage *kimage) return 0; } +#if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_ARM64_CTXR_EXECMEM) +/* + * Under a CTXR window every core but the boot CPU fetches code only from the + * window, and the crash kernel sits in ordinary memory. A crash handled on a + * confined core therefore offers the jump to CPU 0, which waits for it in its + * stop handler. The state makes taking the offer and withdrawing it after a + * timeout exclusive, so exactly one of the two CPUs goes on. + */ +enum { + CRASH_HANDOFF_IDLE, + CRASH_HANDOFF_OFFERED, + CRASH_HANDOFF_TAKEN, + CRASH_HANDOFF_WITHDRAWN, +}; + +static int crash_handoff_state; +static struct kimage *crash_handoff_image; + +static bool crash_needs_handoff(void) +{ + return execmem_confine_active() && smp_processor_id() != 0; +} + +/* + * Called by every CPU told to stop, with interrupts masked. On CPU 0, with a + * window and a crash kernel loaded, wait for a confined core to hand the crash + * kernel over instead of parking. Waiting is parking until that happens. + */ +void arm64_crash_handoff_wait(void) +{ + if (!execmem_confine_active() || smp_processor_id() != 0 || + !kexec_crash_loaded()) + return; + + for (;;) { + if (cmpxchg(&crash_handoff_state, CRASH_HANDOFF_OFFERED, + CRASH_HANDOFF_TAKEN) == CRASH_HANDOFF_OFFERED) + machine_kexec(READ_ONCE(crash_handoff_image)); + wfe(); + } +} +NOKPROBE_SYMBOL(arm64_crash_handoff_wait); + +/* + * Offer the crash kernel to CPU 0 and park. Returns only when CPU 0 can't take + * it: it never acknowledged the stop, or didn't answer within a second. The + * panic then goes on to its normal restart. + */ +static void crash_handoff(struct kimage *kimage) +{ + unsigned int cpu = smp_processor_id(); + unsigned long timeout = USEC_PER_SEC; + + if (cpu_online(0)) { + pr_err("CPU 0 did not stop, so the crash kernel can't start from confined CPU %u\n", + cpu); + return; + } + + pr_info("CPU %u is confined to the CTXR window, CPU 0 starts the crash kernel\n", + cpu); + WRITE_ONCE(crash_handoff_image, kimage); + /* CPU 0 then sees no other CPU online, as after any crash stop. */ + set_cpu_online(cpu, false); + if (cmpxchg(&crash_handoff_state, CRASH_HANDOFF_IDLE, + CRASH_HANDOFF_OFFERED) != CRASH_HANDOFF_IDLE) + cpu_park_loop(); + dsb(ish); + sev(); + + while (timeout--) { + if (READ_ONCE(crash_handoff_state) == CRASH_HANDOFF_TAKEN) + cpu_park_loop(); + udelay(1); + } + + if (cmpxchg(&crash_handoff_state, CRASH_HANDOFF_OFFERED, + CRASH_HANDOFF_WITHDRAWN) == CRASH_HANDOFF_TAKEN) + cpu_park_loop(); + + set_cpu_online(cpu, true); + pr_err("CPU 0 did not take the crash kernel\n"); +} +#else +static bool crash_needs_handoff(void) +{ + return false; +} + +static void crash_handoff(struct kimage *kimage) +{ +} +#endif + /** * machine_kexec - Do the kexec reboot. * @@ -175,6 +273,11 @@ void machine_kexec(struct kimage *kimage) bool in_kexec_crash = (kimage == kexec_crash_image); bool stuck_cpus = cpus_are_stuck_in_kernel(); + if (in_kexec_crash && crash_needs_handoff()) { + crash_handoff(kimage); + return; + } + /* * New cpus may have become stuck_in_kernel after we loaded the image. */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 9cd9bc7e40812..78d9b1857e147 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -924,6 +924,9 @@ void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash) sdei_mask_local_cpu(); + /* Under a CTXR window, CPU 0 may have to start the crash kernel. */ + arm64_crash_handoff_wait(); + if (crash && die_on_crash) __cpu_try_die(cpu); diff --git a/arch/arm64/mm/ctxr_execmem.c b/arch/arm64/mm/ctxr_execmem.c index 58f95322bd5d8..7851030724dca 100644 --- a/arch/arm64/mm/ctxr_execmem.c +++ b/arch/arm64/mm/ctxr_execmem.c @@ -22,6 +22,7 @@ #define pr_fmt(fmt) "ctxr-execmem: " fmt +#include #include #include #include @@ -255,7 +256,15 @@ void __init ctxr_exec_reserve(void) * core. So a command-line window can only replace the device-tree one * when it is identical; a different one (a sub-window, say) is ignored. */ - if (dt < 0) { + if (is_kdump_kernel() && (dt || cmd)) { + /* + * A crash kernel runs on the boot CPU only, from ordinary memory. + * The confined secondaries stay parked in the crashed kernel, + * whose window holds that kernel's code, not this one's. + */ + ctxr_hold_cpus = true; + pr_info("crash kernel: running on the boot CPU, outside the window\n"); + } else if (dt < 0) { ctxr_hold_secondaries("malformed aurora,ctxr-exec-window"); } else if (dt || cmd) { if (!is_kernel_in_hyp_mode()) { diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index e7f6751b718e3..bfb02a5524780 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -109,17 +109,10 @@ static void __init arch_reserve_crashkernel(void) return; /* - * A CTXR window refuses kexec (the relocation code runs outside it), so - * a crash kernel can never run. Don't reserve and waste its memory. + * Under a CTXR window the crash kernel runs on the boot CPU, which the + * window doesn't confine. The window is already reserved, so the crash + * kernel lands outside it. */ - if (ctxr_exec_window_reserved()) { - if (parse_crashkernel(boot_command_line, memblock_phys_mem_size(), - &crash_size, &crash_base, &low_size, - &cma_size, &high) == 0 && crash_size) - pr_warn("ctxr-execmem: ignoring crashkernel=, kexec is unavailable under a window\n"); - return; - } - ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), &crash_size, &crash_base, &low_size, &cma_size, &high); diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 732d7b63be3d4..f843f6d4a7939 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -1612,6 +1613,38 @@ static void apple_nvme_wait_freeze(struct apple_nvme *anv) } } +/* + * Stop the pre-M4 ANS session a crashed kernel left running, so that the + * reset path restarts the coprocessor from scratch. Each step is best effort: + * if the old session does not answer, the PMGR reset still stops it. + */ +static void apple_nvme_stop_crashed_session(struct apple_nvme *anv) +{ + int ret; + + anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); + anv->ctrl.ctrl_config = readl(anv->mmio_nvme + NVME_REG_CC); + dev_info(anv->dev, + "crash kernel: ANS still running (CC=%#x CSTS=%#x), restarting it\n", + anv->ctrl.ctrl_config, readl(anv->mmio_nvme + NVME_REG_CSTS)); + + ret = apple_rtkit_adopt_running(anv->rtk); + if (!ret && (anv->ctrl.ctrl_config & NVME_CC_ENABLE)) + ret = nvme_disable_ctrl(&anv->ctrl, false); + if (!ret) + ret = apple_rtkit_shutdown(anv->rtk); + if (ret) { + dev_warn(anv->dev, + "crash kernel: old ANS session did not stop (%d), resetting it\n", + ret); + /* Forget the old session; the reset below discards it. */ + apple_rtkit_reinit(anv->rtk); + } + + writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); + anv->ctrl.ctrl_config = 0; +} + static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; @@ -1621,6 +1654,7 @@ static void apple_nvme_reset_work(struct work_struct *work) struct apple_nvme *anv = container_of(work, struct apple_nvme, ctrl.reset_work); enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl); + bool inherited = !anv->owns_rtkit; if (anv->hw->needs_ioq_registers) { mutex_lock(&anv->disable_lock); @@ -1677,6 +1711,17 @@ static void apple_nvme_reset_work(struct work_struct *work) goto rtkit_ready; } + /* + * A crashed kernel never stops ANS, so on pre-M4 the crash kernel finds + * the coprocessor still running the old kernel's RTKit session. That + * session does not repeat HELLO and EPMAP, so a wake cannot work. Take + * the session over, stop it and restart ANS through the cold path below. + */ + if (inherited && !anv->hw->needs_ioq_registers && is_kdump_kernel() && + (readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & + APPLE_ANS_COPROC_CPU_CONTROL_RUN)) + apple_nvme_stop_crashed_session(anv); + /* * RTKit must be shut down cleanly for the (soft)-reset to work, except * on post-M4: that firmware stays live and only its queue DMA is @@ -1772,6 +1817,24 @@ static void apple_nvme_reset_work(struct work_struct *work) dev_dbg(anv->dev, "ANS booted successfully."); + /* + * Pre-M4 ANS keeps running across a kexec, and a crashed kernel hands + * it to the crash kernel with the controller still enabled on its own + * queues. Woken this way, it would ignore the queue registers below, + * which may only change while CC.EN is clear. Disable it first. + */ + if (inherited && !anv->hw->needs_ioq_registers) { + anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); + anv->ctrl.ctrl_config = readl(anv->mmio_nvme + NVME_REG_CC); + if (anv->ctrl.ctrl_config & NVME_CC_ENABLE) { + dev_info(anv->dev, + "controller handed over enabled, disabling it\n"); + ret = nvme_disable_ctrl(&anv->ctrl, false); + if (ret) + goto out; + } + } + /* * Limit the max command size to prevent iod->sg allocations going * over a single page. diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c index 46a00a7993a6e..3c867a3ce636e 100644 --- a/drivers/watchdog/apple_wdt.c +++ b/drivers/watchdog/apple_wdt.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -185,12 +186,25 @@ static int apple_wdt_probe(struct platform_device *pdev) wdt->wdd.timeout = APPLE_WDT_TIMEOUT_DEFAULT; wdt_ctrl = readl_relaxed(wdt->regs + APPLE_WDT_WD1_CTRL); - if (wdt_ctrl & APPLE_WDT_CTRL_RESET_EN) + if ((wdt_ctrl & APPLE_WDT_CTRL_RESET_EN) && !is_kdump_kernel()) set_bit(WDOG_HW_RUNNING, &wdt->wdd.status); watchdog_set_nowayout(&wdt->wdd, WATCHDOG_NOWAYOUT); watchdog_init_timeout(&wdt->wdd, 0, dev); apple_wdt_set_timeout(&wdt->wdd, wdt->wdd.timeout); + + /* + * A crash kernel only saves the dump and restarts. If it stalls, the + * machine must still restart rather than hang: arm the watchdog at its + * longest timeout and leave it unpinged, unless userspace opens it. + */ + if (is_kdump_kernel()) { + apple_wdt_set_timeout(&wdt->wdd, + wdt->wdd.max_hw_heartbeat_ms / 1000); + apple_wdt_start(&wdt->wdd); + dev_info(dev, "crash kernel: restarting in %u s unless done\n", + wdt->wdd.timeout); + } watchdog_stop_on_unregister(&wdt->wdd); watchdog_set_restart_priority(&wdt->wdd, 128); diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 3a9892c93eb58..e36e12bb18c43 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -142,6 +142,9 @@ void __noclone __crash_kexec(struct pt_regs *regs) } else { struct pt_regs fixed_regs; + if (kexec_blocked()) + pr_warn("crash kernel starts while devices keep DMA to retained memory; first reason: %s\n", + kexec_block_reason()); crash_setup_regs(&fixed_regs, regs); crash_save_vmcoreinfo(); machine_crash_shutdown(&fixed_regs); diff --git a/kernel/kexec.c b/kernel/kexec.c index 34c9e8a79ba49..41d3145062194 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -102,7 +102,8 @@ int do_kexec_load(unsigned long entry, unsigned long nr_segments, */ if (!kexec_trylock()) return -EBUSY; - if (nr_segments && kexec_blocked()) { + if (nr_segments && ((flags & KEXEC_ON_CRASH) ? kexec_crash_blocked() : + kexec_blocked())) { pr_warn_ratelimited("kexec load refused by active interlock; first reason: %s\n", kexec_block_reason()); ret = -EOPNOTSUPP; diff --git a/kernel/kexec_block_test.c b/kernel/kexec_block_test.c index 495e37612fe0c..cc9146bd5ab8f 100644 --- a/kernel/kexec_block_test.c +++ b/kernel/kexec_block_test.c @@ -139,12 +139,17 @@ static void kexec_block_legacy_load_test(struct kunit *test) { /* Bad alignment also makes a missing guard fail safely in validation. */ struct kexec_segment segment = { .mem = 1, .memsz = PAGE_SIZE }; + int ret; kexec_test_acquire(test); kexec_test_require_empty_images(test); KUNIT_EXPECT_EQ(test, do_kexec_load(0, 1, &segment, 0), -EOPNOTSUPP); - KUNIT_EXPECT_EQ(test, do_kexec_load(0, 1, &segment, KEXEC_ON_CRASH), - -EOPNOTSUPP); + /* A crash kernel only passes the guard without CMA crash memory. */ + ret = do_kexec_load(0, 1, &segment, KEXEC_ON_CRASH); + if (crashk_cma_cnt) + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + else + KUNIT_EXPECT_EQ(test, ret, -EADDRNOTAVAIL); /* Unload an empty slot through the actual loader while still blocked. */ KUNIT_EXPECT_EQ(test, do_kexec_load(0, 0, NULL, 0), 0); KUNIT_EXPECT_EQ(test, do_kexec_load(0, 0, NULL, KEXEC_ON_CRASH), 0); @@ -166,9 +171,10 @@ static void kexec_block_file_load_test(struct kunit *test) if (ret == -EPERM) kunit_skip(test, "file loads disabled by permissions or attempt limit"); KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + /* Past the guard, the invalid fd stops a crash load before any image. */ KUNIT_EXPECT_EQ(test, do_kexec_file_load(-1, -1, 0, NULL, KEXEC_FILE_ON_CRASH), - -EOPNOTSUPP); + crashk_cma_cnt ? -EOPNOTSUPP : -EBADF); KUNIT_EXPECT_EQ(test, do_kexec_file_load(-1, -1, 0, NULL, KEXEC_FILE_UNLOAD), 0); KUNIT_EXPECT_EQ(test, do_kexec_file_load(-1, -1, 0, NULL, @@ -188,7 +194,8 @@ static void kexec_block_crash_policy_test(struct kunit *test) loaded_allowed = kexec_crash_image_allowed(true); kexec_unlock(); KUNIT_EXPECT_FALSE(test, empty_allowed); - KUNIT_EXPECT_FALSE(test, loaded_allowed); + /* Retained memory never overlaps a memblock crash reservation. */ + KUNIT_EXPECT_EQ(test, loaded_allowed, !crashk_cma_cnt); kexec_test_release(test); KUNIT_ASSERT_TRUE(test, kexec_trylock()); /* A real guard may have appeared since this case's initial skip check. */ @@ -197,7 +204,7 @@ static void kexec_block_crash_policy_test(struct kunit *test) loaded_allowed = kexec_crash_image_allowed(true); kexec_unlock(); KUNIT_EXPECT_FALSE(test, empty_allowed); - KUNIT_EXPECT_EQ(test, loaded_allowed, !blocked); + KUNIT_EXPECT_EQ(test, loaded_allowed, !(blocked && crashk_cma_cnt)); } static struct kunit_case kexec_block_test_cases[] = { diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 815588ee6d68b..c7c782fbbfb9a 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -382,7 +382,8 @@ int do_kexec_file_load(int kernel_fd, int initrd_fd, unsigned long cmdline_len, if (!kexec_trylock()) return -EBUSY; - if (!(flags & KEXEC_FILE_UNLOAD) && kexec_blocked()) { + if (!(flags & KEXEC_FILE_UNLOAD) && + ((flags & KEXEC_FILE_ON_CRASH) ? kexec_crash_blocked() : kexec_blocked())) { pr_warn_ratelimited("kexec file load refused by active interlock; first reason: %s\n", kexec_block_reason()); kexec_unlock(); diff --git a/kernel/kexec_internal.h b/kernel/kexec_internal.h index a42b95f65a43f..7f02aa1fd8302 100644 --- a/kernel/kexec_internal.h +++ b/kernel/kexec_internal.h @@ -26,10 +26,23 @@ const char *notrace kexec_block_reason(void); #ifdef CONFIG_KEXEC_BLOCK_KUNIT_TEST bool kexec_test_load_limits_unlimited(void); #endif +/* + * A blocker keeps a new kernel from reusing memory that a device still reaches + * by DMA. A crash kernel reuses none of it: it runs in the crash kernel + * reservation, which memblock set aside at boot, before any driver could give + * memory to a device. Only CMA-backed crash memory (crashkernel=,cma) is lent + * to the page allocator while the system runs, so blockers still stop a crash + * kernel that has some. + */ +static __always_inline bool notrace kexec_crash_blocked(void) +{ + return kexec_blocked() && crashk_cma_cnt; +} + /* Caller holds the existing NMI-safe kexec lock; no image is dereferenced. */ static __always_inline bool kexec_crash_image_allowed(bool image_loaded) { - return image_loaded && !kexec_blocked(); + return image_loaded && !kexec_crash_blocked(); } #ifdef CONFIG_KEXEC