diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 18f2e943bfb29..9526a1b399fcd 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1138,6 +1138,24 @@ Kernel parameters option depends on the CSD_LOCK_WAIT_DEBUG_DEFAULT Kconfig option. + ctxr_exec_enforce= + [ARM64,EARLY] Whether executable code that the boot-time + check finds outside the CTXR executable window is + fatal. Applies to a window from the device tree or + from ctxr_exec_window=. + Format: + Default: 1 + + ctxr_exec_window= + [ARM64,EARLY] Confine kernel executable code to a + physical window. This is for bring-up and VMs. A value + that doesn't parse is ignored with a warning. The + kernel must be at EL2, and when the device tree carries + a window the given one must lie inside it; otherwise the + secondary CPUs are kept offline. + See Documentation/arch/arm64/ctxr-execmem.rst. + Format: [KMG]@[KMG] + dasd= [HW,NET] See header of drivers/s390/block/dasd_devmap.c. diff --git a/Documentation/arch/arm64/ctxr-execmem.rst b/Documentation/arch/arm64/ctxr-execmem.rst new file mode 100644 index 0000000000000..2e131d2403839 --- /dev/null +++ b/Documentation/arch/arm64/ctxr-execmem.rst @@ -0,0 +1,155 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================================== +Confining kernel executable code (CTXR) +========================================== + +Some Apple-silicon machines (T8140/M4-class and later) have boot firmware that +arms a per-CPU *code text execute range* (CTXR) on the secondary cores: at the +privileged exception level, an instruction may only be fetched from inside a +fixed physical window. Outside it, the fetch takes an instruction abort. + +To run the kernel at EL2 behind such a window (instead of dropping it to EL1), +**every** executable physical page must live inside one window: the static image +(placed there by the bootloader) and every run-time executable allocation. + +``CONFIG_ARM64_CTXR_EXECMEM`` makes that possible. It is inert unless a window +is supplied at boot, so it is safe to leave enabled on machines that do not need +it. + +Supplying the window +==================== + +Device tree (production) + The bootloader emits, under ``/chosen``:: + + aurora,ctxr-exec-window = ; + + ``base`` and ``size`` are in bytes, page aligned and not 0. See + ``Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml``. + +Command line (bring-up / VM) + :: + + ctxr_exec_window=@ e.g. ctxr_exec_window=128M@0x40200000 + ctxr_exec_enforce= (default: enforcing) + + ``size`` and ``base`` accept the usual ``K``/``M``/``G`` suffixes, and both + are required. The last use of an option wins. The kernel must be at EL2, and + when the device tree also carries a window the command-line one must lie + inside it: the command line can only pick a window within what the firmware + armed, never a different region. A value that doesn't parse is ignored with + a warning. + +The secondary CPUs can only run code inside the window, and the boot CPU isn't +confined. So when a window is given but can't be used, the kernel logs why, +leaves the secondaries in the bootloader and boots on the boot CPU alone, as with +``nosmp``. That happens when the device-tree property is malformed (not four +cells, a base or size of 0 or not page aligned, or a window that wraps), when the +image doesn't fit the window or the window isn't in DRAM, when it can't be +reserved, when no free memory is left in it for executable allocations, when the +command-line window isn't inside the device-tree one, and when the kernel isn't +running at EL2 (``kvm-arm.mode=nvhe`` or ``protected``, for example). With no +window from either place, nothing changes. + +The kernel doesn't check the device tree's window against ``CTXR_D``. The +bootloader reads ``CTXR_D`` when it chooses the window and reserves the rest of +the armed range, so the check belongs there. The registers also don't exist on +every Apple core, and an undefined system register read this early can't be +recovered. + +``ctxr_exec_enforce=`` only decides whether a violation found by the boot-time +verifier (see below) is fatal, for a window from either place. Whether the +hardware confines a core does not depend on it, so everything else in this +document applies whenever a window is active, enforcing or not. + +How it works +============ + +* ``ctxr_exec_reserve()`` (from ``arm64_memblock_init()``) parses the window, + checks that ``[_text, _end)`` fits inside it, and reserves it from memblock. +* ``execmem_arch_setup()`` carves the window's free memory — a snapshot of the + ranges still free when the window was reserved, so the image and everything + else already reserved stay out — into a ``gen_pool`` and registers it with the + generic confinement core + (``CONFIG_EXECMEM_CONFINE``). The executable ranges (module text, BPF, + kprobes and, by inheritance, ftrace) are flagged ``EXECMEM_CONFINE``. +* A confined allocation reserves its virtual area in the normal module range — + preserving the +/-2 GiB branch reachability to the kernel — but backs it with + physical pages drawn from the window. Module *data* is not confined. +* If the window's pool is exhausted the allocation fails, exactly as a hardware + CTXR fault would prevent executing from outside the window; there is no buddy + fallback. +* Code in the window can't execute at EL1 on a confined core. The boot CPU isn't + confined and enters as it always has, through EL1 and ``finalise_el2``. Once it + has reserved a window at EL2, it sets a flag that later entries read with the + MMU off, and those switch to VHE in ``init_kernel_el`` without running at EL1. + With no window, every core enters as it always has. + +Userspace +========= + +On a core that enforces the window, EL0 can't execute from it either. The fetch +takes a permission fault that the page tables don't explain, so nothing can fix +it up. Anything EL0 can execute therefore lives outside the window: + +* The vDSO's code is part of the image, so userspace gets a copy in ordinary + memory. +* No window page reaches the page allocator. ``free_reserved_area()`` skips them, + so freed ``__init`` memory stays reserved, as does an initrd in the window. A + CMA area that overlaps the window isn't activated, and a reserved-memory node + whose init fails stays reserved. RAM in the window that the pool doesn't use is + not available to the rest of the system, so a window should be no larger than + the executable code needs. + +* The zero page is part of the image too, and every untouched anonymous page + that is read before it is written maps it, executable mappings included. With + a window active, ``arch_setup_zero_pages()`` allocates the zero pages from + memblock instead, outside the window. + +As a backstop, when a window is active, an EL0 instruction permission fault on a +present, executable mapping of a window page logs the task and its address and +PFN, then kills it with an uncatchable ``SIGSEGV`` rather than retrying. The +mapping is resolved to a PFN either through its folio or, for a ``VM_PFNMAP`` or +``VM_MIXEDMAP`` mapping with no struct page (a driver's ``remap_pfn_range()`` of +an image buffer), through a raw PFN walk. Any other fault takes the usual path. + +EFI runtime services +==================== + +EFI runtime code runs at EL2 on whichever core ``efi_rts_wq`` happens to be on, +so on a confined core it has to be inside the window too. When a window is +active and any ``EFI_RUNTIME_SERVICES_CODE`` region is outside it, or isn't +aligned to the kernel's page size, the kernel warns once and turns runtime +services off for that boot, as ``efi=noruntime`` would. When all of it is inside, +runtime services stay on. + +What a confined machine gives up +================================ + +CPU hotplug + The kernel runs at EL2 with no monitor below it, so no PSCI, and CPUs start + through spin-table, which has no way to take a CPU down. The CPUs have no + ``online`` file in sysfs, and anything that needs them offline fails. + +kexec and crash kernels + The relocation code runs from an ordinary page, outside the window, and a + crash dump can start on a confined core, where it would hang during a panic. + While a window is active, ``machine_kexec_prepare()`` refuses both, so + ``kexec_load()`` and ``kexec_file_load()`` fail with ``EOPNOTSUPP``. + +Hibernate + Resume runs from a ``trans_pgd`` page outside the window. This is unchanged + and untested under a window. + +Verifying confinement +===================== + +At ``late_initcall`` the kernel sweeps its page tables, and ``efi_mm`` when +runtime services are on, and logs the number of EL1-executable leaf mappings and +how many resolve outside the window. It also counts window pages that reached +the page allocator, and confined allocations whose backing is outside the +window, as violations. Any violation is fatal unless ``ctxr_exec_enforce=0`` is +given. This is also what stands in for the hardware fault when testing in a VM. +The check runs once: memory freed after it, such as init memory, relies on +``free_reserved_area()`` skipping window pages. diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index d335b9e43fe99..cd82b573e7bfd 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -24,6 +24,7 @@ ARM64 Architecture booting cpu-feature-registers cpu-hotplug + ctxr-execmem elf_hwcaps gcs hugetlbpage diff --git a/Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml b/Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml new file mode 100644 index 0000000000000..c40ff69fa37aa --- /dev/null +++ b/Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/chosen/aurora,ctxr-exec-window.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Executable window for a kernel running behind a CTXR + +maintainers: + - Diljit Singh + +description: + On Apple silicon from the M4 generation on, the boot firmware arms a code + text execute range on the secondary CPUs, so privileged code there can only + be fetched from one physical window. The bootloader tells the kernel where + that window is, and the kernel keeps all of its executable code inside it. + See Documentation/arch/arm64/ctxr-execmem.rst. + +select: + properties: + $nodename: + const: chosen + required: + - $nodename + +properties: + $nodename: + const: chosen + + aurora,ctxr-exec-window: + $ref: /schemas/types.yaml#/definitions/uint64-array + description: The physical window the kernel's executable code has to stay in. + items: + - description: + Physical base of the window, not zero and aligned to the kernel's + page size. + - description: + Size of the window in bytes, not zero and aligned to the kernel's + page size. + +additionalProperties: true +... diff --git a/MAINTAINERS b/MAINTAINERS index 32d275ec22fa5..94db75cad4059 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2090,6 +2090,7 @@ F: drivers/soc/apple/pmp.rs F: include/linux/mt7932.h F: include/linux/soc/apple/pmp-report.h F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml +F: Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/PCI/controller/apple-t8140.rst diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index a108e83dbc8a4..81f92a6fb50f7 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2535,6 +2535,31 @@ config RANDOMIZE_MODULE_REGION_FULL the region is exhausted. In this particular case of region exhaustion, modules might be able to fall back to a larger 2GB area. +config ARM64_CTXR_EXECMEM + bool "Confine kernel executable code to a fixed physical window (CTXR)" + depends on EXECMEM && MMU + select EXECMEM_CONFINE + select GENERIC_ALLOCATOR + select PTDUMP + help + On some Apple-silicon machines the boot firmware arms a CTXR (code + text execute range) on the secondary CPUs: privileged code can only + be fetched from inside a fixed physical window. To run the kernel at + EL2 behind such a window (rather than dropping it to EL1), every + executable physical page must live inside it. + + With this option the kernel reserves a single window — described by + the bootloader in /chosen/aurora,ctxr-exec-window, where it has already + placed the static image, or by the "ctxr_exec_window=" command-line + option for VM bring-up — and backs every run-time executable allocation + (modules, BPF, kprobes, ftrace) with pages drawn from the window's free + memory. A verifier sweeps the kernel page tables and reports (or, when + the window is enforcing, panics on) any executable mapping that escapes. + + This has no effect unless the bootloader or the command line supplies + a window, so a distribution kernel for Apple silicon can enable it. + Say Y for such a kernel, and N otherwise. + config CC_HAVE_STACKPROTECTOR_SYSREG def_bool $(cc-option,-mstack-protector-guard=sysreg -mstack-protector-guard-reg=sp_el0 -mstack-protector-guard-offset=0) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 15108bd7c3228..2689839863cf7 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -147,3 +147,8 @@ CONFIG_DMABUF_HEAPS_SYSTEM=y CONFIG_UDMABUF=y CONFIG_DETECT_HUNG_TASK=y CONFIG_EDAC_APPLE_SOC=m +# Confine kernel executable code to a fixed physical CTXR window so the kernel +# can run at EL2 on chips whose firmware locks the code-execute range. Inert +# unless the bootloader supplies a window (/chosen/aurora,ctxr-exec-window) or +# the ctxr_exec_window= command-line option is given. +CONFIG_ARM64_CTXR_EXECMEM=y diff --git a/arch/arm64/include/asm/ctxr_execmem.h b/arch/arm64/include/asm/ctxr_execmem.h new file mode 100644 index 0000000000000..b7bf460ddfcfa --- /dev/null +++ b/arch/arm64/include/asm/ctxr_execmem.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Confine the kernel's executable code to a fixed physical window so it can + * run at EL2 behind an Apple-silicon CTXR (code text execute range). + */ +#ifndef __ASM_CTXR_EXECMEM_H +#define __ASM_CTXR_EXECMEM_H + +#include + +#ifdef CONFIG_ARM64_CTXR_EXECMEM +/* Set once the boot CPU has a window at EL2; see init_kernel_el. */ +extern u32 ctxr_stay_el2; + +/* + * Parse the window (from the kernel command line or the device tree), validate + * that the kernel image fits inside it, and reserve it from memblock. Called + * once from arm64_memblock_init(), while memblock is still authoritative. + */ +void ctxr_exec_reserve(void); + +/* Close any hole left in the window before memblock releases free memory. */ +void ctxr_exec_keep_reserved(void); + +/* + * Build the gen_pool over the reserved window and register it with the execmem + * confinement core. Called from execmem_arch_setup(). Returns true when + * confinement is active and the caller should flag its executable ranges with + * EXECMEM_CONFINE. + */ +bool ctxr_exec_execmem_setup(void); + +/* + * False if a window is active and EFI runtime code can't run behind it. + * Warns once when it returns false. + */ +bool ctxr_exec_efi_runtime_ok(void); + +/* + * True once ctxr_exec_reserve() has reserved a window. Valid from + * arm64_memblock_init() until the end of init. + */ +bool ctxr_exec_window_reserved(void); + +/* + * Bytes of the window the executable pool gets, or 0 with no window. Valid as + * soon as ctxr_exec_reserve() has run, before the pool itself exists. + */ +u64 ctxr_exec_pool_bytes(void); + +/* True if cores confined to the window can't run this kernel. */ +bool ctxr_exec_secondaries_held(void); +#else +static inline void ctxr_exec_reserve(void) {} +static inline void ctxr_exec_keep_reserved(void) {} +static inline bool ctxr_exec_execmem_setup(void) { return false; } +static inline bool ctxr_exec_efi_runtime_ok(void) { return true; } +static inline bool ctxr_exec_window_reserved(void) { return false; } +static inline bool ctxr_exec_secondaries_held(void) { return false; } +static inline u64 ctxr_exec_pool_bytes(void) { return 0; } +#endif + +#endif /* __ASM_CTXR_EXECMEM_H */ diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 6a12d12effacb..f8116630c743b 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -314,6 +314,8 @@ static bool has_impdef_pmuv3(const struct arm64_cpu_capabilities *entry, int sco static const struct midr_range apple_broken_seis_cpus[] = { MIDR_ALL_VERSIONS(MIDR_APPLE_T6040_E), MIDR_ALL_VERSIONS(MIDR_APPLE_T6040_P), + MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_E), + MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_P), {}, }; diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 0ec90fd1754ed..0f080509f2dfe 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -249,6 +250,11 @@ static int __init arm64_efi_rt_init(void) if (!efi_enabled(EFI_RUNTIME_SERVICES)) return 0; + if (!ctxr_exec_efi_runtime_ok()) { + clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); + return 0; + } + p = arch_alloc_vmap_stack(THREAD_SIZE, NUMA_NO_NODE); if (!p) { pr_warn("Failed to allocate EFI runtime stack\n"); diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 87a822e5c4ca8..53e1c9af9fd22 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -117,6 +117,9 @@ SYM_CODE_START(primary_entry) blr x2 1: mov x0, x19 +#ifdef CONFIG_ARM64_CTXR_EXECMEM + mov x22, xzr // boot CPU: never read the CTXR flag +#endif bl init_kernel_el // w0=cpu_boot_mode mov x20, x0 @@ -307,6 +310,46 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) msr vbar_el2, x0 isb +#ifdef CONFIG_ARM64_CTXR_EXECMEM + /* + * A core confined to a CTXR window can't run the window's code at EL1, + * so once a window is active it switches to VHE here instead of going + * through EL1 and finalise_el2. Only a secondary or a resuming CPU is + * ever confined (x22 == 1): the boot CPU's first entry (x22 == 0) is + * not, and it runs before ctxr_stay_el2 is set and cleaned to the PoC, + * so it must not read the flag at all. + */ + cbz x22, .Lctxr_no_direct_vhe + adr_l x0, ctxr_stay_el2 + ldr w0, [x0] + cbz w0, .Lctxr_no_direct_vhe + + finalise_el2_state + + mov_q x0, HCR_HOST_VHE_FLAGS + msr_hcr_el2 x0 + isb + + mrs x0, mdcr_el2 + bic x0, x0, #MDCR_EL2_E2PB_MASK + bic x0, x0, #MDCR_EL2_E2TB_MASK + msr mdcr_el2, x0 + + mov_q x0, INIT_SCTLR_EL1_MMU_OFF + msr_s SYS_SCTLR_EL12, x0 + pre_disable_mmu_workaround + msr sctlr_el1, x0 + isb + + mov x0, #(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL2h) + msr spsr_el2, x0 + + mov w0, #BOOT_CPU_MODE_EL2 + orr x0, x0, #BOOT_CPU_FLAG_E2H + eret +.Lctxr_no_direct_vhe: +#endif + mov_q x1, INIT_SCTLR_EL1_MMU_OFF mrs x0, hcr_el2 @@ -336,6 +379,9 @@ SYM_FUNC_END(init_kernel_el) */ SYM_FUNC_START(secondary_holding_pen) mov x0, xzr +#ifdef CONFIG_ARM64_CTXR_EXECMEM + mov x22, #1 // a secondary may be confined +#endif bl init_kernel_el // w0=cpu_boot_mode mrs x2, mpidr_el1 mov_q x1, MPIDR_HWID_BITMASK @@ -354,6 +400,9 @@ SYM_FUNC_END(secondary_holding_pen) */ SYM_FUNC_START(secondary_entry) mov x0, xzr +#ifdef CONFIG_ARM64_CTXR_EXECMEM + mov x22, #1 // a secondary may be confined +#endif bl init_kernel_el // w0=cpu_boot_mode b secondary_startup SYM_FUNC_END(secondary_entry) diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 8f9bc2327dc85..9c4a407222ea1 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -6,6 +6,7 @@ * Copyright (C) Huawei Futurewei Technologies. */ +#include #include #include #include @@ -54,6 +55,15 @@ void machine_kexec_cleanup(struct kimage *kimage) */ int machine_kexec_prepare(struct kimage *kimage) { + /* + * The relocation code runs from an ordinary page, outside the window, + * and a crash dump can start on a core enforcing it. + */ + if (execmem_confine_active()) { + pr_err("Can't kexec: the kernel's code is confined to a CTXR window.\n"); + return -EOPNOTSUPP; + } + if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) { pr_err("Can't kexec: CPUs are stuck in the kernel.\n"); return -EBUSY; diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index f093cdf71be11..11a08cb03009c 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -100,6 +100,9 @@ SYM_FUNC_END(__cpu_suspend_enter) .pushsection ".idmap.text", "a" SYM_CODE_START(cpu_resume) mov x0, xzr +#ifdef CONFIG_ARM64_CTXR_EXECMEM + mov x22, #1 // a resuming CPU may be confined +#endif bl init_kernel_el mov x19, x0 // preserve boot mode bl __cpu_setup diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 1a30dcbaedb94..9cd9bc7e40812 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -799,6 +800,10 @@ void __init smp_prepare_cpus(unsigned int max_cpus) if (max_cpus == 0) return; + /* Leave CPUs confined to an unusable CTXR window in the bootloader. */ + if (ctxr_exec_secondaries_held()) + return; + /* * Initialise the present map (which describes the set of CPUs * actually populated at the present time) and release the diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index 592dd8668de46..962a7501cbae3 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -65,6 +66,24 @@ static int vdso_mremap(const struct vm_special_mapping *sm, return 0; } +/* + * The kernel image sits in the CTXR window, where EL0 can't execute on a core + * the window is enforced on, so EL0 gets a copy in ordinary memory. + */ +static struct page * __init vdso_code_page(unsigned long pfn) +{ + struct page *page; + + if (!execmem_confine_active()) + return pfn_to_page(pfn); + + page = alloc_page(GFP_KERNEL); + if (page) + copy_page(page_address(page), pfn_to_kaddr(pfn)); + + return page; +} + static int __init __vdso_init(enum vdso_abi abi) { int i; @@ -88,12 +107,23 @@ static int __init __vdso_init(enum vdso_abi abi) /* Grab the vDSO code pages. */ pfn = sym_to_pfn(vdso_info[abi].vdso_code_start); - for (i = 0; i < vdso_info[abi].vdso_pages; i++) - vdso_pagelist[i] = pfn_to_page(pfn + i); + for (i = 0; i < vdso_info[abi].vdso_pages; i++) { + vdso_pagelist[i] = vdso_code_page(pfn + i); + if (!vdso_pagelist[i]) + goto err_free; + } vdso_info[abi].cm->pages = vdso_pagelist; return 0; + +err_free: + /* Only the confined copies were allocated; image pages must not go. */ + if (execmem_confine_active()) + while (i--) + __free_page(vdso_pagelist[i]); + kfree(vdso_pagelist); + return -ENOMEM; } static int __setup_additional_pages(enum vdso_abi abi, diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 757f7eb07ccac..3c01c922582fb 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -6,6 +6,7 @@ obj-y := dma-mapping.o extable.o fault.o init.o \ obj-$(CONFIG_ARM64_CONTPTE) += contpte.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_PTDUMP) += ptdump.o +obj-$(CONFIG_ARM64_CTXR_EXECMEM) += ctxr_execmem.o obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o obj-$(CONFIG_TRANS_TABLE) += trans_pgd.o obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o diff --git a/arch/arm64/mm/ctxr_execmem.c b/arch/arm64/mm/ctxr_execmem.c new file mode 100644 index 0000000000000..dca84c0657b85 --- /dev/null +++ b/arch/arm64/mm/ctxr_execmem.c @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Confine the kernel's executable code to a fixed physical window. + * + * On Apple-silicon machines whose boot firmware arms a CTXR (code text execute + * range) on the secondary CPUs, privileged code may only be fetched from inside + * that hardware window. To run the kernel at EL2 rather than dropping it to EL1, + * every executable physical page must live inside one such window: the static + * image (placed there by the bootloader) and every run-time executable + * allocation (modules, BPF, kprobes, ftrace). + * + * This file owns the arm64 end of that contract: + * - it learns the window from the device tree (/chosen/aurora,ctxr-exec-window, + * emitted by the bootloader) or the command line (ctxr_exec_window=, used + * for VM/simulated bring-up), + * - reserves it from memblock and checks the image fits, + * - carves the memory still free when it was reserved into a gen_pool and + * hands it to the generic execmem confinement core, and + * - provides the page-table sweep that the verifier uses to prove, in a VM + * without real CTXR hardware, that no EL1-executable mapping escapes. + */ + +#define pr_fmt(fmt) "ctxr-execmem: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +static phys_addr_t ctxr_base __initdata; +static phys_addr_t ctxr_size __initdata; +static bool ctxr_enforce __initdata = true; +static bool ctxr_active __initdata; +/* What the executable pool will get, known as soon as the window is reserved. */ +static u64 ctxr_pool_bytes __ro_after_init; +static bool ctxr_hold_cpus __initdata; + +/* Read with the MMU off by cores entering init_kernel_el. */ +u32 ctxr_stay_el2 __section(".mmuoff.data.read"); + +/* + * Free, linear-mapped RAM inside the window, snapshotted just before the + * window is reserved. Anything already reserved at that point (the kernel + * image, the device tree, an initrd, /reserved-memory and memreserve entries, + * firmware or loader regions) and any hole or no-map region is excluded, so + * the executable pool can never hand out memory something else owns. + */ +#define CTXR_MAX_FREE 64 +static struct { + phys_addr_t start; + phys_addr_t end; +} ctxr_free[CTXR_MAX_FREE] __initdata; +static int ctxr_nr_free __initdata; + +/* From the command line. A size of 0 means none was given. */ +static phys_addr_t ctxr_cmd_base __initdata; +static phys_addr_t ctxr_cmd_size __initdata; + +/* A page-aligned base and size, neither of them 0, that don't wrap. */ +static bool __init ctxr_window_valid(phys_addr_t base, phys_addr_t size) +{ + return base && size && PAGE_ALIGNED(base) && PAGE_ALIGNED(size) && + base + size > base; +} + +/* ctxr_exec_window=@ */ +static int __init ctxr_exec_window_param(char *arg) +{ + phys_addr_t base, size; + char *p, *end; + + if (!arg) + goto bad; + size = memparse(arg, &end); + if (end == arg || *end != '@') + goto bad; + p = end + 1; + base = memparse(p, &end); + if (end == p || *end || !ctxr_window_valid(base, size)) + goto bad; + + ctxr_cmd_base = base; + ctxr_cmd_size = size; + return 0; + +bad: + pr_warn("ignoring ctxr_exec_window=%s\n", arg ?: ""); + return 0; +} +early_param("ctxr_exec_window", ctxr_exec_window_param); + +static int __init ctxr_exec_enforce_param(char *arg) +{ + if (kstrtobool(arg, &ctxr_enforce)) + pr_warn("ignoring ctxr_exec_enforce=%s\n", arg ?: ""); + return 0; +} +early_param("ctxr_exec_enforce", ctxr_exec_enforce_param); + +/* 1 for a window, 0 for none, -EINVAL for one that can't be used as given. */ +static int __init ctxr_parse_dt(phys_addr_t *base, phys_addr_t *size) +{ + unsigned long chosen; + const __be32 *prop; + int len; + + chosen = of_get_flat_dt_subnode_by_name(of_get_flat_dt_root(), "chosen"); + if ((long)chosen < 0) + return 0; + + /* */ + prop = of_get_flat_dt_prop(chosen, "aurora,ctxr-exec-window", &len); + if (!prop) + return 0; + if (len != 4 * sizeof(__be32)) + return -EINVAL; + + *base = ((u64)be32_to_cpu(prop[0]) << 32) | be32_to_cpu(prop[1]); + *size = ((u64)be32_to_cpu(prop[2]) << 32) | be32_to_cpu(prop[3]); + return ctxr_window_valid(*base, *size) ? 1 : -EINVAL; +} + +/* Returns how much the pool would get, in whole pages. */ +static phys_addr_t __init ctxr_snapshot_free(phys_addr_t base, phys_addr_t size) +{ + phys_addr_t start, end, wend = base + size, usable = 0; + u64 i; + + ctxr_nr_free = 0; + for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, + NULL) { + if (start >= wend) + break; + start = max(start, base); + end = min(end, wend); + if (start >= end) + continue; + if (ctxr_nr_free == CTXR_MAX_FREE) { + pr_warn("more than %d free ranges in the window; ignoring the rest\n", + CTXR_MAX_FREE); + break; + } + ctxr_free[ctxr_nr_free].start = start; + ctxr_free[ctxr_nr_free].end = end; + ctxr_nr_free++; + if (round_down(end, PAGE_SIZE) > PAGE_ALIGN(start)) + usable += round_down(end, PAGE_SIZE) - PAGE_ALIGN(start); + } + + return usable; +} + +/* Reserve a window. False, with the reason logged, if it can't work. */ +static bool __init ctxr_establish(phys_addr_t base, phys_addr_t size) +{ + phys_addr_t img_start = __pa_symbol(_text), img_end = __pa_symbol(_end); + phys_addr_t usable; + phys_addr_t dram_start = memblock_start_of_DRAM(); + phys_addr_t dram_end = memblock_end_of_DRAM(); + + if (img_start < base || img_end > base + size) { + pr_err("kernel image [%pa, %pa) does not fit window [%pa, +%llu MiB)\n", + &img_start, &img_end, &base, + (unsigned long long)(size >> 20)); + return false; + } + + if (base < dram_start || base + size > dram_end) { + pr_err("window [%pa, +%llu MiB) is not within DRAM\n", + &base, (unsigned long long)(size >> 20)); + return false; + } + + /* + * Must run before the window is reserved, or every range looks taken. + * The pool is built after the secondaries start, so check it now. + */ + usable = ctxr_snapshot_free(base, size); + if (!usable) { + pr_err("window has no free space for executable allocations\n"); + return false; + } + + if (memblock_reserve(base, size)) { + pr_err("failed to reserve window [%pa, +%llu MiB)\n", + &base, (unsigned long long)(size >> 20)); + return false; + } + + execmem_confine_set_window(base, size); + + ctxr_base = base; + ctxr_size = size; + ctxr_pool_bytes = usable; + ctxr_active = true; + + pr_info("reserved exec window [%pa, +%llu MiB), %s\n", + &base, (unsigned long long)(size >> 20), + ctxr_enforce ? "enforcing" : "reporting"); + return true; +} + +/* + * A reserved-memory node whose init fails goes back to memblock, which can leave a + * hole in the window for memblock_free_all() to release. + */ +void __init ctxr_exec_keep_reserved(void) +{ + if (ctxr_active) + memblock_reserve(ctxr_base, ctxr_size); +} + +/* + * The secondaries are confined to a window this kernel can't use, so they + * can't run it. The boot CPU isn't confined and carries on alone. + */ +static void __init ctxr_hold_secondaries(const char *why) +{ + ctxr_hold_cpus = true; + pr_err("%s: keeping the secondary CPUs offline\n", why); +} + +bool __init ctxr_exec_secondaries_held(void) +{ + return ctxr_hold_cpus; +} + +/* Is [b, b + s) wholly inside [base, base + size)? */ +static bool __init ctxr_within(phys_addr_t b, phys_addr_t s, + phys_addr_t base, phys_addr_t size) +{ + return b >= base && b + s <= base + size; +} + +void __init ctxr_exec_reserve(void) +{ + phys_addr_t base = 0, size = 0; + int dt = ctxr_parse_dt(&base, &size); + + /* + * A command-line window and a device-tree window take the same EL2 and + * hold checks: a core confined to the firmware's window can't run this + * kernel from outside it, whichever supplied the bounds. The command + * line can only pick a window inside the firmware's one. + */ + if (dt < 0) + ctxr_hold_secondaries("malformed aurora,ctxr-exec-window"); + else if (dt || ctxr_cmd_size) { + if (!is_kernel_in_hyp_mode()) + ctxr_hold_secondaries("window given but the kernel is not at EL2"); + else if (ctxr_cmd_size && dt && + !ctxr_within(ctxr_cmd_base, ctxr_cmd_size, base, size)) + ctxr_hold_secondaries("ctxr_exec_window= is not inside the device-tree window"); + else if (ctxr_cmd_size && !ctxr_establish(ctxr_cmd_base, ctxr_cmd_size)) + ctxr_hold_secondaries("unusable ctxr_exec_window="); + else if (!ctxr_cmd_size && !ctxr_establish(base, size)) + ctxr_hold_secondaries("unusable window"); + } + + /* + * Secondaries and resuming CPUs read ctxr_stay_el2 with the MMU off, so + * set it on every boot (0 when no window is active) and clean it to the + * PoC here, before any secondary is released. The boot CPU never reads + * it. See init_kernel_el. + */ + WRITE_ONCE(ctxr_stay_el2, ctxr_active); + dcache_clean_inval_poc((unsigned long)&ctxr_stay_el2, + (unsigned long)&ctxr_stay_el2 + sizeof(ctxr_stay_el2)); +} + +static void __init ctxr_pool_add_gap(struct gen_pool *pool, phys_addr_t from, + phys_addr_t to) +{ + from = PAGE_ALIGN(from); + to = round_down(to, PAGE_SIZE); + if (to <= from) + return; + + if (gen_pool_add_virt(pool, (unsigned long)__va(from), from, to - from, + NUMA_NO_NODE)) + pr_err("failed to add [%pa, %pa) to the pool\n", &from, &to); +} + +bool __init ctxr_exec_execmem_setup(void) +{ + struct gen_pool *pool; + int i; + + if (!ctxr_active) + return false; + + pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE); + if (!pool) + goto fail; + + /* + * Only memory that was free before the window was reserved joins the + * pool; the image and everything else already reserved stays out. + */ + for (i = 0; i < ctxr_nr_free; i++) + ctxr_pool_add_gap(pool, ctxr_free[i].start, ctxr_free[i].end); + + if (!gen_pool_size(pool) || execmem_confine_init(pool, ctxr_enforce)) + goto fail; + + pr_info("executable pool: %zu MiB usable in %d range(s) of a %llu MiB window (%llu MiB already reserved)\n", + gen_pool_size(pool) >> 20, ctxr_nr_free, + (unsigned long long)(ctxr_size >> 20), + (unsigned long long)((ctxr_size - gen_pool_size(pool)) >> 20)); + return true; + +fail: + /* Only an allocation failure gets here, and confined CPUs are running. */ + panic("ctxr-execmem: cannot build the executable pool\n"); +} + +u64 ctxr_exec_pool_bytes(void) +{ + return ctxr_pool_bytes; +} + +bool __init ctxr_exec_window_reserved(void) +{ + return ctxr_active; +} + +/* + * A core enforcing the window can't fetch firmware code outside it, and + * efi_rts_wq can run on any core. Misaligned code also makes the misaligned + * regions after it executable, wherever they are. + */ +bool __init ctxr_exec_efi_runtime_ok(void) +{ + efi_memory_desc_t *md; + + if (!IS_ENABLED(CONFIG_EFI) || !ctxr_active) + return true; + + for_each_efi_memory_desc(md) { + phys_addr_t start = md->phys_addr; + phys_addr_t end = start + (md->num_pages << EFI_PAGE_SHIFT); + + if (md->type != EFI_RUNTIME_SERVICES_CODE) + continue; + if (start >= ctxr_base && end <= ctxr_base + ctxr_size && + PAGE_ALIGNED(start) && PAGE_ALIGNED(end)) + continue; + + pr_warn("EFI runtime services off: firmware code [%pa, %pa) is outside the window or misaligned\n", + &start, &end); + return false; + } + + return true; +} + +/* + * Page-table sweep used by the confinement verifier. For every privileged + * (PXN-clear) executable leaf in the kernel and EFI page tables, count it and + * flag any whose physical output lies outside the window. This is what stands + * in, in a VM, for the hardware CTXR fault. + */ +struct ctxr_verify { + struct ptdump_state ptdump; + phys_addr_t base; + phys_addr_t end; + unsigned long nr_exec; + unsigned long violations; +}; + +static void ctxr_check_leaf(struct ctxr_verify *v, unsigned long addr, + pteval_t val, phys_addr_t leaf_size) +{ + phys_addr_t pa, last; + + if (!(val & PTE_VALID)) + return; + if (val & PTE_PXN) /* not executable at EL1 */ + return; + + v->nr_exec++; + + pa = PFN_PHYS(pte_pfn(__pte(val))); + last = pa + leaf_size - 1; + if (pa < v->base || last >= v->end) { + v->violations++; + pr_warn("EL1-exec mapping va %#lx -> phys [%pa, %pa] outside window\n", + addr, &pa, &last); + } +} + +static void ctxr_note_pte(struct ptdump_state *st, unsigned long addr, pte_t pte) +{ + ctxr_check_leaf(container_of(st, struct ctxr_verify, ptdump), addr, + pte_val(pte), PAGE_SIZE); +} + +static void ctxr_note_pmd(struct ptdump_state *st, unsigned long addr, pmd_t pmd) +{ + ctxr_check_leaf(container_of(st, struct ctxr_verify, ptdump), addr, + pmd_val(pmd), PMD_SIZE); +} + +static void ctxr_note_pud(struct ptdump_state *st, unsigned long addr, pud_t pud) +{ + ctxr_check_leaf(container_of(st, struct ctxr_verify, ptdump), addr, + pud_val(pud), PUD_SIZE); +} + +/* + * p4d/pgd leaves never carry executable kernel mappings on arm64, but the + * page-table walker calls every level's note callback (including for holes and + * a final flush), so each must be non-NULL. + */ +static void ctxr_note_p4d(struct ptdump_state *st, unsigned long addr, + p4d_t p4d) +{ +} + +static void ctxr_note_pgd(struct ptdump_state *st, unsigned long addr, + pgd_t pgd) +{ +} + +static void ctxr_note_flush(struct ptdump_state *st) +{ +} + +unsigned long execmem_confine_arch_verify(phys_addr_t base, phys_addr_t size, + unsigned long *nr_exec) +{ + struct ctxr_verify v = { + .base = base, + .end = base + size, + .ptdump = { + .note_page_pte = ctxr_note_pte, + .note_page_pmd = ctxr_note_pmd, + .note_page_pud = ctxr_note_pud, + .note_page_p4d = ctxr_note_p4d, + .note_page_pgd = ctxr_note_pgd, + .note_page_flush = ctxr_note_flush, + .range = (struct ptdump_range[]) { + { _PAGE_OFFSET(vabits_actual), ~0UL }, + { 0, 0 } + }, + }, + }; + + ptdump_walk_pgd(&v.ptdump, &init_mm, NULL); + + /* EFI runtime code has its own page tables, at user addresses. */ + if (efi_enabled(EFI_RUNTIME_SERVICES)) { + v.ptdump.range = (struct ptdump_range[]) { + { 0, TASK_SIZE_64 }, + { 0, 0 } + }; + ptdump_walk_pgd(&v.ptdump, &efi_mm, NULL); + } + + if (nr_exec) + *nr_exec = v.nr_exec; + return v.violations; +} diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 2ea95d7490dca..c95f3ca4c6700 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -602,6 +604,52 @@ static bool is_invalid_gcs_access(struct vm_area_struct *vma, u64 esr) return false; } +/* + * A core enforcing the CTXR window refuses EL0 fetches from it with a permission + * fault that the page tables don't explain. Nothing can fix that up, so a retry + * would fault forever with the task still runnable. The window page may be + * mapped without a struct page (a driver's remap_pfn_range() of an image + * buffer), so fall back to a raw PFN walk and only ever compare the PFN. + */ +static bool el0_fetch_refused_by_window(struct mm_struct *mm, unsigned long addr, + unsigned long esr, unsigned int mm_flags) +{ + struct vm_area_struct *vma; + unsigned long pfn = 0; + + if (!is_el0_instruction_abort(esr) || !esr_fsc_is_permission_fault(esr) || + !execmem_confine_active()) + return false; + + if (mmap_read_lock_killable(mm)) + return false; + + vma = vma_lookup(mm, addr); + if (vma && (vma->vm_flags & VM_EXEC) && !fault_from_pkey(vma, mm_flags)) { + struct follow_pfnmap_args pfnmap = { .vma = vma, .address = addr, + .allow_subpage = true }; + struct folio_walk fw; + + if (folio_walk_start(&fw, vma, addr, FW_ZEROPAGE)) { + if (fw.level == FW_LEVEL_PTE && pte_user_exec(fw.pte)) + pfn = pte_pfn(fw.pte); + folio_walk_end(&fw, vma); + } else if ((vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) && + !follow_pfnmap_start(&pfnmap)) { + pfn = pfnmap.pfn; + follow_pfnmap_end(&pfnmap); + } + } + mmap_read_unlock(mm); + + if (!pfn || !execmem_confine_pfn_in_window(pfn)) + return false; + + pr_warn_ratelimited("%s[%d]: EL0 fetch at %#lx from pfn %#lx refused by the CTXR window, killing it\n", + current->comm, task_pid_nr(current), addr, pfn); + return true; +} + static int __kprobes do_page_fault(unsigned long far, unsigned long esr, struct pt_regs *regs) { @@ -679,6 +727,11 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); + if (el0_fetch_refused_by_window(mm, addr, esr, mm_flags)) { + force_fatal_sig(SIGSEGV); + return 0; + } + if (!(mm_flags & FAULT_FLAG_USER)) goto lock_mmap; diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index a3acc7c802955..f5d7f5d25f741 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -36,12 +36,14 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -298,12 +300,22 @@ void __init arm64_memblock_init(void) } early_init_fdt_scan_reserved_mem(); + + ctxr_exec_reserve(); } void __init bootmem_init(void) { unsigned long min, max; + /* + * unflatten_device_tree() has run fdt_scan_reserved_mem_late(), which + * can free a failed reserved-memory node and punch a hole in the + * window. Close it before arch_reserve_crashkernel() can place the + * crash kernel there, only for it to be freed later as a window page. + */ + ctxr_exec_keep_reserved(); + min = PFN_UP(memblock_start_of_DRAM()); max = PFN_DOWN(memblock_end_of_DRAM()); @@ -333,7 +345,34 @@ void __init bootmem_init(void) void __init arch_setup_zero_pages(void) { - __zero_page = phys_to_page(__pa_symbol(empty_zero_page)); + unsigned long off; + void *zero = NULL; + + /* + * Every untouched anonymous page that is read before it is written + * maps the zero page, executable mappings included, and EL0 can't + * execute from an active CTXR window, where empty_zero_page lies. + */ + if (ctxr_exec_window_reserved()) { + zero = memblock_alloc(ARCH_ZERO_PAGE_SIZE, ARCH_ZERO_PAGE_SIZE); + if (!zero) + pr_warn("ctxr-execmem: no memory for a zero page outside the window\n"); + } + + if (!zero) { + __zero_page = phys_to_page(__pa_symbol(empty_zero_page)); + return; + } + + /* + * As cpu_enable_mte() does for empty_zero_page: the zero page is + * always mapped pte_special(), so nothing else clears its tags. + */ + if (system_supports_mte()) + for (off = 0; off < ARCH_ZERO_PAGE_SIZE; off += PAGE_SIZE) + mte_clear_page_tags(zero + off); + + __zero_page = virt_to_page(zero); } void __init arch_mm_preinit(void) @@ -516,9 +555,20 @@ struct execmem_info __init *execmem_arch_setup(void) { unsigned long fallback_start = 0, fallback_end = 0; unsigned long start = 0, end = 0; + enum execmem_range_flags cflags = 0; module_init_limits(); + /* + * When a CTXR executable window is active, back the executable ranges + * (text, BPF, kprobes and — by inheritance — ftrace) with pages drawn + * from the window so every executable physical page stays confined. + * Module *data* (EXECMEM_MODULE_DATA, set explicitly below) is never + * executed and is left on ordinary vmalloc backing. + */ + if (ctxr_exec_execmem_setup()) + cflags = EXECMEM_CONFINE; + /* * Where possible, prefer to allocate within direct branch range of the * kernel such that no PLTs are necessary. @@ -545,18 +595,30 @@ struct execmem_info __init *execmem_arch_setup(void) .alignment = 1, .fallback_start = fallback_start, .fallback_end = fallback_end, + .flags = cflags, + }, + [EXECMEM_MODULE_DATA] = { + .start = start, + .end = end, + .pgprot = PAGE_KERNEL, + .alignment = 1, + .fallback_start = fallback_start, + .fallback_end = fallback_end, }, [EXECMEM_KPROBES] = { .start = VMALLOC_START, .end = VMALLOC_END, .pgprot = PAGE_KERNEL_ROX, .alignment = 1, + .flags = cflags, }, [EXECMEM_BPF] = { .start = VMALLOC_START, .end = VMALLOC_END, .pgprot = PAGE_KERNEL, .alignment = 1, + /* JIT images can't use up what modules need. */ + .flags = cflags ? cflags | EXECMEM_CONFINE_BOUNDED : 0, }, }, }; diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c1279fabce473..1cb77cbf298b4 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -2409,7 +2410,15 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len) u64 bpf_jit_alloc_exec_limit(void) { - return VMALLOC_END - VMALLOC_START; + /* + * Under a CTXR window every JIT image comes from the window's pool, which + * is tens of MiB, not the vmalloc space. Sizing the limit from the pool + * keeps unprivileged programs (bpf_jit_limit defaults to half of this) from + * using up the memory that module text and privileged BPF need. + */ + u64 pool = ctxr_exec_pool_bytes(); + + return pool ?: VMALLOC_END - VMALLOC_START; } /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index 37d4c94afc31e..38e5de49a1e3b 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -122,22 +122,25 @@ static struct cpuidle_driver apple_idle_driver = { .state_count = STATE_COUNT, }; +/* SoCs whose CYC_OVRD the OS can write: M1 to M3. */ +static const char *const apple_cyc_ovrd_machines[] = { + "apple,t8103", "apple,t6000", "apple,t6001", "apple,t6002", + "apple,t8112", "apple,t6020", "apple,t6021", "apple,t6022", + "apple,t8122", "apple,t6030", "apple,t6031", "apple,t6032", + "apple,t6034", + NULL +}; + static int apple_cpuidle_probe(struct platform_device *pdev) { - struct device_node *np = pdev->dev.of_node; - bool wfi_only = of_device_is_compatible(np, "apple,t6041-cpuidle") || - of_device_is_compatible(np, "apple,t8132-cpuidle"); - /* - * CPU PD writes CYC_OVRD, which is an undefined instruction on - * T8140, the M4 SoCs (T6041, T8132) and T6050, where it is also not - * available to a guest. Their WFI already lets the hardware - * power-gate idle cores, so both entries must use WFI. Keep index 1: + * CPU PD writes CYC_OVRD, which only M1 to M3 let the OS write. From + * M4 on it is locked and the write is undefined, and WFI already + * power-gates an idle core, so both entries use WFI. Keep index 1: * the generic s2idle selector skips fallback index 0, otherwise * tick_freeze() and timekeeping suspension are never reached. */ - if (of_machine_is_compatible("apple,t8140") || - of_machine_is_compatible("apple,t6050") || wfi_only) + if (!of_machine_compatible_match(apple_cyc_ovrd_machines)) apple_idle_driver.states[STATE_PWRDOWN] = apple_idle_driver.states[STATE_WFI]; return cpuidle_register(&apple_idle_driver, NULL); @@ -178,21 +181,14 @@ static int __init apple_cpuidle_init(void) if (!of_machine_is_compatible("apple,arm-platform")) return 0; - if (!(of_machine_is_compatible("apple,t8103") || - of_machine_is_compatible("apple,t8112") || - of_machine_is_compatible("apple,t8122") || - of_machine_is_compatible("apple,t6000") || - of_machine_is_compatible("apple,t6001") || - of_machine_is_compatible("apple,t6002") || - of_machine_is_compatible("apple,t6020") || - of_machine_is_compatible("apple,t6021") || - of_machine_is_compatible("apple,t6022") || - of_machine_is_compatible("apple,t6030") || - of_machine_is_compatible("apple,t6031") || - of_machine_is_compatible("apple,t6032") || - of_machine_is_compatible("apple,t6034") || - of_machine_is_compatible("apple,t6050") || - of_machine_is_compatible("apple,t8140"))) + /* + * The M1 to M3 machines (CPU PD) plus the M4-and-later ones that load + * the driver for its WFI state. The device-tree parts above are matched + * by their cpuidle node instead. + */ + if (!of_machine_compatible_match(apple_cyc_ovrd_machines) && + !of_machine_is_compatible("apple,t6050") && + !of_machine_is_compatible("apple,t8140")) return 0; pdev = platform_device_register_simple("cpuidle-apple", -1, NULL, 0); diff --git a/include/linux/execmem.h b/include/linux/execmem.h index 1bd34925d1aa9..2ab917b2d4bd6 100644 --- a/include/linux/execmem.h +++ b/include/linux/execmem.h @@ -48,10 +48,20 @@ enum execmem_type { * enum execmem_range_flags - options for executable memory allocations * @EXECMEM_KASAN_SHADOW: allocate kasan shadow * @EXECMEM_ROX_CACHE: allocations should use ROX cache of huge pages + * @EXECMEM_CONFINE: back allocations for this range with physical + * pages drawn from the confined executable window + * (see CONFIG_EXECMEM_CONFINE). The virtual range + * is still taken from [start, end); only the + * physical backing is restricted. + * @EXECMEM_CONFINE_BOUNDED: with EXECMEM_CONFINE, allocations for this range + * never take the confined pool below the share + * kept for the other confined ranges (module text) */ enum execmem_range_flags { EXECMEM_KASAN_SHADOW = (1 << 0), EXECMEM_ROX_CACHE = (1 << 1), + EXECMEM_CONFINE = (1 << 2), + EXECMEM_CONFINE_BOUNDED = (1 << 3), }; #ifdef CONFIG_ARCH_HAS_EXECMEM_ROX @@ -204,4 +214,62 @@ void execmem_init(void); static inline void execmem_init(void) {} #endif +#ifdef CONFIG_EXECMEM_CONFINE +struct gen_pool; +struct execmem_range; + +/** + * execmem_confine_init - register the confined executable physical window + * @pool: gen_pool that hands out page-aligned chunks of the window + * (entries added with linear-map virtual addresses) + * @enforce: if true, a confinement violation found by the verifier is + * fatal; if false it is only reported + * + * Must be called before the first confined allocation, i.e. before + * execmem_arch_setup() marks any range with EXECMEM_CONFINE. + * + * Return: 0 on success, negative errno otherwise. + */ +int execmem_confine_init(struct gen_pool *pool, bool enforce); + +/* True once a window has been registered. */ +bool execmem_confine_active(void); + +/* + * Record the window once it is reserved. This comes ahead of the pool, so that + * nothing can release window pages between the two. + */ +void execmem_confine_set_window(phys_addr_t base, phys_addr_t size); + +/* + * True if @pfn is in the window. Userspace can't execute from the window, so + * such a page must never reach the page allocator. + */ +bool execmem_confine_pfn_in_window(unsigned long pfn); + +/* True if any of [@start, @start + @size) is in the window. */ +bool execmem_confine_overlaps(phys_addr_t start, phys_addr_t size); + +/* + * Internal backing allocator/free used by mm/execmem.c for EXECMEM_CONFINE + * ranges. Not for general use. + */ +void *execmem_confine_alloc(struct execmem_range *range, size_t size, + pgprot_t pgprot, unsigned long vm_flags); +bool execmem_confine_free(void *ptr); + +/* + * Architecture page-table sweep. Walk the kernel page tables and, for every + * privileged-executable leaf entry, add 1 to *@nr_exec and, if its output + * physical address lies outside [base, base + size), add 1 to the return + * value. A weak default returns 0 (no sweep available). + */ +unsigned long execmem_confine_arch_verify(phys_addr_t base, phys_addr_t size, + unsigned long *nr_exec); +#else +static inline bool execmem_confine_active(void) { return false; } +static inline bool execmem_confine_pfn_in_window(unsigned long pfn) { return false; } +static inline bool execmem_confine_overlaps(phys_addr_t start, phys_addr_t size) { return false; } +#endif /* CONFIG_EXECMEM_CONFINE */ + #endif /* _LINUX_EXECMEM_ALLOC_H */ diff --git a/mm/Kconfig b/mm/Kconfig index 587af2f594350..3dcecd4a28e62 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1474,6 +1474,19 @@ config IOMMU_MM_DATA config EXECMEM bool +config EXECMEM_CONFINE + bool + depends on EXECMEM && MMU + select GENERIC_ALLOCATOR + help + Back executable-memory ranges flagged EXECMEM_CONFINE with physical + pages drawn from a single fixed window registered by the architecture, + rather than from arbitrary buddy pages. This keeps every run-time + executable physical page (modules, BPF, kprobes, ftrace) inside a + hardware-enforceable code region such as an Apple-silicon CTXR window. + + Selected by architectures that provide a window; no effect otherwise. + config NUMA_MEMBLKS bool diff --git a/mm/Makefile b/mm/Makefile index 9fa4e1fb7e1ea..188db871956e8 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -145,6 +145,7 @@ obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o obj-$(CONFIG_GENERIC_IOREMAP) += ioremap.o obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o +obj-$(CONFIG_EXECMEM_CONFINE) += execmem_confine.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o diff --git a/mm/cma.c b/mm/cma.c index a10ea37a261da..3894990f588d9 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -138,13 +139,38 @@ bool cma_validate_zones(struct cma *cma) return true; } +/* CMA hands its pages to the page allocator, and window pages must stay out of it. */ +static bool __init cma_overlaps_exec_window(struct cma *cma) +{ + for (int r = 0; r < cma->nranges; r++) { + struct cma_memrange *cmr = &cma->ranges[r]; + + if (execmem_confine_overlaps(PFN_PHYS(cmr->base_pfn), + PFN_PHYS(cmr->count))) + return true; + } + + return false; +} + static void __init cma_activate_area(struct cma *cma) { unsigned long pfn, end_pfn, early_pfn[CMA_MAX_RANGES]; - int allocrange, r; + int allocrange = 0, r; struct cma_memrange *cmr; unsigned long bitmap_count, count; + /* + * Check this before allocating anything: a bitmap-allocation failure + * jumps to cleanup, which would free the pages to the buddy unless the + * reserve-on-error bit is already set. + */ + if (cma_overlaps_exec_window(cma)) { + pr_err("CMA area %s overlaps the executable window\n", cma->name); + cma_reserve_pages_on_error(cma); + goto cleanup; + } + for (allocrange = 0; allocrange < cma->nranges; allocrange++) { cmr = &cma->ranges[allocrange]; early_pfn[allocrange] = cmr->early_pfn; diff --git a/mm/execmem.c b/mm/execmem.c index 74a178a87e758..cc68c2cc0f55d 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -36,6 +36,16 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size, unsigned long end = range->end; void *p; +#ifdef CONFIG_EXECMEM_CONFINE + /* + * Confined ranges draw their physical backing from a fixed window + * rather than from arbitrary buddy pages, so that every executable + * physical page stays inside a CTXR-enrolled region. + */ + if (range->flags & EXECMEM_CONFINE) + return execmem_confine_alloc(range, size, pgprot, vm_flags); +#endif + if (kasan) vm_flags |= VM_DEFER_KMEMLEAK; @@ -503,6 +513,11 @@ void execmem_free(void *ptr) */ WARN_ON(in_interrupt()); +#ifdef CONFIG_EXECMEM_CONFINE + if (execmem_confine_free(ptr)) + return; +#endif + if (!execmem_cache_free(ptr)) vfree(ptr); } diff --git a/mm/execmem_confine.c b/mm/execmem_confine.c new file mode 100644 index 0000000000000..c839a2e07537a --- /dev/null +++ b/mm/execmem_confine.c @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Confined executable memory. + * + * Some platforms can only execute privileged code whose *physical* address + * lies inside a fixed, hardware-enforced window (for example an Apple-silicon + * CTXR "code text execute range" armed by the boot firmware). To run at such a + * privilege level the kernel must keep every executable physical page inside + * that window: the static image is placed there by the bootloader, and every + * run-time executable allocation (modules, BPF, kprobes, ftrace trampolines) + * must draw its backing pages from it too. + * + * execmem already funnels all run-time executable allocations through a small + * set of ranges. This file backs the ranges flagged EXECMEM_CONFINE with pages + * taken from a gen_pool carved over the window, mapped into the range's virtual + * area so that branch-reachability (the +/-2G module window) is preserved while + * the physical backing stays confined. + * + * The architecture registers the window and pool via execmem_confine_init() and + * sets EXECMEM_CONFINE on the relevant ranges in execmem_arch_setup(). + */ + +#define pr_fmt(fmt) "execmem-confine: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" +#include "vmalloc.h" + +struct confine_chunk { + struct vm_struct *area; + size_t size; + struct page **pages; /* one window page each */ +}; + +static phys_addr_t confine_base __ro_after_init; +static phys_addr_t confine_size __ro_after_init; +static struct gen_pool *confine_pool __ro_after_init; +static bool confine_enforce __ro_after_init; +static bool confine_ready __ro_after_init; +/* + * Pool bytes an EXECMEM_CONFINE_BOUNDED range (BPF) may never take, so that a + * pool filled with JIT images still loads modules. + */ +static size_t confine_reserve __ro_after_init; + +/* keyed by mapped-VA PFN -> struct confine_chunk */ +static DEFINE_XARRAY(confine_chunks); +/* serialises pool (de)allocation and the xarray */ +static DEFINE_MUTEX(confine_lock); + +/* Return a chunk's pages to the window pool. The caller holds confine_lock. */ +static void __confine_pool_free(struct page **pages, unsigned int nr) +{ + unsigned int i; + + lockdep_assert_held(&confine_lock); + for (i = 0; i < nr; i++) + gen_pool_free(confine_pool, + (unsigned long)page_address(pages[i]), PAGE_SIZE); +} + +/* Return a chunk's pages to the window pool. The caller holds no lock. */ +static void confine_pool_free(struct page **pages, unsigned int nr) +{ + mutex_lock(&confine_lock); + __confine_pool_free(pages, nr); + mutex_unlock(&confine_lock); +} + +void __init execmem_confine_set_window(phys_addr_t base, phys_addr_t size) +{ + confine_base = base; + confine_size = size; +} + +int __init execmem_confine_init(struct gen_pool *pool, bool enforce) +{ + if (WARN_ON(confine_ready)) + return -EBUSY; + if (WARN_ON(!pool || !confine_size)) + return -EINVAL; + + confine_pool = pool; + confine_enforce = enforce; + confine_reserve = PAGE_ALIGN(gen_pool_size(pool) / 4); + smp_wmb(); + confine_ready = true; + + { + phys_addr_t end = confine_base + confine_size; + + pr_info("window [%pa, %pa) (%llu MiB), %s; %zu KiB kept for module text\n", + &confine_base, &end, + (unsigned long long)(confine_size >> 20), + enforce ? "enforcing" : "reporting", confine_reserve >> 10); + } + return 0; +} + +bool execmem_confine_active(void) +{ + return confine_ready; +} + +static bool phys_in_window(phys_addr_t pa) +{ + return pa >= confine_base && pa < confine_base + confine_size; +} + +bool execmem_confine_pfn_in_window(unsigned long pfn) +{ + return phys_in_window(PFN_PHYS(pfn)); +} + +bool __init execmem_confine_overlaps(phys_addr_t start, phys_addr_t size) +{ + return start < confine_base + confine_size && confine_base < start + size; +} + +/* Window pages stay reserved for good, so one that isn't went to the allocator. */ +static unsigned long confine_count_freed_pages(void) +{ + unsigned long pfn, nr = 0; + + for_each_valid_pfn(pfn, PFN_DOWN(confine_base), + PFN_DOWN(confine_base + confine_size)) { + if (!PageReserved(pfn_to_page(pfn))) + nr++; + } + + return nr; +} + +void *execmem_confine_alloc(struct execmem_range *range, size_t size, + pgprot_t pgprot, unsigned long vm_flags) +{ + struct confine_chunk *ck; + struct vm_struct *area; + bool bounded = range->flags & EXECMEM_CONFINE_BOUNDED; + struct page **pages; + unsigned long vaddr; + unsigned int i, nr; + int err; + + if (WARN_ON_ONCE(!confine_ready)) + return NULL; + + size = PAGE_ALIGN(size); + nr = size >> PAGE_SHIFT; + + /* + * Reserve the virtual area inside the range (then the fallback range) + * so module code keeps its +/-2G reachability to the kernel image; only + * the physical backing is confined. + */ + area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, + VM_ALLOC | vm_flags, range->start, range->end, + NUMA_NO_NODE, GFP_KERNEL, + __builtin_return_address(0)); + if (!area && range->fallback_start) + area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, + VM_ALLOC | vm_flags, + range->fallback_start, + range->fallback_end, NUMA_NO_NODE, + GFP_KERNEL, __builtin_return_address(0)); + if (!area) + return NULL; + + pages = kvmalloc_array(nr, sizeof(*pages), GFP_KERNEL); + if (!pages) + goto err_area; + + ck = kzalloc(sizeof(*ck), GFP_KERNEL); + if (!ck) + goto err_pages; + + /* + * Draw the backing a page at a time. The pages are vmapped one by one, + * so they need not be physically contiguous; a single gen_pool_alloc() + * of the whole size could fail on a fragmented window with free pages + * to spare. The lock is held across the loop so that the reserve check + * below still holds when the last page is taken. + */ + mutex_lock(&confine_lock); + if (bounded && gen_pool_avail(confine_pool) < size + confine_reserve) { + mutex_unlock(&confine_lock); + pr_warn_ratelimited("%zu bytes refused: %zu KiB of the window is kept for module text\n", + size, confine_reserve >> 10); + goto err_ck; + } + for (i = 0; i < nr; i++) { + unsigned long page_va = gen_pool_alloc(confine_pool, PAGE_SIZE); + + if (!page_va) { + __confine_pool_free(pages, i); + mutex_unlock(&confine_lock); + pr_warn_ratelimited("window exhausted, cannot satisfy %zu bytes\n", + size); + goto err_ck; + } + pages[i] = virt_to_page((void *)page_va); + } + mutex_unlock(&confine_lock); + + vaddr = (unsigned long)area->addr; + err = vmap_pages_range(vaddr, vaddr + size, pgprot, pages, PAGE_SHIFT); + if (err) + goto err_map; /* may be partly mapped */ + + /* + * Publish the backing pages on the vm_struct. arm64's set_memory_rox() + * (used by the module loader to make text executable) walks area->pages + * to mirror the permission change onto the pages' linear-map alias when + * rodata_full is on; without this it would dereference a NULL array. + */ + area->pages = pages; + area->nr_pages = nr; + + ck->area = area; + ck->size = size; + ck->pages = pages; + + mutex_lock(&confine_lock); + err = xa_err(xa_store(&confine_chunks, vaddr >> PAGE_SHIFT, ck, + GFP_KERNEL)); + mutex_unlock(&confine_lock); + if (err) + goto err_map; + + if ((range->flags & EXECMEM_KASAN_SHADOW) && + kasan_alloc_module_shadow(area->addr, size, GFP_KERNEL) < 0) + goto err_xa; + + /* + * __vmalloc_node_range() does these two for vmalloc memory. KASAN doesn't + * tag executable memory, so the address stays as it is. + */ + area->addr = kasan_unpoison_vmalloc(area->addr, size, KASAN_VMALLOC_VM_ALLOC); + kmemleak_vmalloc(area, size, GFP_KERNEL); + + return area->addr; + +err_xa: + mutex_lock(&confine_lock); + xa_erase(&confine_chunks, vaddr >> PAGE_SHIFT); + mutex_unlock(&confine_lock); +err_map: + free_vm_area(area); /* unmaps the range and frees bookkeeping */ + /* + * free_vm_area() leaves the TLB flush to the lazy vmap purge: flush now, + * before the pages can be mapped again for someone else. + */ + flush_tlb_kernel_range(vaddr, vaddr + size); + confine_pool_free(pages, nr); + kfree(ck); + kvfree(pages); + return NULL; +err_ck: + kfree(ck); +err_pages: + kvfree(pages); +err_area: + free_vm_area(area); + return NULL; +} + +bool execmem_confine_free(void *ptr) +{ + unsigned long vaddr = (unsigned long)ptr; + struct confine_chunk *ck; + unsigned int nr; + + if (!confine_ready || !ptr) + return false; + + mutex_lock(&confine_lock); + ck = xa_load(&confine_chunks, vaddr >> PAGE_SHIFT); + if (ck) + xa_erase(&confine_chunks, vaddr >> PAGE_SHIFT); + mutex_unlock(&confine_lock); + if (!ck) + return false; + + nr = ck->size >> PAGE_SHIFT; + + kmemleak_free(ptr); + + /* + * We bypass vfree(), so VM_FLUSH_RESET_PERMS does not run for us: drop + * the mapping back to non-executable, writable before tearing it down, + * so the window never retains a stale executable alias. A failure here + * would leave an executable alias behind, so warn rather than ignore it. + */ + if (set_memory_nx(vaddr, nr) || set_memory_rw(vaddr, nr)) + pr_warn_ratelimited("could not reset perms on %#lx before freeing\n", + vaddr); + + free_vm_area(ck->area); + + /* + * free_vm_area() leaves the TLB flush to the lazy vmap purge. Flush + * before the pages go back to the pool, or a stale writable entry for + * this address could reach the next module's text. + */ + flush_tlb_kernel_range(vaddr, vaddr + ck->size); + + confine_pool_free(ck->pages, nr); + + kvfree(ck->pages); + kfree(ck); + return true; +} + +/* Weak default: no architecture page-table sweep available. */ +unsigned long __weak execmem_confine_arch_verify(phys_addr_t base, + phys_addr_t size, + unsigned long *nr_exec) +{ + return 0; +} + +static unsigned long __init execmem_confine_verify(unsigned long *nr_exec_out) +{ + unsigned long nr_exec = 0, violations = 0, freed; + struct confine_chunk *ck; + unsigned long idx; + + freed = confine_count_freed_pages(); + if (freed) { + pr_err("%lu window page(s) reached the page allocator\n", freed); + violations += freed; + } + + /* Every confined allocation is backed by pool pages, inside the window. */ + mutex_lock(&confine_lock); + xa_for_each(&confine_chunks, idx, ck) { + unsigned int i, nr = ck->size >> PAGE_SHIFT; + + for (i = 0; i < nr; i++) { + phys_addr_t pa = page_to_phys(ck->pages[i]); + + if (!phys_in_window(pa)) { + pr_err("confined chunk at %#lx backs out-of-window phys %pa\n", + (unsigned long)ck->area->addr, &pa); + violations++; + } + } + } + mutex_unlock(&confine_lock); + + /* Architecture page-table sweep over all kernel executable mappings. */ + violations += execmem_confine_arch_verify(confine_base, confine_size, + &nr_exec); + + if (nr_exec_out) + *nr_exec_out = nr_exec; + return violations; +} + +static int __init execmem_confine_late_init(void) +{ + unsigned long nr_exec = 0, violations; + + if (!confine_ready) + return 0; + + violations = execmem_confine_verify(&nr_exec); + pr_info("boot verify: %lu executable mapping(s), %lu violation(s): %s\n", + nr_exec, violations, violations ? "FAIL" : "PASS"); + + if (violations && confine_enforce) + panic("execmem-confine: %lu executable mapping(s) outside the window\n", + violations); + + return 0; +} +late_initcall(execmem_confine_late_init); diff --git a/mm/memblock.c b/mm/memblock.c index e6fdf95a0534d..a2b11c143d396 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -908,6 +909,9 @@ static unsigned long __free_reserved_area(phys_addr_t start, phys_addr_t end, struct page *page = pfn_to_page(pfn); void *direct_map_addr; + if (execmem_confine_pfn_in_window(pfn)) + continue; + /* * 'direct_map_addr' might be different from the kernel virtual * address because some architectures use aliases. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/idle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/idle.py index 4f13a4f2105cc..d699ab4c51f83 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/idle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/idle.py @@ -79,10 +79,8 @@ def function(text, name): static ktime_t ktime_us_delta(ktime_t a,ktime_t b){return (a-b)/1000;} static u64 div_u64(u64 a,u64 b){return a/b;} static u64 mul_u32_u32(unsigned a,unsigned b){return (u64)a*b;} -static bool t8140; -static bool of_machine_is_compatible(const char *s){return t8140&&!strcmp(s,"apple,t8140");} -static bool t6041; -static bool of_device_is_compatible(const struct device_node *np,const char *s){(void)np;return t6041&&!strcmp(s,"apple,t6041-cpuidle");} +static const char *machine; +static bool of_machine_compatible_match(const char *const *c){for(;machine&&*c;c++)if(!strcmp(*c,machine))return true;return false;} static int cpu_pm_enter(void){pm++;return pm_fail;} static void cpu_pm_exit(void){pm--;} static void apple_cpu_deep_wfi(void){deep++;} @@ -100,8 +98,8 @@ def function(text, name): if(faulty_irq)irq[current]=0; } static struct cpuidle_driver original; -static void reset(bool is_t8140){ - apple_idle_driver=original;t8140=is_t8140;t6041=false; current=0; +static void reset(const char *m){ + apple_idle_driver=original;machine=m; current=0; memset(cpuidle_dev,0,sizeof(cpuidle_dev)); memset(registered,0,sizeof(registered)); for(int i=0;i<6;i++){irq[i]=1;rcu[i]=stopped[i]=0;} wfi=deep=pm=pm_fail=warnings=unregistered=0;reg_fail=-1;drv_fail=0; @@ -110,7 +108,7 @@ def function(text, name): } int main(void){ original=apple_idle_driver; - reset(true);check(!apple_cpuidle_probe(&pdev)); + reset("apple,t8140");check(!apple_cpuidle_probe(&pdev)); check(apple_idle_driver.cpumask==cpu_possible_mask);check(!apple_idle_driver.bctimer); for(int i=0;i<6;i++)check(registered[i]&&cpuidle_dev[i].cpu==i); int expected=EXPECT_FIXED?1:0; @@ -126,34 +124,35 @@ def function(text, name): check(apple_idle_driver.states[1].exit_latency_ns==1000); check(apple_idle_driver.states[1].target_residency_ns==1000); check(apple_idle_driver.states[1].power_usage==apple_idle_driver.states[0].power_usage); - reset(true);apple_cpuidle_probe(&pdev);fanout=1; + reset("apple,t8140");apple_cpuidle_probe(&pdev);fanout=1; check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==1); check(wfi==6&&deep==0&&pm==0);check(tk_suspend==1&&tk_resume==1); check(sched_suspend==1&&sched_resume==1&&local_suspend==5&&local_resume==5); check(tick_freeze_depth==0&&!frozen); for(int i=0;i<6;i++)check(cpuidle_dev[i].states_usage[1].s2idle_usage==1&&rcu[i]==0&&stopped[i]==0); - reset(true);apple_cpuidle_probe(&pdev);fanout=1;cpuidle_dev[5].states_usage[1].disable=1; + reset("apple,t8140");apple_cpuidle_probe(&pdev);fanout=1;cpuidle_dev[5].states_usage[1].disable=1; cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000); check(wfi==5&&tk_suspend==0&&tk_resume==0&&tick_freeze_depth==0); - reset(true);apple_cpuidle_probe(&pdev);cpuidle_dev[0].states_usage[1].disable=1; + reset("apple,t8140");apple_cpuidle_probe(&pdev);cpuidle_dev[0].states_usage[1].disable=1; check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==0&&wfi==0); cpuidle_dev[0].states_usage[1].disable=0;apple_idle_driver.states[1].enter_s2idle=NULL; check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==0&&wfi==0); - reset(true);apple_cpuidle_probe(&pdev); + reset("apple,t8140");apple_cpuidle_probe(&pdev); check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],999)==0); check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],1000)==1); - reset(true);apple_cpuidle_probe(&pdev);apple_idle_driver.states[1].exit_latency_ns=0; + reset("apple,t8140");apple_cpuidle_probe(&pdev);apple_idle_driver.states[1].exit_latency_ns=0; check(find_deepest_state(&apple_idle_driver,&cpuidle_dev[0],100000,0,true)==0); - reset(true);apple_cpuidle_probe(&pdev);apple_idle_driver.states[1].flags=CPUIDLE_FLAG_TIMER_STOP; + reset("apple,t8140");apple_cpuidle_probe(&pdev);apple_idle_driver.states[1].flags=CPUIDLE_FLAG_TIMER_STOP; check(find_deepest_state(&apple_idle_driver,&cpuidle_dev[0],100000,CPUIDLE_FLAG_TIMER_STOP,false)==0); - reset(true);apple_cpuidle_probe(&pdev);faulty_irq=1; + reset("apple,t8140");apple_cpuidle_probe(&pdev);faulty_irq=1; check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==1); check(warnings==1&&tick_freeze_depth==0&&rcu[0]==0&&stopped[0]==0); } - reset(true);apple_cpuidle_probe(&pdev); + reset("apple,t8140");apple_cpuidle_probe(&pdev); for(int i=0;i