From 695bc5ed7f3e109b216706840159dc3da2e7b697 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 16 Nov 2026 11:38:20 -0300 Subject: [PATCH 1/2] mm: populate a subpage window's uffd-wp markers with the faulting page In an mm whose user page size is smaller than the native page, a write fault on unpopulated anonymous memory fills the whole native window from one fresh native page, each 4K page at its own offset. That keeps the window physically in VA order, and costs one native page for four user pages. A uffd-wp marker (UFFD_FEATURE_WP_UNPOPULATED) is unpopulated memory too, but anon_subpage_missing_mask() counted only none PTEs as missing. In a VMA write-protected before its first touch, every neighbour holds a marker, so each first write took a native page of its own for one 4K page: four times the memory, and no window was ever one native page in order. A device that maps native pages (a GPU with 16K pages taking user memory through GEM_USERPTR) can then use none of it. Count a uffd-wp marker in a uffd-wp VMA as missing, and give the neighbour it stood for the uffd-wp bit, as the faulting page already gets it from its own marker: the populated neighbours stay write protected and unwritten for PAGEMAP_SCAN. Markers in a VMA that is no longer registered are left as they were. Only do_anonymous_subpage() changes, the first-write path of an mm with a smaller page size (CONFIG_MM_SUBPAGE). Processes with the native page size and every other uffd-wp path are untouched. The next patch needs this: it lets GEM_USERPTR pin user memory in a 4K process only where a native page is mapped whole and in order. A write watch built on userfaultfd (WP_ASYNC on fresh memory, then PAGEMAP_SCAN, as an emulator running under FEX uses it) protects memory this way before touching it, so without this fix none of that memory can be imported. Measured with a probe in a 4K process on an M4 Pro (16K kernel): 8 MiB of anonymous memory, write-protected through userfaultfd with WP_ASYNC | WP_UNPOPULATED before its first touch, then written once per 4K page. Before, 0 of 512 16K blocks were one native page in order; memory registered after its first touch, or never registered, gave 512 of 512. With this change all three give 512 of 512. A second probe writes 3 of 64 pages write-protected the same way: PAGEMAP_SCAN with PAGE_IS_WRITTEN reports exactly those 3, in a native 16K process and in a 4K one. Tested on an M4 Pro (T6040) only. The arm64 user4k uffd-wp-init selftest gains the write-first case: a native-aligned window of four user pages, write-protected with WP_ASYNC | WP_UNPOPULATED, written in its second page first. With a smaller user page size the other three must be present and still write-protected; with the native size they stay markers. It passes in the native and 4K workers in a user4k=on QEMU guest, and fails the 4K worker without this change. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- mm/memory.c | 25 ++++++++--- .../selftests/arm64/user4k/uffd-wp-init.c | 42 +++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index ef71f8b2e406e..fa8bdea4c0d3f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6838,15 +6838,24 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, return err; } +/* + * A uffd-wp marker in a registered VMA is unpopulated memory that remembers its + * protection: populate it like a none neighbour, then protect it (see below). + * Otherwise every first write under WP_UNPOPULATED takes a native page of its own. + */ static unsigned long anon_subpage_missing_mask(pte_t *first, unsigned int nr, - unsigned int target, unsigned int offset) + unsigned int target, unsigned int offset, + bool uffd_wp) { unsigned long mask = 0; unsigned int i; - for (i = 0; i < nr; i++) - if (i == target || pte_none(ptep_get(first + i))) + for (i = 0; i < nr; i++) { + pte_t pte = ptep_get(first + i); + + if (i == target || pte_none(pte) || (uffd_wp && pte_is_uffd_wp_marker(pte))) mask |= BIT(offset + i); + } return mask; } @@ -6860,6 +6869,7 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) unsigned long start, end, lower, mask, current_mask, window_mask; unsigned int first_offset, nr, target, i, index, mapped; bool pooled, fresh, force_private = false, retry_private; + bool uffd_wp = userfaultfd_wp(vma); vm_fault_t ret; pte_t *first; int err; @@ -6880,7 +6890,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) vmf->pte = pte_offset_map_mm(mm, vmf->pmd, vmf->address); if (!vmf->pte) return 0; - mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset); + mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset, + uffd_wp); pte_unmap(vmf->pte); vmf->pte = NULL; err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask, @@ -6930,7 +6941,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) } first = vmf->pte - target; - current_mask = anon_subpage_missing_mask(first, nr, target, first_offset); + current_mask = anon_subpage_missing_mask(first, nr, target, first_offset, + uffd_wp); /* Do not lose newly missing neighbors or publish sparse dense-path supply. * Return to the fault handler on a racing mask change instead of spinning * under repeated zaps. No mapping has been published yet. @@ -6957,7 +6969,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) entry = pte_sw_mkyoung(entry); if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); - if (i == target && vmf_orig_pte_uffd_wp(vmf)) + if (i == target ? vmf_orig_pte_uffd_wp(vmf) : + pte_is_uffd_wp_marker(ptep_get(first + i))) entry = pte_mkuffd(entry); set_pte_at(mm, address, first + i, entry); update_mmu_cache(vma, address, first + i); diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c index f34ab8328721d..7b49575599d90 100644 --- a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -440,6 +440,47 @@ static void markers(size_t ps) printf("ok - uffd ABI %zuK poison: SIGBUS, MADV_FREE recovery, neighbours\n", ps / 1024); } +/* + * A write-first fault under WP_UNPOPULATED: in a smaller-granule mm the + * native window is filled at once, so the marker neighbours become present + * and keep their write protection; with the native granule they stay markers. + */ +static void marker_window(size_t ps, size_t native) +{ + unsigned char *map = mmap(NULL, 4 * ps + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED); + unsigned char *p = (unsigned char *)(((uintptr_t)map + native - 1) & + ~(uintptr_t)(native - 1)); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_WP_UNPOPULATED | + UFFD_FEATURE_WP_ASYNC }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 4 * ps }, + .mode = UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + protect(fd, p, 4 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + p[ps + 17] = 0x5a; + for (unsigned int i = 0; i < 4; i++) { + uint64_t entry = pagemap(p + i * ps, ps); + + if (i == 1) { + CHECK((entry & (1ULL << 63)) && !(entry & (1ULL << 57))); + continue; + } + CHECK(entry & (1ULL << 57)); + CHECK(!!(entry & (1ULL << 63)) == (ps < native)); + CHECK(p[i * ps + 17] == 0); + CHECK(pagemap(p + i * ps, ps) & (1ULL << 57)); + } + CHECK(!close(fd) && !munmap(map, 4 * ps + native)); + printf("ok - uffd ABI %zuK unpopulated WP write-first window keeps neighbour protection\n", + ps / 1024); +} struct pager_request { uintptr_t address; size_t size; @@ -783,6 +824,7 @@ static int worker(unsigned long expected, unsigned long native) missing(ps, native); file_minor(ps, native); markers(ps); + marker_window(ps, native); cross_pager(ps, native, 0); cross_pager(ps, native, 1); private_missing(ps); From 4c15fea9c2aae5c1dd214f0e9b788b084bd33669 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 16 Nov 2026 11:38:20 -0300 Subject: [PATCH 2/2] drm/asahi: GEM_USERPTR in processes with a smaller page size GEM_USERPTR pins its range with pin_user_pages_fast(). In a process whose user page size is smaller than the native one (CONFIG_MM_SUBPAGE, user4k: 4K processes on a 16K kernel, such as everything running under FEX), GUP cannot return pages, and every GEM_USERPTR fails with EOPNOTSUPP, so VK_EXT_external_memory_host imports fail there. Such a process maps each of its pages to a fragment of a native page, and pin_user_fragments_remote() pins those. The GPU maps native pages, so a native page can back the object only where the process maps all of it, its fragments in order at consecutive addresses: then the GPU sees exactly what the CPU sees, and nothing the process does not map. user_pages.c pins the range as fragments and checks that layout, all or nothing (EFAULT where a native page is not mapped whole and in order). The fragment records hold the pins and are released with the object; pinned_vm counts native pages, as for other processes. Processes with the native page size keep pin_user_pages_fast(). The GEM_USERPTR uAPI doc now states the layout rule for such processes. Tested on an M4 Pro (T6040) only, with the previous patch, through VK_EXT_external_memory_host in Mesa: a probe imports 4 MiB of anonymous memory, has the GPU fill one part and copy another, and checks both and an untouched part from the CPU through its own mapping. Native 16K and x86 4K under FEX, plain memory and memory write-protected with userfaultfd before its first touch: 0 wrong words of 262144 in each part. Memory with a 4K quarter of every native page re-faulted elsewhere is refused in a 4K process, and VmPin stays at 0 kB. dEQP-VK.memory.external_memory_host.*: 5 Pass, 8 NotSupported (images), 0 Fail. No GPU recoveries during the runs. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/gem.rs | 46 +++++++++- drivers/gpu/drm/asahi/user_pages.c | 132 +++++++++++++++++++++++++++++ include/uapi/drm/asahi_drm.h | 4 + 4 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/asahi/user_pages.c diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 890d47c40af6b..b17f3d47a3e06 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += aurora.o -aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/sleep.o g17/pmpdash.o +aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o user_pages.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/sleep.o g17/pmpdash.o CFLAGS_g17/trace.o := -I$(src)/g17 aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 40487a77e97ea..da3dc36a2260b 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -459,9 +459,13 @@ pub(crate) struct UserPages { mm: ARef, /// Pages counted in `pinned_vm`: all of the range, including any not pinned yet. counted: i64, + /// The fragment pins of a process with a smaller page size, which hold `pages` (null when + /// `pages` are ordinary pins). + fragments: *mut core::ffi::c_void, } -// SAFETY: The pinned pages may be released from any thread; the mm reference is thread-safe. +// SAFETY: The pinned pages and fragment records may be released from any thread; the mm +// reference is thread-safe. unsafe impl Send for UserPages {} // SAFETY: Shared access only reads the array. unsafe impl Sync for UserPages {} @@ -510,12 +514,33 @@ impl UserPages { writable, mm, counted: added, + fragments: core::ptr::null_mut(), }; this.pages.reserve(count, GFP_KERNEL)?; let mut flags = kernel::bindings::FOLL_LONGTERM; if writable { flags |= kernel::bindings::FOLL_WRITE; } + // SAFETY: No preconditions; it reads the current mm. + if unsafe { asahi_user_mm_is_subpage() } { + // SAFETY: The array has room for `count` entries. On success the call stored `count` + // pages, which the records it returned in `fragments` hold. + let ret = unsafe { + asahi_pin_user_fragments( + addr as _, + count as _, + flags, + this.pages.as_mut_ptr().cast(), + &mut this.fragments, + ) + }; + if ret < 0 { + return Err(Error::from_errno(ret as i32)); + } + // SAFETY: See above. + unsafe { this.pages.inc_len(count) }; + return Ok(this); + } while this.pages.len() < count { let done = this.pages.len(); let chunk = (count - done).min(PIN_CHUNK); @@ -556,7 +581,11 @@ impl UserPages { impl Drop for UserPages { fn drop(&mut self) { - if !self.pages.is_empty() { + if !self.fragments.is_null() { + // SAFETY: The records hold every page of the array and are released only here. + // Pages the device may have written are marked dirty. + unsafe { asahi_unpin_user_fragments(self.fragments, self.writable) }; + } else if !self.pages.is_empty() { // SAFETY: The array holds pages pinned by `pin_user_pages_fast()` and not yet // unpinned. Pages the device may have written are marked dirty. unsafe { @@ -985,6 +1014,19 @@ pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result { Ok(()) } +// user_pages.c: long-term pins in a process whose page size is smaller than the native one. +extern "C" { + fn asahi_user_mm_is_subpage() -> bool; + fn asahi_pin_user_fragments( + start: core::ffi::c_ulong, + nr_pages: core::ffi::c_ulong, + gup_flags: core::ffi::c_uint, + pages: *mut *mut core::ffi::c_void, + out: *mut *mut core::ffi::c_void, + ) -> core::ffi::c_long; + fn asahi_unpin_user_fragments(frags: *mut core::ffi::c_void, dirty: bool); +} + #[cfg(CONFIG_DRM_ASAHI_G15)] extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; diff --git a/drivers/gpu/drm/asahi/user_pages.c b/drivers/gpu/drm/asahi/user_pages.c new file mode 100644 index 0000000000000..7ded04794f626 --- /dev/null +++ b/drivers/gpu/drm/asahi/user_pages.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Long-term pins of user memory as whole native pages, for GEM_USERPTR. + * + * The GPU maps memory in native pages. A process whose user page size is + * smaller (CONFIG_MM_SUBPAGE) maps each of its pages to a fragment of a + * native page, and only the fragment API can pin those. A native page can be + * given to the GPU only where the process maps all of it, its fragments in + * order at consecutive addresses: then the GPU sees what the CPU sees, and no + * memory the process does not map. + */ + +#include +#include +#include +#include + +struct asahi_user_fragments; + +bool asahi_user_mm_is_subpage(void); +long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages, + struct asahi_user_fragments **out); +void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty); + +#ifdef CONFIG_MM_SUBPAGE +/* Pinned fragments of a range; released only through asahi_unpin_user_fragments(). */ +struct asahi_user_fragments { + unsigned long count; + struct user_page_fragment records[]; +}; + +bool asahi_user_mm_is_subpage(void) +{ + return current->mm && mm_page_size(current->mm) < PAGE_SIZE; +} + +void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty) +{ + if (!frags) + return; + release_user_fragments(frags->records, frags->count, dirty); + kvfree(frags); +} + +/* The native page that @rec and its @n - 1 successors map in order, or NULL. */ +static struct page *whole_native_page(const struct user_page_fragment *rec, + unsigned long n, unsigned long granule) +{ + unsigned long i; + + if (!IS_ALIGNED(rec->offset, PAGE_SIZE)) + return NULL; + for (i = 0; i < n; i++) + if (rec[i].folio != rec->folio || rec[i].length != granule || + rec[i].offset != rec->offset + i * granule) + return NULL; + return folio_page(rec->folio, rec->offset >> PAGE_SHIFT); +} + +/* + * Pins @nr_pages native pages' worth of the current process's memory at + * @start (native-page aligned) and stores the native pages in @pages. All or + * nothing: -EFAULT if a page cannot be pinned or a native page is not mapped + * whole and in order; -ENOMEM if the records cannot be allocated. + */ +long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages, + struct asahi_user_fragments **out) +{ + struct mm_struct *mm = current->mm; + unsigned long granule = mm_page_size(mm); + unsigned long per_page = PAGE_SIZE / granule; + unsigned long length = nr_pages << PAGE_SHIFT, done = 0, i; + struct asahi_user_fragments *frags; + long ret = 0; + + if (nr_pages > (ULONG_MAX >> PAGE_SHIFT) / per_page) + return -ENOMEM; + frags = kvzalloc(struct_size(frags, records, nr_pages * per_page), + GFP_KERNEL_ACCOUNT); + if (!frags) + return -ENOMEM; + while (done < length) { + long n = pin_user_fragments_remote(mm, start + done, length - done, + gup_flags, frags->records + frags->count, + nr_pages * per_page - frags->count); + + if (n <= 0) { + ret = (n == 0 || n == -EOPNOTSUPP) ? -EFAULT : n; + goto fail; + } + for (i = frags->count; i < frags->count + n; i++) + done += frags->records[i].length; + frags->count += n; + } + /* Fragments never cross a native page, so a whole page is per_page records. */ + if (frags->count != nr_pages * per_page) { + ret = -EFAULT; + goto fail; + } + for (i = 0; i < nr_pages; i++) { + pages[i] = whole_native_page(&frags->records[i * per_page], per_page, granule); + if (!pages[i]) { + ret = -EFAULT; + goto fail; + } + } + *out = frags; + return 0; + +fail: + asahi_unpin_user_fragments(frags, false); + return ret; +} +#else +bool asahi_user_mm_is_subpage(void) +{ + return false; +} + +void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty) +{ +} + +long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages, + struct asahi_user_fragments **out) +{ + return -EOPNOTSUPP; +} +#endif diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index fc3b32a63b470..9ba120a094487 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -780,6 +780,10 @@ struct drm_asahi_gem_create { * while the object lives. Device and PFN mappings, including CPU mappings of * other buffer objects, are refused (EFAULT). * + * In a process whose page size is below 16 KiB, each 16 KiB of the range must + * be one native page that the process maps whole and in order (EFAULT + * otherwise). + * * The object cannot be mapped with DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET, exported, * given advice with DRM_IOCTL_ASAHI_GEM_MADVISE, bound with * DRM_IOCTL_ASAHI_GEM_BIND_OBJECT or registered as a VM status mirror (all