From 834e0e6e8f6bd68c803419410003762f41c76646 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 16 Nov 2026 11:38:20 -0300 Subject: [PATCH 1/2] mm: populate a subpage window's uffd-wp markers with the faulting page In an mm whose user page size is smaller than the native page, a write fault on unpopulated anonymous memory fills the whole native window from one fresh native page, each 4K page at its own offset. That keeps the window physically in VA order, and costs one native page for four user pages. A uffd-wp marker (UFFD_FEATURE_WP_UNPOPULATED) is unpopulated memory too, but anon_subpage_missing_mask() counted only none PTEs as missing. In a VMA write-protected before its first touch, every neighbour holds a marker, so each first write took a native page of its own for one 4K page: four times the memory, and no window was ever one native page in order. A device that maps native pages (a GPU with 16K pages taking user memory through GEM_USERPTR) can then use none of it. Count a uffd-wp marker in a uffd-wp VMA as missing, and give the neighbour it stood for the uffd-wp bit, as the faulting page already gets it from its own marker: the populated neighbours stay write protected and unwritten for PAGEMAP_SCAN. Markers in a VMA that is no longer registered are left as they were. Measured with a probe in a 4K process on an M4 Pro (16K kernel): 8 MiB of anonymous memory, write-protected through userfaultfd with WP_ASYNC | WP_UNPOPULATED before its first touch, then written once per 4K page. Before, 0 of 512 16K blocks were one native page in order; memory registered after its first touch, or never registered, gave 512 of 512. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- mm/memory.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index ef71f8b2e406e..fa8bdea4c0d3f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6838,15 +6838,24 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, return err; } +/* + * A uffd-wp marker in a registered VMA is unpopulated memory that remembers its + * protection: populate it like a none neighbour, then protect it (see below). + * Otherwise every first write under WP_UNPOPULATED takes a native page of its own. + */ static unsigned long anon_subpage_missing_mask(pte_t *first, unsigned int nr, - unsigned int target, unsigned int offset) + unsigned int target, unsigned int offset, + bool uffd_wp) { unsigned long mask = 0; unsigned int i; - for (i = 0; i < nr; i++) - if (i == target || pte_none(ptep_get(first + i))) + for (i = 0; i < nr; i++) { + pte_t pte = ptep_get(first + i); + + if (i == target || pte_none(pte) || (uffd_wp && pte_is_uffd_wp_marker(pte))) mask |= BIT(offset + i); + } return mask; } @@ -6860,6 +6869,7 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) unsigned long start, end, lower, mask, current_mask, window_mask; unsigned int first_offset, nr, target, i, index, mapped; bool pooled, fresh, force_private = false, retry_private; + bool uffd_wp = userfaultfd_wp(vma); vm_fault_t ret; pte_t *first; int err; @@ -6880,7 +6890,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) vmf->pte = pte_offset_map_mm(mm, vmf->pmd, vmf->address); if (!vmf->pte) return 0; - mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset); + mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset, + uffd_wp); pte_unmap(vmf->pte); vmf->pte = NULL; err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask, @@ -6930,7 +6941,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) } first = vmf->pte - target; - current_mask = anon_subpage_missing_mask(first, nr, target, first_offset); + current_mask = anon_subpage_missing_mask(first, nr, target, first_offset, + uffd_wp); /* Do not lose newly missing neighbors or publish sparse dense-path supply. * Return to the fault handler on a racing mask change instead of spinning * under repeated zaps. No mapping has been published yet. @@ -6957,7 +6969,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) entry = pte_sw_mkyoung(entry); if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); - if (i == target && vmf_orig_pte_uffd_wp(vmf)) + if (i == target ? vmf_orig_pte_uffd_wp(vmf) : + pte_is_uffd_wp_marker(ptep_get(first + i))) entry = pte_mkuffd(entry); set_pte_at(mm, address, first + i, entry); update_mmu_cache(vma, address, first + i); From c13996182ced6eafe09c5873f0d9d40fc082506b Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 19 Nov 2026 01:39:27 -0300 Subject: [PATCH 2/2] selftests/arm64: cover a first write into a uffd-wp-before-touch window Add a uffd-wp-init case that write-protects a native-aligned window with WP_ASYNC | WP_UNPOPULATED before any touch, then writes one byte to its second user page. pagemap must then show every user page of the window present in one native page in VA order, with the uffd-wp bit kept on the neighbours and cleared on the written page. markers() never reached this path: its window is not native-aligned, and its first touch is a read that maps the zero page, so its write is a copy-on-write fault. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- .../selftests/arm64/user4k/uffd-wp-init.c | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c index f34ab8328721d..370d25081ed65 100644 --- a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -440,6 +440,36 @@ static void markers(size_t ps) printf("ok - uffd ABI %zuK poison: SIGBUS, MADV_FREE recovery, neighbours\n", ps / 1024); } +static void marker_window(size_t ps, size_t native) +{ + unsigned char *map = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *a = + (void *)(((uintptr_t)map + native - 1) & ~(native - 1)); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_WP_ASYNC | + UFFD_FEATURE_WP_UNPOPULATED }; + struct uffdio_register reg = { .range = { (uintptr_t)a, native }, + .mode = UFFDIO_REGISTER_MODE_WP }; + uint64_t start; + + CHECK(map != MAP_FAILED && fd >= 0); + CHECK(!ioctl(fd, UFFDIO_API, &api) && !ioctl(fd, UFFDIO_REGISTER, ®)); + protect(fd, a, native, UFFDIO_WRITEPROTECT_MODE_WP); + /* The first touch of the window is a write, so markers are its only neighbours. */ + a[ps + 17] = 0x5a; + start = physical(a, ps); + CHECK(start && !(start & (native - 1))); + for (size_t i = 0; i < native; i += ps) { + CHECK(physical(a + i, ps) == start + i); + CHECK(((pagemap(a + i, ps) >> 57) & 1) == (i != ps)); + } + CHECK(a[ps + 17] == 0x5a && !close(fd) && !munmap(map, 2 * native)); + printf("ok - uffd ABI %zuK unpopulated WP first write: one native page, neighbours WP\n", + ps / 1024); +} struct pager_request { uintptr_t address; size_t size; @@ -783,6 +813,8 @@ static int worker(unsigned long expected, unsigned long native) missing(ps, native); file_minor(ps, native); markers(ps); + if (ps < native) + marker_window(ps, native); cross_pager(ps, native, 0); cross_pager(ps, native, 1); private_missing(ps);