diff --git a/mm/memory.c b/mm/memory.c index ef71f8b2e406e..fa8bdea4c0d3f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6838,15 +6838,24 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, return err; } +/* + * A uffd-wp marker in a registered VMA is unpopulated memory that remembers its + * protection: populate it like a none neighbour, then protect it (see below). + * Otherwise every first write under WP_UNPOPULATED takes a native page of its own. + */ static unsigned long anon_subpage_missing_mask(pte_t *first, unsigned int nr, - unsigned int target, unsigned int offset) + unsigned int target, unsigned int offset, + bool uffd_wp) { unsigned long mask = 0; unsigned int i; - for (i = 0; i < nr; i++) - if (i == target || pte_none(ptep_get(first + i))) + for (i = 0; i < nr; i++) { + pte_t pte = ptep_get(first + i); + + if (i == target || pte_none(pte) || (uffd_wp && pte_is_uffd_wp_marker(pte))) mask |= BIT(offset + i); + } return mask; } @@ -6860,6 +6869,7 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) unsigned long start, end, lower, mask, current_mask, window_mask; unsigned int first_offset, nr, target, i, index, mapped; bool pooled, fresh, force_private = false, retry_private; + bool uffd_wp = userfaultfd_wp(vma); vm_fault_t ret; pte_t *first; int err; @@ -6880,7 +6890,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) vmf->pte = pte_offset_map_mm(mm, vmf->pmd, vmf->address); if (!vmf->pte) return 0; - mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset); + mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset, + uffd_wp); pte_unmap(vmf->pte); vmf->pte = NULL; err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask, @@ -6930,7 +6941,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) } first = vmf->pte - target; - current_mask = anon_subpage_missing_mask(first, nr, target, first_offset); + current_mask = anon_subpage_missing_mask(first, nr, target, first_offset, + uffd_wp); /* Do not lose newly missing neighbors or publish sparse dense-path supply. * Return to the fault handler on a racing mask change instead of spinning * under repeated zaps. No mapping has been published yet. @@ -6957,7 +6969,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) entry = pte_sw_mkyoung(entry); if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); - if (i == target && vmf_orig_pte_uffd_wp(vmf)) + if (i == target ? vmf_orig_pte_uffd_wp(vmf) : + pte_is_uffd_wp_marker(ptep_get(first + i))) entry = pte_mkuffd(entry); set_pte_at(mm, address, first + i, entry); update_mmu_cache(vma, address, first + i); diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c index f34ab8328721d..370d25081ed65 100644 --- a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -440,6 +440,36 @@ static void markers(size_t ps) printf("ok - uffd ABI %zuK poison: SIGBUS, MADV_FREE recovery, neighbours\n", ps / 1024); } +static void marker_window(size_t ps, size_t native) +{ + unsigned char *map = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *a = + (void *)(((uintptr_t)map + native - 1) & ~(native - 1)); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_WP_ASYNC | + UFFD_FEATURE_WP_UNPOPULATED }; + struct uffdio_register reg = { .range = { (uintptr_t)a, native }, + .mode = UFFDIO_REGISTER_MODE_WP }; + uint64_t start; + + CHECK(map != MAP_FAILED && fd >= 0); + CHECK(!ioctl(fd, UFFDIO_API, &api) && !ioctl(fd, UFFDIO_REGISTER, ®)); + protect(fd, a, native, UFFDIO_WRITEPROTECT_MODE_WP); + /* The first touch of the window is a write, so markers are its only neighbours. */ + a[ps + 17] = 0x5a; + start = physical(a, ps); + CHECK(start && !(start & (native - 1))); + for (size_t i = 0; i < native; i += ps) { + CHECK(physical(a + i, ps) == start + i); + CHECK(((pagemap(a + i, ps) >> 57) & 1) == (i != ps)); + } + CHECK(a[ps + 17] == 0x5a && !close(fd) && !munmap(map, 2 * native)); + printf("ok - uffd ABI %zuK unpopulated WP first write: one native page, neighbours WP\n", + ps / 1024); +} struct pager_request { uintptr_t address; size_t size; @@ -783,6 +813,8 @@ static int worker(unsigned long expected, unsigned long native) missing(ps, native); file_minor(ps, native); markers(ps); + if (ps < native) + marker_window(ps, native); cross_pager(ps, native, 0); cross_pager(ps, native, 1); private_missing(ps);