Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 19 additions & 6 deletions mm/memory.c
Original file line number Diff line number Diff line change
Expand Up @@ -6838,15 +6838,24 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm,
return err;
}

/*
* A uffd-wp marker in a registered VMA is unpopulated memory that remembers its
* protection: populate it like a none neighbour, then protect it (see below).
* Otherwise every first write under WP_UNPOPULATED takes a native page of its own.
*/
static unsigned long anon_subpage_missing_mask(pte_t *first, unsigned int nr,
unsigned int target, unsigned int offset)
unsigned int target, unsigned int offset,
bool uffd_wp)
{
unsigned long mask = 0;
unsigned int i;

for (i = 0; i < nr; i++)
if (i == target || pte_none(ptep_get(first + i)))
for (i = 0; i < nr; i++) {
pte_t pte = ptep_get(first + i);

if (i == target || pte_none(pte) || (uffd_wp && pte_is_uffd_wp_marker(pte)))
mask |= BIT(offset + i);
}
return mask;
}

Expand All @@ -6860,6 +6869,7 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf)
unsigned long start, end, lower, mask, current_mask, window_mask;
unsigned int first_offset, nr, target, i, index, mapped;
bool pooled, fresh, force_private = false, retry_private;
bool uffd_wp = userfaultfd_wp(vma);
vm_fault_t ret;
pte_t *first;
int err;
Expand All @@ -6880,7 +6890,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf)
vmf->pte = pte_offset_map_mm(mm, vmf->pmd, vmf->address);
if (!vmf->pte)
return 0;
mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset);
mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset,
uffd_wp);
pte_unmap(vmf->pte);
vmf->pte = NULL;
err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask,
Expand Down Expand Up @@ -6930,7 +6941,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf)
}

first = vmf->pte - target;
current_mask = anon_subpage_missing_mask(first, nr, target, first_offset);
current_mask = anon_subpage_missing_mask(first, nr, target, first_offset,
uffd_wp);
/* Do not lose newly missing neighbors or publish sparse dense-path supply.
* Return to the fault handler on a racing mask change instead of spinning
* under repeated zaps. No mapping has been published yet.
Expand All @@ -6957,7 +6969,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf)
entry = pte_sw_mkyoung(entry);
if (vma->vm_flags & VM_WRITE)
entry = pte_mkwrite(pte_mkdirty(entry), vma);
if (i == target && vmf_orig_pte_uffd_wp(vmf))
if (i == target ? vmf_orig_pte_uffd_wp(vmf) :
pte_is_uffd_wp_marker(ptep_get(first + i)))
entry = pte_mkuffd(entry);
set_pte_at(mm, address, first + i, entry);
update_mmu_cache(vma, address, first + i);
Expand Down
32 changes: 32 additions & 0 deletions tools/testing/selftests/arm64/user4k/uffd-wp-init.c
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,36 @@ static void markers(size_t ps)
printf("ok - uffd ABI %zuK poison: SIGBUS, MADV_FREE recovery, neighbours\n",
ps / 1024);
}
static void marker_window(size_t ps, size_t native)
{
unsigned char *map = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
unsigned char *a =
(void *)(((uintptr_t)map + native - 1) & ~(native - 1));
int fd = syscall(SYS_userfaultfd,
O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY);
struct uffdio_api api = { .api = UFFD_API,
.features = UFFD_FEATURE_WP_ASYNC |
UFFD_FEATURE_WP_UNPOPULATED };
struct uffdio_register reg = { .range = { (uintptr_t)a, native },
.mode = UFFDIO_REGISTER_MODE_WP };
uint64_t start;

CHECK(map != MAP_FAILED && fd >= 0);
CHECK(!ioctl(fd, UFFDIO_API, &api) && !ioctl(fd, UFFDIO_REGISTER, &reg));
protect(fd, a, native, UFFDIO_WRITEPROTECT_MODE_WP);
/* The first touch of the window is a write, so markers are its only neighbours. */
a[ps + 17] = 0x5a;
start = physical(a, ps);
CHECK(start && !(start & (native - 1)));
for (size_t i = 0; i < native; i += ps) {
CHECK(physical(a + i, ps) == start + i);
CHECK(((pagemap(a + i, ps) >> 57) & 1) == (i != ps));
}
CHECK(a[ps + 17] == 0x5a && !close(fd) && !munmap(map, 2 * native));
printf("ok - uffd ABI %zuK unpopulated WP first write: one native page, neighbours WP\n",
ps / 1024);
}
struct pager_request {
uintptr_t address;
size_t size;
Expand Down Expand Up @@ -783,6 +813,8 @@ static int worker(unsigned long expected, unsigned long native)
missing(ps, native);
file_minor(ps, native);
markers(ps);
if (ps < native)
marker_window(ps, native);
cross_pager(ps, native, 0);
cross_pager(ps, native, 1);
private_missing(ps);
Expand Down
Loading