diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 890d47c40af6b..b17f3d47a3e06 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += aurora.o -aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/sleep.o g17/pmpdash.o +aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o user_pages.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/sleep.o g17/pmpdash.o CFLAGS_g17/trace.o := -I$(src)/g17 aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 40487a77e97ea..da3dc36a2260b 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -459,9 +459,13 @@ pub(crate) struct UserPages { mm: ARef, /// Pages counted in `pinned_vm`: all of the range, including any not pinned yet. counted: i64, + /// The fragment pins of a process with a smaller page size, which hold `pages` (null when + /// `pages` are ordinary pins). + fragments: *mut core::ffi::c_void, } -// SAFETY: The pinned pages may be released from any thread; the mm reference is thread-safe. +// SAFETY: The pinned pages and fragment records may be released from any thread; the mm +// reference is thread-safe. unsafe impl Send for UserPages {} // SAFETY: Shared access only reads the array. unsafe impl Sync for UserPages {} @@ -510,12 +514,33 @@ impl UserPages { writable, mm, counted: added, + fragments: core::ptr::null_mut(), }; this.pages.reserve(count, GFP_KERNEL)?; let mut flags = kernel::bindings::FOLL_LONGTERM; if writable { flags |= kernel::bindings::FOLL_WRITE; } + // SAFETY: No preconditions; it reads the current mm. + if unsafe { asahi_user_mm_is_subpage() } { + // SAFETY: The array has room for `count` entries. On success the call stored `count` + // pages, which the records it returned in `fragments` hold. + let ret = unsafe { + asahi_pin_user_fragments( + addr as _, + count as _, + flags, + this.pages.as_mut_ptr().cast(), + &mut this.fragments, + ) + }; + if ret < 0 { + return Err(Error::from_errno(ret as i32)); + } + // SAFETY: See above. + unsafe { this.pages.inc_len(count) }; + return Ok(this); + } while this.pages.len() < count { let done = this.pages.len(); let chunk = (count - done).min(PIN_CHUNK); @@ -556,7 +581,11 @@ impl UserPages { impl Drop for UserPages { fn drop(&mut self) { - if !self.pages.is_empty() { + if !self.fragments.is_null() { + // SAFETY: The records hold every page of the array and are released only here. + // Pages the device may have written are marked dirty. + unsafe { asahi_unpin_user_fragments(self.fragments, self.writable) }; + } else if !self.pages.is_empty() { // SAFETY: The array holds pages pinned by `pin_user_pages_fast()` and not yet // unpinned. Pages the device may have written are marked dirty. unsafe { @@ -985,6 +1014,19 @@ pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result { Ok(()) } +// user_pages.c: long-term pins in a process whose page size is smaller than the native one. +extern "C" { + fn asahi_user_mm_is_subpage() -> bool; + fn asahi_pin_user_fragments( + start: core::ffi::c_ulong, + nr_pages: core::ffi::c_ulong, + gup_flags: core::ffi::c_uint, + pages: *mut *mut core::ffi::c_void, + out: *mut *mut core::ffi::c_void, + ) -> core::ffi::c_long; + fn asahi_unpin_user_fragments(frags: *mut core::ffi::c_void, dirty: bool); +} + #[cfg(CONFIG_DRM_ASAHI_G15)] extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; diff --git a/drivers/gpu/drm/asahi/user_pages.c b/drivers/gpu/drm/asahi/user_pages.c new file mode 100644 index 0000000000000..7ded04794f626 --- /dev/null +++ b/drivers/gpu/drm/asahi/user_pages.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Long-term pins of user memory as whole native pages, for GEM_USERPTR. + * + * The GPU maps memory in native pages. A process whose user page size is + * smaller (CONFIG_MM_SUBPAGE) maps each of its pages to a fragment of a + * native page, and only the fragment API can pin those. A native page can be + * given to the GPU only where the process maps all of it, its fragments in + * order at consecutive addresses: then the GPU sees what the CPU sees, and no + * memory the process does not map. + */ + +#include +#include +#include +#include + +struct asahi_user_fragments; + +bool asahi_user_mm_is_subpage(void); +long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages, + struct asahi_user_fragments **out); +void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty); + +#ifdef CONFIG_MM_SUBPAGE +/* Pinned fragments of a range; released only through asahi_unpin_user_fragments(). */ +struct asahi_user_fragments { + unsigned long count; + struct user_page_fragment records[]; +}; + +bool asahi_user_mm_is_subpage(void) +{ + return current->mm && mm_page_size(current->mm) < PAGE_SIZE; +} + +void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty) +{ + if (!frags) + return; + release_user_fragments(frags->records, frags->count, dirty); + kvfree(frags); +} + +/* The native page that @rec and its @n - 1 successors map in order, or NULL. */ +static struct page *whole_native_page(const struct user_page_fragment *rec, + unsigned long n, unsigned long granule) +{ + unsigned long i; + + if (!IS_ALIGNED(rec->offset, PAGE_SIZE)) + return NULL; + for (i = 0; i < n; i++) + if (rec[i].folio != rec->folio || rec[i].length != granule || + rec[i].offset != rec->offset + i * granule) + return NULL; + return folio_page(rec->folio, rec->offset >> PAGE_SHIFT); +} + +/* + * Pins @nr_pages native pages' worth of the current process's memory at + * @start (native-page aligned) and stores the native pages in @pages. All or + * nothing: -EFAULT if a page cannot be pinned or a native page is not mapped + * whole and in order; -ENOMEM if the records cannot be allocated. + */ +long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages, + struct asahi_user_fragments **out) +{ + struct mm_struct *mm = current->mm; + unsigned long granule = mm_page_size(mm); + unsigned long per_page = PAGE_SIZE / granule; + unsigned long length = nr_pages << PAGE_SHIFT, done = 0, i; + struct asahi_user_fragments *frags; + long ret = 0; + + if (nr_pages > (ULONG_MAX >> PAGE_SHIFT) / per_page) + return -ENOMEM; + frags = kvzalloc(struct_size(frags, records, nr_pages * per_page), + GFP_KERNEL_ACCOUNT); + if (!frags) + return -ENOMEM; + while (done < length) { + long n = pin_user_fragments_remote(mm, start + done, length - done, + gup_flags, frags->records + frags->count, + nr_pages * per_page - frags->count); + + if (n <= 0) { + ret = (n == 0 || n == -EOPNOTSUPP) ? -EFAULT : n; + goto fail; + } + for (i = frags->count; i < frags->count + n; i++) + done += frags->records[i].length; + frags->count += n; + } + /* Fragments never cross a native page, so a whole page is per_page records. */ + if (frags->count != nr_pages * per_page) { + ret = -EFAULT; + goto fail; + } + for (i = 0; i < nr_pages; i++) { + pages[i] = whole_native_page(&frags->records[i * per_page], per_page, granule); + if (!pages[i]) { + ret = -EFAULT; + goto fail; + } + } + *out = frags; + return 0; + +fail: + asahi_unpin_user_fragments(frags, false); + return ret; +} +#else +bool asahi_user_mm_is_subpage(void) +{ + return false; +} + +void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty) +{ +} + +long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages, + struct asahi_user_fragments **out) +{ + return -EOPNOTSUPP; +} +#endif diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index fc3b32a63b470..9ba120a094487 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -780,6 +780,10 @@ struct drm_asahi_gem_create { * while the object lives. Device and PFN mappings, including CPU mappings of * other buffer objects, are refused (EFAULT). * + * In a process whose page size is below 16 KiB, each 16 KiB of the range must + * be one native page that the process maps whole and in order (EFAULT + * otherwise). + * * The object cannot be mapped with DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET, exported, * given advice with DRM_IOCTL_ASAHI_GEM_MADVISE, bound with * DRM_IOCTL_ASAHI_GEM_BIND_OBJECT or registered as a VM status mirror (all diff --git a/mm/memory.c b/mm/memory.c index ef71f8b2e406e..fa8bdea4c0d3f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6838,15 +6838,24 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, return err; } +/* + * A uffd-wp marker in a registered VMA is unpopulated memory that remembers its + * protection: populate it like a none neighbour, then protect it (see below). + * Otherwise every first write under WP_UNPOPULATED takes a native page of its own. + */ static unsigned long anon_subpage_missing_mask(pte_t *first, unsigned int nr, - unsigned int target, unsigned int offset) + unsigned int target, unsigned int offset, + bool uffd_wp) { unsigned long mask = 0; unsigned int i; - for (i = 0; i < nr; i++) - if (i == target || pte_none(ptep_get(first + i))) + for (i = 0; i < nr; i++) { + pte_t pte = ptep_get(first + i); + + if (i == target || pte_none(pte) || (uffd_wp && pte_is_uffd_wp_marker(pte))) mask |= BIT(offset + i); + } return mask; } @@ -6860,6 +6869,7 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) unsigned long start, end, lower, mask, current_mask, window_mask; unsigned int first_offset, nr, target, i, index, mapped; bool pooled, fresh, force_private = false, retry_private; + bool uffd_wp = userfaultfd_wp(vma); vm_fault_t ret; pte_t *first; int err; @@ -6880,7 +6890,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) vmf->pte = pte_offset_map_mm(mm, vmf->pmd, vmf->address); if (!vmf->pte) return 0; - mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset); + mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset, + uffd_wp); pte_unmap(vmf->pte); vmf->pte = NULL; err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask, @@ -6930,7 +6941,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) } first = vmf->pte - target; - current_mask = anon_subpage_missing_mask(first, nr, target, first_offset); + current_mask = anon_subpage_missing_mask(first, nr, target, first_offset, + uffd_wp); /* Do not lose newly missing neighbors or publish sparse dense-path supply. * Return to the fault handler on a racing mask change instead of spinning * under repeated zaps. No mapping has been published yet. @@ -6957,7 +6969,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) entry = pte_sw_mkyoung(entry); if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); - if (i == target && vmf_orig_pte_uffd_wp(vmf)) + if (i == target ? vmf_orig_pte_uffd_wp(vmf) : + pte_is_uffd_wp_marker(ptep_get(first + i))) entry = pte_mkuffd(entry); set_pte_at(mm, address, first + i, entry); update_mmu_cache(vma, address, first + i); diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c index f34ab8328721d..7b49575599d90 100644 --- a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -440,6 +440,47 @@ static void markers(size_t ps) printf("ok - uffd ABI %zuK poison: SIGBUS, MADV_FREE recovery, neighbours\n", ps / 1024); } +/* + * A write-first fault under WP_UNPOPULATED: in a smaller-granule mm the + * native window is filled at once, so the marker neighbours become present + * and keep their write protection; with the native granule they stay markers. + */ +static void marker_window(size_t ps, size_t native) +{ + unsigned char *map = mmap(NULL, 4 * ps + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED); + unsigned char *p = (unsigned char *)(((uintptr_t)map + native - 1) & + ~(uintptr_t)(native - 1)); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_WP_UNPOPULATED | + UFFD_FEATURE_WP_ASYNC }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 4 * ps }, + .mode = UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + protect(fd, p, 4 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + p[ps + 17] = 0x5a; + for (unsigned int i = 0; i < 4; i++) { + uint64_t entry = pagemap(p + i * ps, ps); + + if (i == 1) { + CHECK((entry & (1ULL << 63)) && !(entry & (1ULL << 57))); + continue; + } + CHECK(entry & (1ULL << 57)); + CHECK(!!(entry & (1ULL << 63)) == (ps < native)); + CHECK(p[i * ps + 17] == 0); + CHECK(pagemap(p + i * ps, ps) & (1ULL << 57)); + } + CHECK(!close(fd) && !munmap(map, 4 * ps + native)); + printf("ok - uffd ABI %zuK unpopulated WP write-first window keeps neighbour protection\n", + ps / 1024); +} struct pager_request { uintptr_t address; size_t size; @@ -783,6 +824,7 @@ static int worker(unsigned long expected, unsigned long native) missing(ps, native); file_minor(ps, native); markers(ps); + marker_window(ps, native); cross_pager(ps, native, 0); cross_pager(ps, native, 1); private_missing(ps);