Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion drivers/gpu/drm/asahi/Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: GPL-2.0

obj-$(CONFIG_DRM_ASAHI) += aurora.o
aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/sleep.o g17/pmpdash.o
aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o user_pages.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/sleep.o g17/pmpdash.o
CFLAGS_g17/trace.o := -I$(src)/g17

aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o
Expand Down
46 changes: 44 additions & 2 deletions drivers/gpu/drm/asahi/gem.rs
Original file line number Diff line number Diff line change
Expand Up @@ -459,9 +459,13 @@ pub(crate) struct UserPages {
mm: ARef<Mm>,
/// Pages counted in `pinned_vm`: all of the range, including any not pinned yet.
counted: i64,
/// The fragment pins of a process with a smaller page size, which hold `pages` (null when
/// `pages` are ordinary pins).
fragments: *mut core::ffi::c_void,
}

// SAFETY: The pinned pages may be released from any thread; the mm reference is thread-safe.
// SAFETY: The pinned pages and fragment records may be released from any thread; the mm
// reference is thread-safe.
unsafe impl Send for UserPages {}
// SAFETY: Shared access only reads the array.
unsafe impl Sync for UserPages {}
Expand Down Expand Up @@ -510,12 +514,33 @@ impl UserPages {
writable,
mm,
counted: added,
fragments: core::ptr::null_mut(),
};
this.pages.reserve(count, GFP_KERNEL)?;
let mut flags = kernel::bindings::FOLL_LONGTERM;
if writable {
flags |= kernel::bindings::FOLL_WRITE;
}
// SAFETY: No preconditions; it reads the current mm.
if unsafe { asahi_user_mm_is_subpage() } {
// SAFETY: The array has room for `count` entries. On success the call stored `count`
// pages, which the records it returned in `fragments` hold.
let ret = unsafe {
asahi_pin_user_fragments(
addr as _,
count as _,
flags,
this.pages.as_mut_ptr().cast(),
&mut this.fragments,
)
};
if ret < 0 {
return Err(Error::from_errno(ret as i32));
}
// SAFETY: See above.
unsafe { this.pages.inc_len(count) };
return Ok(this);
}
while this.pages.len() < count {
let done = this.pages.len();
let chunk = (count - done).min(PIN_CHUNK);
Expand Down Expand Up @@ -556,7 +581,11 @@ impl UserPages {

impl Drop for UserPages {
fn drop(&mut self) {
if !self.pages.is_empty() {
if !self.fragments.is_null() {
// SAFETY: The records hold every page of the array and are released only here.
// Pages the device may have written are marked dirty.
unsafe { asahi_unpin_user_fragments(self.fragments, self.writable) };
} else if !self.pages.is_empty() {
// SAFETY: The array holds pages pinned by `pin_user_pages_fast()` and not yet
// unpinned. Pages the device may have written are marked dirty.
unsafe {
Expand Down Expand Up @@ -985,6 +1014,19 @@ pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result {
Ok(())
}

// user_pages.c: long-term pins in a process whose page size is smaller than the native one.
extern "C" {
fn asahi_user_mm_is_subpage() -> bool;
fn asahi_pin_user_fragments(
start: core::ffi::c_ulong,
nr_pages: core::ffi::c_ulong,
gup_flags: core::ffi::c_uint,
pages: *mut *mut core::ffi::c_void,
out: *mut *mut core::ffi::c_void,
) -> core::ffi::c_long;
fn asahi_unpin_user_fragments(frags: *mut core::ffi::c_void, dirty: bool);
}

#[cfg(CONFIG_DRM_ASAHI_G15)]
extern "C" {
fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool;
Expand Down
132 changes: 132 additions & 0 deletions drivers/gpu/drm/asahi/user_pages.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
// SPDX-License-Identifier: GPL-2.0-only OR MIT
/*
* Long-term pins of user memory as whole native pages, for GEM_USERPTR.
*
* The GPU maps memory in native pages. A process whose user page size is
* smaller (CONFIG_MM_SUBPAGE) maps each of its pages to a fragment of a
* native page, and only the fragment API can pin those. A native page can be
* given to the GPU only where the process maps all of it, its fragments in
* order at consecutive addresses: then the GPU sees what the CPU sees, and no
* memory the process does not map.
*/

#include <linux/mm.h>
#include <linux/mm_subpage.h>
#include <linux/sched.h>
#include <linux/slab.h>

struct asahi_user_fragments;

bool asahi_user_mm_is_subpage(void);
long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
struct asahi_user_fragments **out);
void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty);

#ifdef CONFIG_MM_SUBPAGE
/* Pinned fragments of a range; released only through asahi_unpin_user_fragments(). */
struct asahi_user_fragments {
unsigned long count;
struct user_page_fragment records[];
};

bool asahi_user_mm_is_subpage(void)
{
return current->mm && mm_page_size(current->mm) < PAGE_SIZE;
}

void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty)
{
if (!frags)
return;
release_user_fragments(frags->records, frags->count, dirty);
kvfree(frags);
}

/* The native page that @rec and its @n - 1 successors map in order, or NULL. */
static struct page *whole_native_page(const struct user_page_fragment *rec,
unsigned long n, unsigned long granule)
{
unsigned long i;

if (!IS_ALIGNED(rec->offset, PAGE_SIZE))
return NULL;
for (i = 0; i < n; i++)
if (rec[i].folio != rec->folio || rec[i].length != granule ||
rec[i].offset != rec->offset + i * granule)
return NULL;
return folio_page(rec->folio, rec->offset >> PAGE_SHIFT);
}

/*
* Pins @nr_pages native pages' worth of the current process's memory at
* @start (native-page aligned) and stores the native pages in @pages. All or
* nothing: -EFAULT if a page cannot be pinned or a native page is not mapped
* whole and in order; -ENOMEM if the records cannot be allocated.
*/
long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
struct asahi_user_fragments **out)
{
struct mm_struct *mm = current->mm;
unsigned long granule = mm_page_size(mm);
unsigned long per_page = PAGE_SIZE / granule;
unsigned long length = nr_pages << PAGE_SHIFT, done = 0, i;
struct asahi_user_fragments *frags;
long ret = 0;

if (nr_pages > (ULONG_MAX >> PAGE_SHIFT) / per_page)
return -ENOMEM;
frags = kvzalloc(struct_size(frags, records, nr_pages * per_page),
GFP_KERNEL_ACCOUNT);
if (!frags)
return -ENOMEM;
while (done < length) {
long n = pin_user_fragments_remote(mm, start + done, length - done,
gup_flags, frags->records + frags->count,
nr_pages * per_page - frags->count);

if (n <= 0) {
ret = (n == 0 || n == -EOPNOTSUPP) ? -EFAULT : n;
goto fail;
}
for (i = frags->count; i < frags->count + n; i++)
done += frags->records[i].length;
frags->count += n;
}
/* Fragments never cross a native page, so a whole page is per_page records. */
if (frags->count != nr_pages * per_page) {
ret = -EFAULT;
goto fail;
}
for (i = 0; i < nr_pages; i++) {
pages[i] = whole_native_page(&frags->records[i * per_page], per_page, granule);
if (!pages[i]) {
ret = -EFAULT;
goto fail;
}
}
*out = frags;
return 0;

fail:
asahi_unpin_user_fragments(frags, false);
return ret;
}
#else
bool asahi_user_mm_is_subpage(void)
{
return false;
}

void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty)
{
}

long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
struct asahi_user_fragments **out)
{
return -EOPNOTSUPP;
}
#endif
4 changes: 4 additions & 0 deletions include/uapi/drm/asahi_drm.h
Original file line number Diff line number Diff line change
Expand Up @@ -780,6 +780,10 @@ struct drm_asahi_gem_create {
* while the object lives. Device and PFN mappings, including CPU mappings of
* other buffer objects, are refused (EFAULT).
*
* In a process whose page size is below 16 KiB, each 16 KiB of the range must
* be one native page that the process maps whole and in order (EFAULT
* otherwise).
*
* The object cannot be mapped with DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET, exported,
* given advice with DRM_IOCTL_ASAHI_GEM_MADVISE, bound with
* DRM_IOCTL_ASAHI_GEM_BIND_OBJECT or registered as a VM status mirror (all
Expand Down
25 changes: 19 additions & 6 deletions mm/memory.c
Original file line number Diff line number Diff line change
Expand Up @@ -6838,15 +6838,24 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm,
return err;
}

/*
* A uffd-wp marker in a registered VMA is unpopulated memory that remembers its
* protection: populate it like a none neighbour, then protect it (see below).
* Otherwise every first write under WP_UNPOPULATED takes a native page of its own.
*/
static unsigned long anon_subpage_missing_mask(pte_t *first, unsigned int nr,
unsigned int target, unsigned int offset)
unsigned int target, unsigned int offset,
bool uffd_wp)
{
unsigned long mask = 0;
unsigned int i;

for (i = 0; i < nr; i++)
if (i == target || pte_none(ptep_get(first + i)))
for (i = 0; i < nr; i++) {
pte_t pte = ptep_get(first + i);

if (i == target || pte_none(pte) || (uffd_wp && pte_is_uffd_wp_marker(pte)))
mask |= BIT(offset + i);
}
return mask;
}

Expand All @@ -6860,6 +6869,7 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf)
unsigned long start, end, lower, mask, current_mask, window_mask;
unsigned int first_offset, nr, target, i, index, mapped;
bool pooled, fresh, force_private = false, retry_private;
bool uffd_wp = userfaultfd_wp(vma);
vm_fault_t ret;
pte_t *first;
int err;
Expand All @@ -6880,7 +6890,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf)
vmf->pte = pte_offset_map_mm(mm, vmf->pmd, vmf->address);
if (!vmf->pte)
return 0;
mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset);
mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset,
uffd_wp);
pte_unmap(vmf->pte);
vmf->pte = NULL;
err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask,
Expand Down Expand Up @@ -6930,7 +6941,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf)
}

first = vmf->pte - target;
current_mask = anon_subpage_missing_mask(first, nr, target, first_offset);
current_mask = anon_subpage_missing_mask(first, nr, target, first_offset,
uffd_wp);
/* Do not lose newly missing neighbors or publish sparse dense-path supply.
* Return to the fault handler on a racing mask change instead of spinning
* under repeated zaps. No mapping has been published yet.
Expand All @@ -6957,7 +6969,8 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf)
entry = pte_sw_mkyoung(entry);
if (vma->vm_flags & VM_WRITE)
entry = pte_mkwrite(pte_mkdirty(entry), vma);
if (i == target && vmf_orig_pte_uffd_wp(vmf))
if (i == target ? vmf_orig_pte_uffd_wp(vmf) :
pte_is_uffd_wp_marker(ptep_get(first + i)))
entry = pte_mkuffd(entry);
set_pte_at(mm, address, first + i, entry);
update_mmu_cache(vma, address, first + i);
Expand Down
42 changes: 42 additions & 0 deletions tools/testing/selftests/arm64/user4k/uffd-wp-init.c
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,47 @@ static void markers(size_t ps)
printf("ok - uffd ABI %zuK poison: SIGBUS, MADV_FREE recovery, neighbours\n",
ps / 1024);
}
/*
* A write-first fault under WP_UNPOPULATED: in a smaller-granule mm the
* native window is filled at once, so the marker neighbours become present
* and keep their write protection; with the native granule they stay markers.
*/
static void marker_window(size_t ps, size_t native)
{
unsigned char *map = mmap(NULL, 4 * ps + native, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
CHECK(map != MAP_FAILED);
unsigned char *p = (unsigned char *)(((uintptr_t)map + native - 1) &
~(uintptr_t)(native - 1));
int fd = syscall(SYS_userfaultfd,
O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY);
CHECK(fd >= 0);
struct uffdio_api api = { .api = UFFD_API,
.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
UFFD_FEATURE_WP_UNPOPULATED |
UFFD_FEATURE_WP_ASYNC };
CHECK(!ioctl(fd, UFFDIO_API, &api));
struct uffdio_register reg = { .range = { (uintptr_t)p, 4 * ps },
.mode = UFFDIO_REGISTER_MODE_WP };
CHECK(!ioctl(fd, UFFDIO_REGISTER, &reg));
protect(fd, p, 4 * ps, UFFDIO_WRITEPROTECT_MODE_WP);
p[ps + 17] = 0x5a;
for (unsigned int i = 0; i < 4; i++) {
uint64_t entry = pagemap(p + i * ps, ps);

if (i == 1) {
CHECK((entry & (1ULL << 63)) && !(entry & (1ULL << 57)));
continue;
}
CHECK(entry & (1ULL << 57));
CHECK(!!(entry & (1ULL << 63)) == (ps < native));
CHECK(p[i * ps + 17] == 0);
CHECK(pagemap(p + i * ps, ps) & (1ULL << 57));
}
CHECK(!close(fd) && !munmap(map, 4 * ps + native));
printf("ok - uffd ABI %zuK unpopulated WP write-first window keeps neighbour protection\n",
ps / 1024);
}
struct pager_request {
uintptr_t address;
size_t size;
Expand Down Expand Up @@ -783,6 +824,7 @@ static int worker(unsigned long expected, unsigned long native)
missing(ps, native);
file_minor(ps, native);
markers(ps);
marker_window(ps, native);
cross_pager(ps, native, 0);
cross_pager(ps, native, 1);
private_missing(ps);
Expand Down
Loading