Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion drivers/gpu/drm/asahi/Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: GPL-2.0

obj-$(CONFIG_DRM_ASAHI) += aurora.o
aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/sleep.o g17/pmpdash.o
aurora-y := asahi_core.o fwid.o legacy_sched.o sysfs.o user_pages.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o g17/sleep.o g17/pmpdash.o
CFLAGS_g17/trace.o := -I$(src)/g17

aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o
Expand Down
50 changes: 47 additions & 3 deletions drivers/gpu/drm/asahi/gem.rs
Original file line number Diff line number Diff line change
Expand Up @@ -459,9 +459,13 @@ pub(crate) struct UserPages {
mm: ARef<Mm>,
/// Pages counted in `pinned_vm`: all of the range, including any not pinned yet.
counted: i64,
/// The fragment pins of a process with a smaller page size, which hold `pages` (null when
/// `pages` are ordinary pins).
fragments: *mut core::ffi::c_void,
}

// SAFETY: The pinned pages may be released from any thread; the mm reference is thread-safe.
// SAFETY: The pinned pages and fragment records may be released from any thread; the mm
// reference is thread-safe.
unsafe impl Send for UserPages {}
// SAFETY: Shared access only reads the array.
unsafe impl Sync for UserPages {}
Expand All @@ -475,7 +479,9 @@ impl UserPages {
///
/// All of the range is pinned, long-term, or none of it. Fails with `EFAULT` when a page cannot
/// be pinned: unmapped, a device or PFN mapping, or a writable pin of a mapping that is not
/// writable or whose filesystem tracks writes (`FOLL_LONGTERM` refuses those). Fails with
/// writable or whose filesystem tracks writes (`FOLL_LONGTERM` refuses those). In a process
/// whose page size is smaller than the native one, a native page is accepted only where the
/// process maps all of it, in order at consecutive addresses (`EFAULT` otherwise). Fails with
/// `ENOMEM` when the pages would take the process past RLIMIT_MEMLOCK without CAP_IPC_LOCK,
/// counting every page pinned this way or by other long-term pinners.
///
Expand Down Expand Up @@ -510,12 +516,33 @@ impl UserPages {
writable,
mm,
counted: added,
fragments: core::ptr::null_mut(),
};
this.pages.reserve(count, GFP_KERNEL)?;
let mut flags = kernel::bindings::FOLL_LONGTERM;
if writable {
flags |= kernel::bindings::FOLL_WRITE;
}
// SAFETY: No preconditions; it reads the current mm.
if unsafe { asahi_user_mm_is_subpage() } {
// SAFETY: The array has room for `count` entries. On success the call stored `count`
// pages, which the records it returned in `fragments` hold.
let ret = unsafe {
asahi_pin_user_fragments(
addr as _,
count as _,
flags,
this.pages.as_mut_ptr().cast(),
&mut this.fragments,
)
};
if ret < 0 {
return Err(Error::from_errno(ret as i32));
}
// SAFETY: See above.
unsafe { this.pages.inc_len(count) };
return Ok(this);
}
while this.pages.len() < count {
let done = this.pages.len();
let chunk = (count - done).min(PIN_CHUNK);
Expand Down Expand Up @@ -556,7 +583,11 @@ impl UserPages {

impl Drop for UserPages {
fn drop(&mut self) {
if !self.pages.is_empty() {
if !self.fragments.is_null() {
// SAFETY: The records hold every page of the array and are released only here.
// Pages the device may have written are marked dirty.
unsafe { asahi_unpin_user_fragments(self.fragments, self.writable) };
} else if !self.pages.is_empty() {
// SAFETY: The array holds pages pinned by `pin_user_pages_fast()` and not yet
// unpinned. Pages the device may have written are marked dirty.
unsafe {
Expand Down Expand Up @@ -985,6 +1016,19 @@ pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result {
Ok(())
}

// user_pages.c: long-term pins in a process whose page size is smaller than the native one.
extern "C" {
fn asahi_user_mm_is_subpage() -> bool;
fn asahi_pin_user_fragments(
start: core::ffi::c_ulong,
nr_pages: core::ffi::c_ulong,
gup_flags: core::ffi::c_uint,
pages: *mut *mut core::ffi::c_void,
out: *mut *mut core::ffi::c_void,
) -> core::ffi::c_long;
fn asahi_unpin_user_fragments(frags: *mut core::ffi::c_void, dirty: bool);
}

#[cfg(CONFIG_DRM_ASAHI_G15)]
extern "C" {
fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool;
Expand Down
121 changes: 121 additions & 0 deletions drivers/gpu/drm/asahi/user_pages.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// SPDX-License-Identifier: GPL-2.0-only OR MIT
/*
* Long-term pins of user memory as whole native pages, for GEM_USERPTR.
*
* The GPU maps memory in native pages. A process whose user page size is
* smaller (CONFIG_MM_SUBPAGE) maps each of its pages to a fragment of a
* native page, and only the fragment API can pin those. A native page can be
* given to the GPU only where the process maps all of it, its fragments in
* order at consecutive addresses: then the GPU sees what the CPU sees, and no
* memory the process does not map.
*/

#include <linux/mm.h>
#include <linux/mm_subpage.h>
#include <linux/sched.h>
#include <linux/slab.h>

#include "user_pages.h"

#ifdef CONFIG_MM_SUBPAGE
/* Pinned fragments of a range; released only through asahi_unpin_user_fragments(). */
struct asahi_user_fragments {
unsigned long count;
struct user_page_fragment records[];
};

bool asahi_user_mm_is_subpage(void)
{
return current->mm && mm_page_size(current->mm) < PAGE_SIZE;
}

void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty)
{
if (!frags)
return;
release_user_fragments(frags->records, frags->count, dirty);
kvfree(frags);
}

/* The native page that @rec and its @n - 1 successors map in order, or NULL. */
static struct page *whole_native_page(const struct user_page_fragment *rec,
unsigned long n, unsigned long granule)
{
unsigned long i;

if (!IS_ALIGNED(rec->offset, PAGE_SIZE))
return NULL;
for (i = 0; i < n; i++)
if (rec[i].folio != rec->folio || rec[i].length != granule ||
rec[i].offset != rec->offset + i * granule)
return NULL;
return folio_page(rec->folio, rec->offset >> PAGE_SHIFT);
}

/*
* Pins @nr_pages native pages' worth of the current process's memory at
* @start (native-page aligned) and stores the native pages in @pages. All or
* nothing: -EFAULT if a page cannot be pinned or a native page is not mapped
* whole and in order; -ENOMEM if the records cannot be allocated.
*/
long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
struct asahi_user_fragments **out)
{
struct mm_struct *mm = current->mm;
unsigned long granule = mm_page_size(mm);
unsigned long per_page = PAGE_SIZE / granule;
unsigned long length = nr_pages << PAGE_SHIFT, done = 0, i;
struct asahi_user_fragments *frags;
long ret = 0;

if (nr_pages > (ULONG_MAX >> PAGE_SHIFT) / per_page)
return -ENOMEM;
frags = kvzalloc(struct_size(frags, records, nr_pages * per_page),
GFP_KERNEL_ACCOUNT);
if (!frags)
return -ENOMEM;
while (done < length) {
long n = pin_user_fragments_remote(mm, start + done, length - done,
gup_flags, frags->records + frags->count,
nr_pages * per_page - frags->count);

if (n <= 0) {
ret = n ?: -EFAULT;
goto fail;
}
for (i = frags->count; i < frags->count + n; i++)
done += frags->records[i].length;
frags->count += n;
}
for (i = 0; i < nr_pages; i++) {
pages[i] = whole_native_page(&frags->records[i * per_page], per_page, granule);
if (!pages[i]) {
ret = -EFAULT;
goto fail;
}
}
*out = frags;
return 0;

fail:
asahi_unpin_user_fragments(frags, false);
return ret;
}
#else
bool asahi_user_mm_is_subpage(void)
{
return false;
}

void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty)
{
}

long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
struct asahi_user_fragments **out)
{
return -EOPNOTSUPP;
}
#endif
17 changes: 17 additions & 0 deletions drivers/gpu/drm/asahi/user_pages.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
#ifndef _ASAHI_USER_PAGES_H
#define _ASAHI_USER_PAGES_H

#include <linux/types.h>

struct asahi_user_fragments;
struct page;

/* Called from gem.rs. */
bool asahi_user_mm_is_subpage(void);
long asahi_pin_user_fragments(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
struct asahi_user_fragments **out);
void asahi_unpin_user_fragments(struct asahi_user_fragments *frags, bool dirty);

#endif
4 changes: 4 additions & 0 deletions include/uapi/drm/asahi_drm.h
Original file line number Diff line number Diff line change
Expand Up @@ -780,6 +780,10 @@ struct drm_asahi_gem_create {
* while the object lives. Device and PFN mappings, including CPU mappings of
* other buffer objects, are refused (EFAULT).
*
* In a process whose page size is smaller than the native one, a native page
* is accepted only where the process maps all of it, in order at consecutive
* addresses; otherwise the call fails with EFAULT.
*
* The object cannot be mapped with DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET, exported,
* given advice with DRM_IOCTL_ASAHI_GEM_MADVISE, bound with
* DRM_IOCTL_ASAHI_GEM_BIND_OBJECT or registered as a VM status mirror (all
Expand Down
1 change: 1 addition & 0 deletions tools/testing/selftests/arm64/user4k/.gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
page-contract
gem-userptr

swap-fixture-init
same-quarter
Expand Down
4 changes: 3 additions & 1 deletion tools/testing/selftests/arm64/user4k/Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
TEST_GEN_PROGS := page-contract proc-args
TEST_GEN_PROGS := page-contract proc-args gem-userptr
TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init
TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init
TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init cow-pool
Expand Down Expand Up @@ -120,6 +120,8 @@ $(OUTPUT)/iommufd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536
$(OUTPUT)/iommufd-init: LDLIBS += -pthread
$(OUTPUT)/iommufd-init: CFLAGS += $(KHDR_INCLUDES)

$(OUTPUT)/gem-userptr: CFLAGS += $(KHDR_INCLUDES)

# Privileged recording-domain fixture; run only in a disposable test VM.
$(OUTPUT)/vfio-fixture-init: LDFLAGS += -static -Wl,-z,max-page-size=65536
$(OUTPUT)/compat-vfio-probe: compat-vfio-probe.c
Expand Down
Loading
Loading