diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index e73d1cbd6c..0375bad7ac 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -100,6 +100,8 @@ void BufferCache
::RunGarbageCollector(bool aggressive_gc) {
}
}
+/// End-of-frame bookkeeping: retires async buffer downloads, drops stale uniform ranges and
+/// the occasional threshold report.
template
void BufferCache::TickFrame() {
// Homebrew console apps don't create or bind any channels, so this will be nullptr.
@@ -253,6 +255,9 @@ bool BufferCache
::OnCPUWrite(DAddr device_addr, u64 size) {
return false;
}
+/// Resolves the guest read of [device_addr, device_addr + size) to a download area: page
+/// aligned, and marked preemtive (no synchronous flush needed) only when the data is already
+/// back in guest memory — downloaded at a fence and not still pending a frame drain.
template
std::optional BufferCache::GetFlushArea(DAddr device_addr,
u64 size) {
@@ -263,8 +268,10 @@ std::optional BufferCache::GetFlushArea(DA
area->start_address = device_addr_start_aligned;
area->end_address = device_addr_end_aligned;
// Preflushable data is downloaded at fences, but only where it is marked (QueueGpuWrite): the
- // rest of the range still needs this flush.
- if (memory_tracker.IsRegionPreflushable(device_addr, size) &&
+ // rest of the range still needs this flush. With LazyPreflushDownloads nothing downloads at
+ // fences, so this fast path must not claim the data is already back: every read of a
+ // GPU-modified range flushes (preemtive false) through OnCPURead's synchronous flush.
+ if (!LazyPreflushDownloads() && memory_tracker.IsRegionPreflushable(device_addr, size) &&
!HasGpuWritesNotDownloadedAtFence(device_addr_start_aligned,
device_addr_end_aligned - device_addr_start_aligned)) {
area->preemtive = true;
diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h
index 33ada38210..a6ae645a16 100644
--- a/src/video_core/buffer_cache/buffer_cache_base.h
+++ b/src/video_core/buffer_cache/buffer_cache_base.h
@@ -605,6 +605,25 @@ class BufferCache : public VideoCommon::ChannelSetupCachesUsesNativeTiling();
}
+ /// Reads EDEN_NX2_LAZY_PREFLUSH=1 once (see LazyPreflushDownloads).
+ static bool EnvLazyPreflush() {
+ const char* const value = std::getenv("EDEN_NX2_LAZY_PREFLUSH");
+ return value != nullptr && value[0] == '1';
+ }
+
+ /// EDEN_NX2_LAZY_PREFLUSH=1: ranges the guest CPU has read once (Preflushable) are not
+ /// re-downloaded at every fence either. Every fence that follows a GPU write to such a range
+ /// copies it back whether anything reads it again or not; Ocarina of Time read GPU-written
+ /// memory ~3 times a second (single pages, via OnCPURead) while its Preflushable ranges
+ /// were copied back at 1.9 GiB/s in 135k copies/s (forest scenes, GPU thread saturated).
+ /// Instead each GPU write arms the guest read traps over the whole range (the generation
+ /// bump alone invalidates the cores' rasterizer read areas), and a read flushes once,
+ /// synchronously, through OnCPURead — the path a first read already takes.
+ bool LazyPreflushDownloads() const {
+ static const bool lazy = EnvLazyPreflush();
+ return lazy;
+ }
+
/// Whether a GPU write to this range is copied back at the next fence. Ranges the guest CPU
/// has read (Preflushable, set by the first reactive flush) always are: later reads skip the
/// flush and rely on these downloads, e.g. collision meshes skinned by compute shaders.
@@ -636,11 +655,16 @@ class BufferCache : public VideoCommon::ChannelSetupCaches parameters;
std::copy_n(base_start, amount, parameters.begin());
ConsumeSink(system);
diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp
index 11115710e7..2c2fd671e5 100644
--- a/src/video_core/memory_manager.cpp
+++ b/src/video_core/memory_manager.cpp
@@ -117,6 +117,9 @@ inline void MemoryManager::SetBigPageContinuous(size_t big_page_index, bool valu
}
GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) {
+ // Seqlock open: lock-free readers (GpuToCpuAddress's cache, MaxContinuousRange) retry a
+ // walk whose serial moved, so a mutation in flight is never read as a torn table.
+ map_serial.fetch_add(1, std::memory_order_release);
[[maybe_unused]] u64 remaining_size{size};
if (entry_type == EntryType::Mapped) {
page_table.ReserveRange(gpu_addr, size);
@@ -158,6 +161,8 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr de
}
GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) {
+ // Seqlock open, as in PageTableOp above.
+ map_serial.fetch_add(1, std::memory_order_release);
[[maybe_unused]] u64 remaining_size{size};
GPUVAddr modified_start{};
u64 modified_size{};
@@ -583,6 +588,16 @@ bool MemoryManager::IsMemoryDirty(GPUVAddr gpu_addr, size_t size,
return result;
}
+/// Reads EDEN_NX2_MEMSTATS once: whether the MaxContinuousRange cache logs its hit/miss and
+/// page-table-change counters (see MaxContinuousRange below).
+static bool MemstatsEnabled() {
+ static const bool on = std::getenv("EDEN_NX2_MEMSTATS") != nullptr;
+ return on;
+}
+
+/// Length of the largest continuously mapped range starting at gpu_addr and not exceeding
+/// size, answered from a per-thread direct-mapped cache invalidated by page-table changes;
+/// with EDEN_NX2_MEMSTATS=1, hit/miss and remap counters are logged every 500k queries.
size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const {
// Native storage buffers ask this for up to 8 MiB on each bind, ~70,000 times a second in
// title 1's forest (17% of the GPU thread walking pages): keep answers per thread until the
@@ -594,13 +609,35 @@ size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const {
u64 serial;
size_t result;
};
- thread_local std::array cache{};
+ // 1024 entries: Ocarina of Time binds more distinct (address, size) pairs than 64 held —
+ // a vista run thrashed 22% of ~85k queries/s with a stable page table (785 serial changes,
+ // all at boot; measured with EDEN_NX2_MEMSTATS), each miss re-walking up to 8 MiB.
+ thread_local std::array cache{};
const u64 serial = map_serial.load(std::memory_order_acquire);
- Entry& entry = cache[((gpu_addr ^ size) * 0x9E3779B97F4A7C15ULL) >> 58];
+ Entry& entry = cache[((gpu_addr ^ size) * 0x9E3779B97F4A7C15ULL) >> 54];
+ // EDEN_NX2_MEMSTATS=1: per-thread hit/miss and page-table-change counts, logged every
+ // 500k queries (finds whether the miss cost of this cache is thrash or remaps).
+ thread_local u64 stat_queries{}, stat_hits{}, stat_misses{}, stat_serials{}, stat_last_serial{};
+ const bool stats = MemstatsEnabled();
+ if (stats) [[unlikely]] {
+ ++stat_queries;
+ stat_serials += serial != stat_last_serial;
+ stat_last_serial = serial;
+ }
if (entry.owner == unique_identifier + 1 && entry.gpu_addr == gpu_addr && entry.size == size &&
entry.serial == serial) {
+ if (stats) [[unlikely]] {
+ if (++stat_hits % 500000 == 0) {
+ LOG_INFO(HW_GPU,
+ "MaxContinuousRange: {} queries ({} hit, {} miss, {} serial changes)",
+ stat_queries, stat_hits, stat_misses, stat_serials);
+ }
+ }
return entry.result;
}
+ if (stats) [[unlikely]] {
+ ++stat_misses;
+ }
const size_t result = MaxContinuousRangeUncached(gpu_addr, size);
entry = {unique_identifier + 1, gpu_addr, size, serial, result};
return result;