diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e73d1cbd6c..0375bad7ac 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -100,6 +100,8 @@ void BufferCache

::RunGarbageCollector(bool aggressive_gc) { } } +/// End-of-frame bookkeeping: retires async buffer downloads, drops stale uniform ranges and +/// the occasional threshold report. template void BufferCache

::TickFrame() { // Homebrew console apps don't create or bind any channels, so this will be nullptr. @@ -253,6 +255,9 @@ bool BufferCache

::OnCPUWrite(DAddr device_addr, u64 size) { return false; } +/// Resolves the guest read of [device_addr, device_addr + size) to a download area: page +/// aligned, and marked preemtive (no synchronous flush needed) only when the data is already +/// back in guest memory — downloaded at a fence and not still pending a frame drain. template std::optional BufferCache

::GetFlushArea(DAddr device_addr, u64 size) { @@ -263,8 +268,10 @@ std::optional BufferCache

::GetFlushArea(DA area->start_address = device_addr_start_aligned; area->end_address = device_addr_end_aligned; // Preflushable data is downloaded at fences, but only where it is marked (QueueGpuWrite): the - // rest of the range still needs this flush. - if (memory_tracker.IsRegionPreflushable(device_addr, size) && + // rest of the range still needs this flush. With LazyPreflushDownloads nothing downloads at + // fences, so this fast path must not claim the data is already back: every read of a + // GPU-modified range flushes (preemtive false) through OnCPURead's synchronous flush. + if (!LazyPreflushDownloads() && memory_tracker.IsRegionPreflushable(device_addr, size) && !HasGpuWritesNotDownloadedAtFence(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned)) { area->preemtive = true; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 33ada38210..a6ae645a16 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -605,6 +605,25 @@ class BufferCache : public VideoCommon::ChannelSetupCachesUsesNativeTiling(); } + /// Reads EDEN_NX2_LAZY_PREFLUSH=1 once (see LazyPreflushDownloads). + static bool EnvLazyPreflush() { + const char* const value = std::getenv("EDEN_NX2_LAZY_PREFLUSH"); + return value != nullptr && value[0] == '1'; + } + + /// EDEN_NX2_LAZY_PREFLUSH=1: ranges the guest CPU has read once (Preflushable) are not + /// re-downloaded at every fence either. Every fence that follows a GPU write to such a range + /// copies it back whether anything reads it again or not; Ocarina of Time read GPU-written + /// memory ~3 times a second (single pages, via OnCPURead) while its Preflushable ranges + /// were copied back at 1.9 GiB/s in 135k copies/s (forest scenes, GPU thread saturated). + /// Instead each GPU write arms the guest read traps over the whole range (the generation + /// bump alone invalidates the cores' rasterizer read areas), and a read flushes once, + /// synchronously, through OnCPURead — the path a first read already takes. + bool LazyPreflushDownloads() const { + static const bool lazy = EnvLazyPreflush(); + return lazy; + } + /// Whether a GPU write to this range is copied back at the next fence. Ranges the guest CPU /// has read (Preflushable, set by the first reactive flush) always are: later reads skip the /// flush and rely on these downloads, e.g. collision meshes skinned by compute shaders. @@ -636,11 +655,16 @@ class BufferCache : public VideoCommon::ChannelSetupCaches parameters; std::copy_n(base_start, amount, parameters.begin()); ConsumeSink(system); diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 11115710e7..2c2fd671e5 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -117,6 +117,9 @@ inline void MemoryManager::SetBigPageContinuous(size_t big_page_index, bool valu } GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) { + // Seqlock open: lock-free readers (GpuToCpuAddress's cache, MaxContinuousRange) retry a + // walk whose serial moved, so a mutation in flight is never read as a torn table. + map_serial.fetch_add(1, std::memory_order_release); [[maybe_unused]] u64 remaining_size{size}; if (entry_type == EntryType::Mapped) { page_table.ReserveRange(gpu_addr, size); @@ -158,6 +161,8 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr de } GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) { + // Seqlock open, as in PageTableOp above. + map_serial.fetch_add(1, std::memory_order_release); [[maybe_unused]] u64 remaining_size{size}; GPUVAddr modified_start{}; u64 modified_size{}; @@ -583,6 +588,16 @@ bool MemoryManager::IsMemoryDirty(GPUVAddr gpu_addr, size_t size, return result; } +/// Reads EDEN_NX2_MEMSTATS once: whether the MaxContinuousRange cache logs its hit/miss and +/// page-table-change counters (see MaxContinuousRange below). +static bool MemstatsEnabled() { + static const bool on = std::getenv("EDEN_NX2_MEMSTATS") != nullptr; + return on; +} + +/// Length of the largest continuously mapped range starting at gpu_addr and not exceeding +/// size, answered from a per-thread direct-mapped cache invalidated by page-table changes; +/// with EDEN_NX2_MEMSTATS=1, hit/miss and remap counters are logged every 500k queries. size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const { // Native storage buffers ask this for up to 8 MiB on each bind, ~70,000 times a second in // title 1's forest (17% of the GPU thread walking pages): keep answers per thread until the @@ -594,13 +609,35 @@ size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const { u64 serial; size_t result; }; - thread_local std::array cache{}; + // 1024 entries: Ocarina of Time binds more distinct (address, size) pairs than 64 held — + // a vista run thrashed 22% of ~85k queries/s with a stable page table (785 serial changes, + // all at boot; measured with EDEN_NX2_MEMSTATS), each miss re-walking up to 8 MiB. + thread_local std::array cache{}; const u64 serial = map_serial.load(std::memory_order_acquire); - Entry& entry = cache[((gpu_addr ^ size) * 0x9E3779B97F4A7C15ULL) >> 58]; + Entry& entry = cache[((gpu_addr ^ size) * 0x9E3779B97F4A7C15ULL) >> 54]; + // EDEN_NX2_MEMSTATS=1: per-thread hit/miss and page-table-change counts, logged every + // 500k queries (finds whether the miss cost of this cache is thrash or remaps). + thread_local u64 stat_queries{}, stat_hits{}, stat_misses{}, stat_serials{}, stat_last_serial{}; + const bool stats = MemstatsEnabled(); + if (stats) [[unlikely]] { + ++stat_queries; + stat_serials += serial != stat_last_serial; + stat_last_serial = serial; + } if (entry.owner == unique_identifier + 1 && entry.gpu_addr == gpu_addr && entry.size == size && entry.serial == serial) { + if (stats) [[unlikely]] { + if (++stat_hits % 500000 == 0) { + LOG_INFO(HW_GPU, + "MaxContinuousRange: {} queries ({} hit, {} miss, {} serial changes)", + stat_queries, stat_hits, stat_misses, stat_serials); + } + } return entry.result; } + if (stats) [[unlikely]] { + ++stat_misses; + } const size_t result = MaxContinuousRangeUncached(gpu_addr, size); entry = {unique_identifier + 1, gpu_addr, size, serial, result}; return result;