macro stack fast path in High accuracy + Preflushable readback throttle experiments (measured)

src/video_core/buffer_cache/buffer_cache.h

@@ -100,6 +100,8 @@ void BufferCache<P>::RunGarbageCollector(bool aggressive_gc) {
     }
 }
 
+/// End-of-frame bookkeeping: retires async buffer downloads, drops stale uniform ranges and
+/// the occasional threshold report.
 template <class P>
 void BufferCache<P>::TickFrame() {
     // Homebrew console apps don't create or bind any channels, so this will be nullptr.
@@ -253,6 +255,9 @@ bool BufferCache<P>::OnCPUWrite(DAddr device_addr, u64 size) {
     return false;
 }
 
+/// Resolves the guest read of [device_addr, device_addr + size) to a download area: page
+/// aligned, and marked preemtive (no synchronous flush needed) only when the data is already
+/// back in guest memory — downloaded at a fence and not still pending a frame drain.
 template <class P>
 std::optional<VideoCore::RasterizerDownloadArea> BufferCache<P>::GetFlushArea(DAddr device_addr,
                                                                               u64 size) {
@@ -263,8 +268,10 @@ std::optional<VideoCore::RasterizerDownloadArea> BufferCache<P>::GetFlushArea(DA
     area->start_address = device_addr_start_aligned;
     area->end_address = device_addr_end_aligned;
     // Preflushable data is downloaded at fences, but only where it is marked (QueueGpuWrite): the
-    // rest of the range still needs this flush.
-    if (memory_tracker.IsRegionPreflushable(device_addr, size) &&
+    // rest of the range still needs this flush. With LazyPreflushDownloads nothing downloads at
+    // fences, so this fast path must not claim the data is already back: every read of a
+    // GPU-modified range flushes (preemtive false) through OnCPURead's synchronous flush.
+    if (!LazyPreflushDownloads() && memory_tracker.IsRegionPreflushable(device_addr, size) &&
         !HasGpuWritesNotDownloadedAtFence(device_addr_start_aligned,
                                           device_addr_end_aligned - device_addr_start_aligned)) {
         area->preemtive = true;

src/video_core/buffer_cache/buffer_cache_base.h

@@ -605,6 +605,25 @@ class BufferCache : public VideoCommon::ChannelSetupCaches<BufferCacheChannelInf
         return gpu_memory == nullptr || !gpu_memory->UsesNativeTiling();
     }
 
+    /// Reads EDEN_NX2_LAZY_PREFLUSH=1 once (see LazyPreflushDownloads).
+    static bool EnvLazyPreflush() {
+        const char* const value = std::getenv("EDEN_NX2_LAZY_PREFLUSH");
+        return value != nullptr && value[0] == '1';
+    }
+
+    /// EDEN_NX2_LAZY_PREFLUSH=1: ranges the guest CPU has read once (Preflushable) are not
+    /// re-downloaded at every fence either. Every fence that follows a GPU write to such a range
+    /// copies it back whether anything reads it again or not; Ocarina of Time read GPU-written
+    /// memory ~3 times a second (single pages, via OnCPURead) while its Preflushable ranges
+    /// were copied back at 1.9 GiB/s in 135k copies/s (forest scenes, GPU thread saturated).
+    /// Instead each GPU write arms the guest read traps over the whole range (the generation
+    /// bump alone invalidates the cores' rasterizer read areas), and a read flushes once,
+    /// synchronously, through OnCPURead — the path a first read already takes.
+    bool LazyPreflushDownloads() const {
+        static const bool lazy = EnvLazyPreflush();
+        return lazy;
+    }
+
     /// Whether a GPU write to this range is copied back at the next fence. Ranges the guest CPU
     /// has read (Preflushable, set by the first reactive flush) always are: later reads skip the
     /// flush and rely on these downloads, e.g. collision meshes skinned by compute shaders.
@@ -636,11 +655,16 @@ class BufferCache : public VideoCommon::ChannelSetupCaches<BufferCacheChannelInf
     /// A GPU write: queues its Preflushable parts for download at the next fence and traps reads
     /// of the rest. Queuing the whole range when any page of it was Preflushable copied whole
     /// written storage buffers (up to 8 MiB) back at every fence: ~5 GB/s in title 1's forest.
+    /// With LazyPreflushDownloads nothing is queued at fences at all; reads flush on demand.
     void QueueGpuWrite(DAddr device_addr, u64 size) {
         if (AsyncBufferDownloads()) {
             uncommitted_gpu_modified_ranges.Add(device_addr, size);
             return;
         }
+        if (LazyPreflushDownloads()) {
+            ArmReadTraps(device_addr, size);
+            return;
+        }
         const DAddr end = device_addr + size;
         DAddr cursor = device_addr;
         memory_tracker.ForEachPreflushableRange(device_addr, size, [&](DAddr start, u64 range_size) {

src/video_core/engines/maxwell_3d.cpp

@@ -211,6 +211,10 @@ bool Maxwell3D::IsMethodExecutable(u32 method) {
     }
 }
 
+/// Handles a write to a macro method or its ARGS register: accumulates the call's parameter
+/// words and dispatches the macro when the call is complete, choosing between the stack
+/// snapshot path (parameters copied at arrival) and the per-call vector that accurate mode
+/// refreshes in place.
 void Maxwell3D::ProcessMacro(Core::System& system, u32 method, const u32* base_start, u32 amount, bool is_last_call) {
     if (executing_macro == 0) {
         // A macro call must begin by writing the macro method's register, not its argument.
@@ -228,9 +232,19 @@ void Maxwell3D::ProcessMacro(Core::System& system, u32 method, const u32* base_s
     // (and the macro memo stores the parameters after it ran). Only accurate GPU mode rewrites
     // the parameters in place (RefreshParameters), so it keeps the vector.
     // EDEN_NX2_MACRO_COPY=1: always use the vector (for comparisons).
+    // EDEN_NX2_MACRO_STACK_HIGH=1: the stack path in accurate mode too — the parameter copy
+    // snapshots the words at arrival, so nothing can rewrite them under the macro and the
+    // per-call vector/segment bookkeeping (and RefreshParameters re-reads) are not needed.
+    // Ocarina of Time makes ~900k single-run macro calls a second; the vector path costs
+    // ~100-200 ns of plumbing per call on the GPU thread for a semantic (re-fetch at use)
+    // that a snapshot satisfies for every macro the title runs.
     static const bool always_copy = std::getenv("EDEN_NX2_MACRO_COPY") != nullptr;
+    static const bool stack_in_high = [] {
+        const char* value = std::getenv("EDEN_NX2_MACRO_STACK_HIGH");
+        return value != nullptr && value[0] == '1';
+    }();
     if (is_last_call && macro_params.empty() && amount <= 64 && !always_copy &&
-        !Settings::IsGPULevelHigh()) {
+        (!Settings::IsGPULevelHigh() || stack_in_high)) {
         std::array<u32, 64> parameters;
         std::copy_n(base_start, amount, parameters.begin());
         ConsumeSink(system);

src/video_core/memory_manager.cpp

@@ -117,6 +117,9 @@ inline void MemoryManager::SetBigPageContinuous(size_t big_page_index, bool valu
 }
 
 GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) {
+    // Seqlock open: lock-free readers (GpuToCpuAddress's cache, MaxContinuousRange) retry a
+    // walk whose serial moved, so a mutation in flight is never read as a torn table.
+    map_serial.fetch_add(1, std::memory_order_release);
     [[maybe_unused]] u64 remaining_size{size};
     if (entry_type == EntryType::Mapped) {
         page_table.ReserveRange(gpu_addr, size);
@@ -158,6 +161,8 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr de
 }
 
 GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) {
+    // Seqlock open, as in PageTableOp above.
+    map_serial.fetch_add(1, std::memory_order_release);
     [[maybe_unused]] u64 remaining_size{size};
     GPUVAddr modified_start{};
     u64 modified_size{};
@@ -583,6 +588,16 @@ bool MemoryManager::IsMemoryDirty(GPUVAddr gpu_addr, size_t size,
     return result;
 }
 
+/// Reads EDEN_NX2_MEMSTATS once: whether the MaxContinuousRange cache logs its hit/miss and
+/// page-table-change counters (see MaxContinuousRange below).
+static bool MemstatsEnabled() {
+    static const bool on = std::getenv("EDEN_NX2_MEMSTATS") != nullptr;
+    return on;
+}
+
+/// Length of the largest continuously mapped range starting at gpu_addr and not exceeding
+/// size, answered from a per-thread direct-mapped cache invalidated by page-table changes;
+/// with EDEN_NX2_MEMSTATS=1, hit/miss and remap counters are logged every 500k queries.
 size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const {
     // Native storage buffers ask this for up to 8 MiB on each bind, ~70,000 times a second in
     // title 1's forest (17% of the GPU thread walking pages): keep answers per thread until the
@@ -594,13 +609,35 @@ size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const {
         u64 serial;
         size_t result;
     };
-    thread_local std::array<Entry, 64> cache{};
+    // 1024 entries: Ocarina of Time binds more distinct (address, size) pairs than 64 held —
+    // a vista run thrashed 22% of ~85k queries/s with a stable page table (785 serial changes,
+    // all at boot; measured with EDEN_NX2_MEMSTATS), each miss re-walking up to 8 MiB.
+    thread_local std::array<Entry, 1024> cache{};
     const u64 serial = map_serial.load(std::memory_order_acquire);
-    Entry& entry = cache[((gpu_addr ^ size) * 0x9E3779B97F4A7C15ULL) >> 58];
+    Entry& entry = cache[((gpu_addr ^ size) * 0x9E3779B97F4A7C15ULL) >> 54];
+    // EDEN_NX2_MEMSTATS=1: per-thread hit/miss and page-table-change counts, logged every
+    // 500k queries (finds whether the miss cost of this cache is thrash or remaps).
+    thread_local u64 stat_queries{}, stat_hits{}, stat_misses{}, stat_serials{}, stat_last_serial{};
+    const bool stats = MemstatsEnabled();
+    if (stats) [[unlikely]] {
+        ++stat_queries;
+        stat_serials += serial != stat_last_serial;
+        stat_last_serial = serial;
+    }
     if (entry.owner == unique_identifier + 1 && entry.gpu_addr == gpu_addr && entry.size == size &&
         entry.serial == serial) {
+        if (stats) [[unlikely]] {
+            if (++stat_hits % 500000 == 0) {
+                LOG_INFO(HW_GPU,
+                         "MaxContinuousRange: {} queries ({} hit, {} miss, {} serial changes)",
+                         stat_queries, stat_hits, stat_misses, stat_serials);
+            }
+        }
         return entry.result;
     }
+    if (stats) [[unlikely]] {
+        ++stat_misses;
+    }
     const size_t result = MaxContinuousRangeUncached(gpu_addr, size);
     entry = {unique_identifier + 1, gpu_addr, size, serial, result};
     return result;