experiment: defer large linear Pokopia texture uploads to Vulkan worker

.github/workflows/fork-windows-check.yml

@@ -8,6 +8,8 @@ on:
       - .github/workflows/fork-windows-check.yml
       - .github/workflows/build-windows.yml
       - src/CMakeLists.txt
+      - src/video_core/**
+      - src/shader_recompiler/**
       - src/yuzu/**
       - src/yuzu_cmd/**
       - src/qt_common/**

src/video_core/renderer_vulkan/vk_texture_cache.cpp

@@ -1257,6 +1257,15 @@ StagingBufferRef TextureCacheRuntime::UploadStagingBuffer(size_t size, bool defe
     return staging_buffer_pool.Request(size, MemoryUsage::Upload, deferred);
 }
 
+void TextureCacheRuntime::DeferGuestRead(u8* dst, Tegra::MaxwellDeviceMemoryManager& memory,
+                                         DAddr address, size_t size) {
+    // Runs when the worker records this chunk, before its commands are submitted: the copies
+    // recorded after it read the staging memory only once the GPU executes them.
+    scheduler.Record([dst, memory = &memory, address, size](vk::CommandBuffer) {
+        memory->ReadBlockUnsafe(address, dst, size);
+    });
+}
+
 StagingBufferRef TextureCacheRuntime::DownloadStagingBuffer(size_t size, bool deferred) {
     return staging_buffer_pool.Request(size, MemoryUsage::Download, deferred);
 }

src/video_core/renderer_vulkan/vk_texture_cache.h

@@ -63,6 +63,12 @@ class TextureCacheRuntime {
 
     StagingBufferRef UploadStagingBuffer(size_t size, bool deferred = false);
 
+    /// Reads `size` bytes of guest memory at device address `address` into `dst` (mapped staging
+    /// memory) on the worker thread, in order with the commands recorded around it: the GPU
+    /// thread hands the copy off instead of making it.
+    void DeferGuestRead(u8* dst, Tegra::MaxwellDeviceMemoryManager& memory, DAddr address,
+                        size_t size);
+
     StagingBufferRef DownloadStagingBuffer(size_t size, bool deferred = false);
 
     void FreeDeferredStagingBuffer(StagingBufferRef& ref);

src/video_core/texture_cache/texture_cache.h

@@ -2001,12 +2001,19 @@ void TextureCache<P>::UploadLayers(Image& image, std::span<const u32> layers) {
             std::vector<BufferImageCopy> copies;
             for (size_t i = 0; i < layers.size(); ++i) {
                 const GPUVAddr layer_addr = image.gpu_addr + u64{layers[i]} * image.info.layer_stride;
-                Tegra::Memory::GpuGuestMemory<u8, Tegra::Memory::GuestMemoryFlags::UnsafeRead>
-                    swizzle_data(*gpu_memory, layer_addr, image.info.layer_stride,
-                                 &swizzle_data_buffer);
-                auto layer_copies =
-                    UnswizzleImage(*gpu_memory, layer_addr, layer_info, swizzle_data,
-                                   staging.mapped_span.subspan(i * layer_size, layer_size));
+                const std::span<u8> layer_out = staging.mapped_span.subspan(i * layer_size, layer_size);
+                auto layer_copies = [&] {
+                    if (layer_info.type == ImageType::Linear) {
+                        // Read straight from guest memory; no copy into swizzle_data first.
+                        return UnswizzleLinearImage(*gpu_memory, layer_addr, layer_info,
+                                                    image.info.layer_stride, layer_out);
+                    }
+                    Tegra::Memory::GpuGuestMemory<u8, Tegra::Memory::GuestMemoryFlags::UnsafeRead>
+                        swizzle_data(*gpu_memory, layer_addr, image.info.layer_stride,
+                                     &swizzle_data_buffer);
+                    return UnswizzleImage(*gpu_memory, layer_addr, layer_info, swizzle_data,
+                                          layer_out);
+                }();
                 for (BufferImageCopy copy : layer_copies) {
                     copy.buffer_offset += i * layer_size;
                     copy.image_subresource.base_layer = static_cast<s32>(layers[i]);
@@ -2073,6 +2080,45 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging)
         return;
     }
 
+    if (image.info.type == ImageType::Linear) {
+        // A linear image is read straight from guest memory into its destination: copying it
+        // into swizzle_data first, as below, read it twice (3% of the split's back thread with
+        // Pokopia's town in view).
+        if (True(image.flags & ImageFlagBits::Converted)) {
+            unswizzle_data_buffer.resize_destructive(image.unswizzled_size_bytes);
+            auto copies = UnswizzleLinearImage(*gpu_memory, gpu_addr, image.info,
+                                               image.guest_size_bytes, unswizzle_data_buffer);
+            ConvertImage(unswizzle_data_buffer, image.info, mapped_span, copies);
+            image.UploadMemory(staging, copies);
+        } else {
+            // Pokopia rewrites three 4 MiB linear textures in turn and each is uploaded again
+            // every 3 frames: the worker thread copies guest memory into the staging buffer,
+            // right before the image copy it records, instead of this thread (~2.5% of the
+            // split's back thread). EDEN_NX2_DEFER_UPLOADS=0 copies here.
+            if constexpr (requires(u8* p) {
+                              runtime.DeferGuestRead(p, device_memory, DAddr{}, size_t{});
+                          }) {
+                static const bool defer_uploads = [] {
+                    const char* const value = std::getenv("EDEN_NX2_DEFER_UPLOADS");
+                    return value == nullptr || value[0] != '0';
+                }();
+                constexpr size_t DeferMinBytes = 64_KiB;
+                if (defer_uploads && image.guest_size_bytes >= DeferMinBytes &&
+                    False(image.flags & ImageFlagBits::Sparse) &&
+                    mapped_span.size_bytes() >= image.guest_size_bytes) {
+                    runtime.DeferGuestRead(mapped_span.data(), device_memory, image.cpu_addr,
+                                           image.guest_size_bytes);
+                    const auto copies = LinearImageCopies(image.info, image.guest_size_bytes);
+                    image.UploadMemory(staging, copies);
+                    return;
+                }
+            }
+            const auto copies = UnswizzleLinearImage(*gpu_memory, gpu_addr, image.info,
+                                                     image.guest_size_bytes, mapped_span);
+            image.UploadMemory(staging, copies);
+        }
+        return;
+    }
     Tegra::Memory::GpuGuestMemory<u8, Tegra::Memory::GuestMemoryFlags::UnsafeRead> swizzle_data(
         *gpu_memory, gpu_addr, image.guest_size_bytes, &swizzle_data_buffer);
     if (True(image.flags & ImageFlagBits::Converted)) {

src/video_core/texture_cache/util.cpp

@@ -850,6 +850,36 @@ bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config
     return gpu_memory.GpuToCpuAddress(address, guest_size_bytes).has_value();
 }
 
+boost::container::small_vector<BufferImageCopy, 16> UnswizzleLinearImage(
+    Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info,
+    size_t guest_size_bytes, std::span<u8> output) {
+    ASSERT(output.size_bytes() >= guest_size_bytes);
+    gpu_memory.ReadBlockUnsafe(gpu_addr, output.data(), guest_size_bytes);
+    return LinearImageCopies(info, guest_size_bytes);
+}
+
+boost::container::small_vector<BufferImageCopy, 16> LinearImageCopies(const ImageInfo& info,
+                                                                      size_t guest_size_bytes) {
+    const u32 bpp_log2 = BytesPerBlockLog2(info.format);
+    const Extent2D tile_size = DefaultBlockSize(info.format);
+    const Extent3D size = info.size;
+    ASSERT((info.pitch >> bpp_log2) << bpp_log2 == info.pitch);
+    return {{
+        .buffer_offset = 0,
+        .buffer_size = guest_size_bytes,
+        .buffer_row_length = info.pitch * tile_size.width >> bpp_log2,
+        .buffer_image_height = size.height,
+        .image_subresource =
+            {
+                .base_level = 0,
+                .base_layer = 0,
+                .num_layers = 1,
+            },
+        .image_offset = {0, 0, 0},
+        .image_extent = size,
+    }};
+}
+
 boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::MemoryManager& gpu_memory,
                                                                    GPUVAddr gpu_addr,
                                                                    const ImageInfo& info,
@@ -861,24 +891,7 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory
     const Extent3D size = info.size;
 
     if (info.type == ImageType::Linear) {
-        ASSERT(output.size_bytes() >= guest_size_bytes);
-        gpu_memory.ReadBlockUnsafe(gpu_addr, output.data(), guest_size_bytes);
-
-        ASSERT((info.pitch >> bpp_log2) << bpp_log2 == info.pitch);
-        return {{
-            .buffer_offset = 0,
-            .buffer_size = guest_size_bytes,
-            .buffer_row_length = info.pitch * tile_size.width >> bpp_log2,
-            .buffer_image_height = size.height,
-            .image_subresource =
-                {
-                    .base_level = 0,
-                    .base_layer = 0,
-                    .num_layers = 1,
-                },
-            .image_offset = {0, 0, 0},
-            .image_extent = size,
-        }};
+        return UnswizzleLinearImage(gpu_memory, gpu_addr, info, guest_size_bytes, output);
     }
     const LevelInfo level_info = MakeLevelInfo(info);
     const s32 num_layers = info.resources.layers;

src/video_core/texture_cache/util.h

@@ -103,6 +103,15 @@ void ExecuteUnswizzlePlan(Tegra::MemoryManager& gpu_memory, const UnswizzlePlan&
     Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info,
     std::span<const u8> input, std::span<u8> output);
 
+/// Read a linear guest image directly into the staging destination.
+/// Returns copy metadata for a linear image without reading guest memory.
+[[nodiscard]] boost::container::small_vector<BufferImageCopy, 16> LinearImageCopies(
+    const ImageInfo& info, size_t guest_size_bytes);
+
+[[nodiscard]] boost::container::small_vector<BufferImageCopy, 16> UnswizzleLinearImage(
+    Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info,
+    size_t guest_size_bytes, std::span<u8> output);
+
 /// For a single-layer 2D block-linear image of 1x1-texel blocks: the host sparse blocks
 /// (tile_width x tile_height texels) that the guest byte ranges (offset, size) touch, merged into
 /// runs along x. Levels from tail_level on (the host's mip tail) come whole.