diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e73d1cbd6c..06e2d59104 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -892,6 +892,16 @@ void BufferCache

::WriteDownloadedMemory(const Common::RangeSet& cpu_wr device_memory.WriteBlockUnsafe(device_addr, data, size); return; } + // Deadlock hunt: a download touching the watched semaphore block logs the block's new + // content (downloads are one of the three writers of the flip counters). + if (const u64 watch = Tegra::Nx2SemaphoreWaiters::Get().WatchBlock(); + watch != 0 && device_addr < watch + 0x60 && watch < device_addr + size) { + u32 words[4]{}; + device_memory.ReadBlock(watch, words, sizeof(words)); + LOG_INFO(HW_GPU, "Semaphore write {:#x} = {:#x} [source download] (+{:#x} {:#x} {:#x})", + watch, words[0], words[1], words[2], words[3]); + return; + } Common::RangeSet parts; parts.Add(device_addr, size); cpu_written.ForEachInRange(device_addr, size, [&parts](DAddr start, DAddr end) { diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 5b428e139a..74f1511877 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -118,11 +118,29 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { pending_acquire->Matches(ReadSemaphoreWord(dma_pusher, pending_acquire->address))) { return true; } + // The value still hasn't landed: flush the scheduler before forcing the release + // again. A forced release waits on the fence chain, which can sit on recorded but + // unsubmitted commands (DeferFlush's batching, or a back whose idle hook already + // ran and sleeps); waiting without a flush can wedge the back itself, and with + // every channel parked on GPU semaphores whose payloads ride those fences nothing + // ever recovers (Fast + Balanced/Accurate fences deadlocked about a minute in, + // Ocarina of Time vistas: channels wanting 0xf259/0xf26b while the GPU counter + // semaphores sat 13-19 values behind with an empty buffer cache). + VideoCommon::GpuSplit::Get().Run([rasterizer = dma_pusher.rasterizer] { + rasterizer->FlushCommands(); + rasterizer->ReleaseFences(); + }); } if (!pending_acquire->reported && std::chrono::steady_clock::now() - pending_acquire->since > std::chrono::seconds{2}) { pending_acquire->reported = true; const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address); + // Deadlock hunt: arm the semaphore write watch on this block so every later write + // (and its source) lands in the log. + Nx2SemaphoreWaiters::Get().MaybeArmWatchFromEnv(); + if (device) { + Nx2SemaphoreWaiters::Get().ArmWatch(*device); + } LOG_WARNING(HW_GPU, "Channel {} waits on semaphore {:#x} (op {}, {:#x}, now {:#x}) for 2 s; " "device {:#x}, buffer cache holds GPU data: {}, words {:08x} {:08x} {:08x} {:08x}", dma_pusher.channel_state.bind_id, pending_acquire->address, @@ -133,6 +151,52 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { dma_pusher.memory_manager.Read((pending_acquire->address & ~GPUVAddr{0xf}) + 8), dma_pusher.memory_manager.Read((pending_acquire->address & ~GPUVAddr{0xf}) + 12)); } + // A passed equality never returns. These semaphores are monotonically bumped counters, + // and with the emulator's looser channel ordering the writers can already be past an + // AcquireEqual target by the time the waiting channel reaches it (2026-11-06 hang: + // channels wanting 0x48 while the word read 0xac) — hardware orders the channel against + // the writers, we cannot. A short grace period (10 ms) covers a write racing the read; + // after that the greater-or-equal reading the game's protocol implies is taken. This + // fires constantly during boot (the ring passes targets by whole rounds there), so the + // grace must stay far below the 2 s warning — gating it on the warning turned boot into + // one round per two seconds. + if (std::chrono::steady_clock::now() - pending_acquire->since >= + std::chrono::milliseconds{10} && + static_cast(pending_acquire->operation) == + GpuSemaphoreOperation::AcquireEqual && + current > pending_acquire->value) { + LOG_INFO(HW_GPU, + "Channel {} acquire satisfied by passage: semaphore {:#x} wanted {:#x}, now {:#x}", + dma_pusher.channel_state.bind_id, pending_acquire->address, + pending_acquire->value, current); + return true; + } + // A ring round that never lands. The flip-counter semaphores advance in lockstep as a + // depth-19 ring of fence-carried releases, and every stuck wait's target is exactly + // current + 19: the value the next round writes, already committed by the game's + // protocol (2026-11-06 hangs: 0x1a25b at 0x1a248, 0x10d05 at 0x10cf2, 0x2657d at + // 0x2656a — all 19). When every carrier channel is itself parked behind such an + // acquire, the fence chain holds no future release and nothing ever recovers. Complete + // the round ourselves: write the target the game already queued, notify the waiters. + // 100 ms: above normal frame pacing (a legitimate round completes within a frame or + // two), far below the seconds a real stall would cost. + if (std::chrono::steady_clock::now() - pending_acquire->since >= + std::chrono::milliseconds{100} && + static_cast(pending_acquire->operation) == + GpuSemaphoreOperation::AcquireEqual && + pending_acquire->value > current && pending_acquire->value - current <= 0x40) { + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address); + if (device) { + dma_pusher.memory_manager.Write(pending_acquire->address, pending_acquire->value); + Nx2SemaphoreWaiters::Get().OnWrite(*device, pending_acquire->value, "ring-rescue"); + LOG_INFO(HW_GPU, + "Channel {} acquire satisfied by ring rescue: semaphore {:#x} advanced " + "{:#x} -> {:#x}", + dma_pusher.channel_state.bind_id, pending_acquire->address, current, + pending_acquire->value); + return true; + } + } return false; } @@ -227,6 +291,9 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto semaphoreOperationMask = 0xF; const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask); + // Deadlock hunt: arm the EDEN_NX2_SEM_TRACE write watch on the first semaphore op, so + // the flip-counter cadence is logged from boot, not just after a stuck acquire. + Nx2SemaphoreWaiters::Get().MaybeArmWatchFromEnv(); // EDEN_NX2_SEM_WATCH=: log every release/acquire on the 16 bytes there. static const GPUVAddr watch = [] { const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 92714d52d2..da22f41caf 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include "common/common_types.h" +#include "common/logging.h" namespace Tegra { @@ -385,7 +387,12 @@ class Nx2SemaphoreWaiters { } /// A GPU write of `value` to the u32 at `device_address`. - void OnWrite(u64 device_address, u32 value) { + void OnWrite(u64 device_address, u32 value, const char* source = nullptr) { + if (const u64 block = watch_block.load(std::memory_order_relaxed); + block != 0 && device_address >= block - 0x60 && device_address <= block + 0x60) { + LOG_INFO(HW_GPU, "Semaphore write {:#x} = {:#x} [source {}]", device_address, value, + source ? source : "?"); + } if (count.load(std::memory_order_acquire) == 0) { return; } @@ -397,6 +404,31 @@ class Nx2SemaphoreWaiters { } } + /// Deadlock hunt: log every write near `block` (the flip-counter semaphores sit at a + /// stable device address across boots). Armed by EDEN_NX2_SEM_TRACE= at boot or by + /// the puller's stuck-acquire warning. + void ArmWatch(u64 block) { + if (block != 0 && watch_block.load(std::memory_order_relaxed) == 0) { + watch_block.store(block, std::memory_order_relaxed); + LOG_INFO(HW_GPU, "Semaphore write watch armed at {:#x}", block); + } + } + + /// EDEN_NX2_SEM_TRACE=: arm the write watch from boot. + void MaybeArmWatchFromEnv() { + static const u64 from_env = [] { + const char* const value = std::getenv("EDEN_NX2_SEM_TRACE"); + return value ? std::strtoull(value, nullptr, 16) : u64{0}; + }(); + if (from_env != 0) { + ArmWatch(from_env); + } + } + + [[nodiscard]] u64 WatchBlock() const { + return watch_block.load(std::memory_order_relaxed); + } + private: struct Waiter { const void* owner; @@ -422,6 +454,7 @@ class Nx2SemaphoreWaiters { std::mutex mutex; std::vector waiters; std::atomic count{}; + std::atomic watch_block{}; }; } // namespace Tegra diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index 60ca50eb1f..7586a82b09 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -299,7 +299,8 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type u32 value = static_cast(query_base->value); std::memcpy(pointer, &value, sizeof(value)); } - Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, static_cast(query_base->value)); + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, static_cast(query_base->value), + "fence-pop"); if (!is_synced) [[likely]] { impl->pending_unregister.push_back(query_location); } @@ -316,7 +317,7 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type } else { std::memcpy(pointer, &payload, sizeof(payload)); } - Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, payload); + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, payload, "immediate"); streamer->Free(new_query_id); return; }