From 8b2c5959184524117433976b6b7ee21b23113925 Mon Sep 17 00:00:00 2001 From: Brandon Bunnelle Date: Fri, 6 Nov 2026 14:04:00 -0500 Subject: [PATCH 1/6] =?UTF-8?q?puller:=20unwedge=20delayed-fence=20deadloc?= =?UTF-8?q?ks=20=E2=80=94=20flush=20the=20scheduler=20before=20forcing=20a?= =?UTF-8?q?=20release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With Fast accuracy + Balanced/Accurate fence behavior, every channel can park on a GPU semaphore whose payload rides a fence while the fence chain sits on recorded-but-unsubmitted commands (DeferFlush batching, or a back whose idle hook already ran): the 1 ms rescue's forced release then waits on a chain that never executes, and nothing recovers (OoT vistas deadlocked ~60 s in, channels wanting 0xf259/0xf26b while the counters sat 13-19 behind, buffer cache empty). When the rescued value still hasn't landed after a release, flush the scheduler first so the chain is on the GPU and the forced wait is bounded by execution. Co-Authored-By: Claude Code --- src/video_core/engines/puller.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 5b428e139a..42cc32cec7 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -118,6 +118,18 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { pending_acquire->Matches(ReadSemaphoreWord(dma_pusher, pending_acquire->address))) { return true; } + // The value still hasn't landed: flush the scheduler before forcing the release + // again. A forced release waits on the fence chain, which can sit on recorded but + // unsubmitted commands (DeferFlush's batching, or a back whose idle hook already + // ran and sleeps); waiting without a flush can wedge the back itself, and with + // every channel parked on GPU semaphores whose payloads ride those fences nothing + // ever recovers (Fast + Balanced/Accurate fences deadlocked about a minute in, + // Ocarina of Time vistas: channels wanting 0xf259/0xf26b while the GPU counter + // semaphores sat 13-19 values behind with an empty buffer cache). + VideoCommon::GpuSplit::Get().Run([rasterizer = dma_pusher.rasterizer] { + rasterizer->FlushCommands(); + rasterizer->ReleaseFences(); + }); } if (!pending_acquire->reported && std::chrono::steady_clock::now() - pending_acquire->since > std::chrono::seconds{2}) { From b0e1ba2bad21f1b9461953ad67ab02631322eebc Mon Sep 17 00:00:00 2001 From: Brandon Bunnelle Date: Fri, 6 Nov 2026 14:12:09 -0500 Subject: [PATCH 2/6] puller: satisfy a passed AcquireEqual instead of parking forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These semaphores are monotonically bumped counters, and with the emulator's looser channel ordering the writers can already be past an AcquireEqual target when the waiting channel reaches it (2026-11-06 hang: channels wanting 0x48 while the word read 0xac) — the strict-equality wait then never resolves. Hardware orders the channel against the writers; we cannot. After the wait outlives the 2 s warning, take the greater-or-equal reading the protocol implies. Co-Authored-By: Claude Code --- src/video_core/engines/puller.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 42cc32cec7..a7b0f7eeee 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -145,6 +145,22 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { dma_pusher.memory_manager.Read((pending_acquire->address & ~GPUVAddr{0xf}) + 8), dma_pusher.memory_manager.Read((pending_acquire->address & ~GPUVAddr{0xf}) + 12)); } + // A passed equality never returns. These semaphores are monotonically bumped counters, + // and with the emulator's looser channel ordering the writers can already be past an + // AcquireEqual target by the time the waiting channel reaches it (2026-11-06 hang: + // channels wanting 0x48 while the word read 0xac) — hardware orders the channel against + // the writers, we cannot. Once the wait has outlived the warning threshold, take the + // greater-or-equal reading the game's protocol implies instead of parking forever. + if (pending_acquire->reported && + static_cast(pending_acquire->operation) == + GpuSemaphoreOperation::AcquireEqual && + current > pending_acquire->value) { + LOG_INFO(HW_GPU, + "Channel {} acquire satisfied by passage: semaphore {:#x} wanted {:#x}, now {:#x}", + dma_pusher.channel_state.bind_id, pending_acquire->address, + pending_acquire->value, current); + return true; + } return false; } From 554bd5c62fbd3dbed3aabdc7612f73e744b35c95 Mon Sep 17 00:00:00 2001 From: Brandon Bunnelle Date: Fri, 6 Nov 2026 14:50:00 -0500 Subject: [PATCH 3/6] nx2, query_cache, puller, buffer_cache: semaphore write watch for the deadlock hunt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nx2SemaphoreWaiters gains a device-block watch that logs every flip-counter write with its source (fence-pop, immediate, or download). Armed from boot by EDEN_NX2_SEM_TRACE= or automatically at the puller's stuck-acquire warning — the flip-counter block sits at a stable device address (0xd6d2000) across boots, so one Balanced run that freezes shows who stops writing and when: the fence machinery, the carrier channel, or the download path. Co-Authored-By: Claude Code --- src/video_core/buffer_cache/buffer_cache.h | 10 +++++++ src/video_core/engines/puller.cpp | 6 ++++ src/video_core/nx2.h | 35 +++++++++++++++++++++- src/video_core/query_cache/query_cache.h | 5 ++-- 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e73d1cbd6c..06e2d59104 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -892,6 +892,16 @@ void BufferCache

::WriteDownloadedMemory(const Common::RangeSet& cpu_wr device_memory.WriteBlockUnsafe(device_addr, data, size); return; } + // Deadlock hunt: a download touching the watched semaphore block logs the block's new + // content (downloads are one of the three writers of the flip counters). + if (const u64 watch = Tegra::Nx2SemaphoreWaiters::Get().WatchBlock(); + watch != 0 && device_addr < watch + 0x60 && watch < device_addr + size) { + u32 words[4]{}; + device_memory.ReadBlock(watch, words, sizeof(words)); + LOG_INFO(HW_GPU, "Semaphore write {:#x} = {:#x} [source download] (+{:#x} {:#x} {:#x})", + watch, words[0], words[1], words[2], words[3]); + return; + } Common::RangeSet parts; parts.Add(device_addr, size); cpu_written.ForEachInRange(device_addr, size, [&parts](DAddr start, DAddr end) { diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index a7b0f7eeee..f9a33228bd 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -135,6 +135,12 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { std::chrono::steady_clock::now() - pending_acquire->since > std::chrono::seconds{2}) { pending_acquire->reported = true; const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address); + // Deadlock hunt: arm the semaphore write watch on this block so every later write + // (and its source) lands in the log. + Nx2SemaphoreWaiters::Get().MaybeArmWatchFromEnv(); + if (device) { + Nx2SemaphoreWaiters::Get().ArmWatch(*device); + } LOG_WARNING(HW_GPU, "Channel {} waits on semaphore {:#x} (op {}, {:#x}, now {:#x}) for 2 s; " "device {:#x}, buffer cache holds GPU data: {}, words {:08x} {:08x} {:08x} {:08x}", dma_pusher.channel_state.bind_id, pending_acquire->address, diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 92714d52d2..da22f41caf 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include "common/common_types.h" +#include "common/logging.h" namespace Tegra { @@ -385,7 +387,12 @@ class Nx2SemaphoreWaiters { } /// A GPU write of `value` to the u32 at `device_address`. - void OnWrite(u64 device_address, u32 value) { + void OnWrite(u64 device_address, u32 value, const char* source = nullptr) { + if (const u64 block = watch_block.load(std::memory_order_relaxed); + block != 0 && device_address >= block - 0x60 && device_address <= block + 0x60) { + LOG_INFO(HW_GPU, "Semaphore write {:#x} = {:#x} [source {}]", device_address, value, + source ? source : "?"); + } if (count.load(std::memory_order_acquire) == 0) { return; } @@ -397,6 +404,31 @@ class Nx2SemaphoreWaiters { } } + /// Deadlock hunt: log every write near `block` (the flip-counter semaphores sit at a + /// stable device address across boots). Armed by EDEN_NX2_SEM_TRACE= at boot or by + /// the puller's stuck-acquire warning. + void ArmWatch(u64 block) { + if (block != 0 && watch_block.load(std::memory_order_relaxed) == 0) { + watch_block.store(block, std::memory_order_relaxed); + LOG_INFO(HW_GPU, "Semaphore write watch armed at {:#x}", block); + } + } + + /// EDEN_NX2_SEM_TRACE=: arm the write watch from boot. + void MaybeArmWatchFromEnv() { + static const u64 from_env = [] { + const char* const value = std::getenv("EDEN_NX2_SEM_TRACE"); + return value ? std::strtoull(value, nullptr, 16) : u64{0}; + }(); + if (from_env != 0) { + ArmWatch(from_env); + } + } + + [[nodiscard]] u64 WatchBlock() const { + return watch_block.load(std::memory_order_relaxed); + } + private: struct Waiter { const void* owner; @@ -422,6 +454,7 @@ class Nx2SemaphoreWaiters { std::mutex mutex; std::vector waiters; std::atomic count{}; + std::atomic watch_block{}; }; } // namespace Tegra diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index 60ca50eb1f..7586a82b09 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -299,7 +299,8 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type u32 value = static_cast(query_base->value); std::memcpy(pointer, &value, sizeof(value)); } - Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, static_cast(query_base->value)); + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, static_cast(query_base->value), + "fence-pop"); if (!is_synced) [[likely]] { impl->pending_unregister.push_back(query_location); } @@ -316,7 +317,7 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type } else { std::memcpy(pointer, &payload, sizeof(payload)); } - Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, payload); + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, payload, "immediate"); streamer->Free(new_query_id); return; } From 0466dc8816511029f36953a8c7d9b352546fb2eb Mon Sep 17 00:00:00 2001 From: Brandon Bunnelle Date: Fri, 6 Nov 2026 15:21:01 -0500 Subject: [PATCH 4/6] puller: arm the EDEN_NX2_SEM_TRACE watch from the first semaphore op Co-Authored-By: Claude Code --- src/video_core/engines/puller.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index f9a33228bd..c5a56c9463 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -261,6 +261,9 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto semaphoreOperationMask = 0xF; const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask); + // Deadlock hunt: arm the EDEN_NX2_SEM_TRACE write watch on the first semaphore op, so + // the flip-counter cadence is logged from boot, not just after a stuck acquire. + Nx2SemaphoreWaiters::Get().MaybeArmWatchFromEnv(); // EDEN_NX2_SEM_WATCH=: log every release/acquire on the 16 bytes there. static const GPUVAddr watch = [] { const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); From 8305cffa0bf1f48f9cb50ccdf8002864c5c2e804 Mon Sep 17 00:00:00 2001 From: Brandon Bunnelle Date: Fri, 6 Nov 2026 15:29:50 -0500 Subject: [PATCH 5/6] puller: complete a stalled flip-counter ring round instead of deadlocking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flip-counter semaphores advance as a depth-19 ring of fence-carried releases, and every stuck wait's target is exactly current + 19 — the value the next round writes, already committed by the game. When every carrier channel parks behind such an acquire, the fence chain holds no future release and the ring stops mid-round forever (measured: writes silent from t=190.45, channels 19 behind at t=192.45; earlier hangs identical). After the 2 s warning, write the target round ourselves and notify the waiters. Co-Authored-By: Claude Code --- src/video_core/engines/puller.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index c5a56c9463..32cb977ddf 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -167,6 +167,29 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { pending_acquire->value, current); return true; } + // A ring round that never lands. The flip-counter semaphores advance in lockstep as a + // depth-19 ring of fence-carried releases, and every stuck wait's target is exactly + // current + 19: the value the next round writes, already committed by the game's + // protocol (2026-11-06 hangs: 0x1a25b at 0x1a248, 0x10d05 at 0x10cf2, 0x2657d at + // 0x2656a — all 19). When every carrier channel is itself parked behind such an + // acquire, the fence chain holds no future release and nothing ever recovers. Complete + // the round ourselves: write the target the game already queued, notify the waiters. + if (pending_acquire->reported && + static_cast(pending_acquire->operation) == + GpuSemaphoreOperation::AcquireEqual && + pending_acquire->value > current && pending_acquire->value - current <= 0x40) { + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address); + if (device) { + dma_pusher.memory_manager.Write(pending_acquire->address, pending_acquire->value); + Nx2SemaphoreWaiters::Get().OnWrite(*device, pending_acquire->value, "ring-rescue"); + LOG_INFO(HW_GPU, + "Channel {} acquire satisfied by ring rescue: semaphore {:#x} advanced " + "{:#x} -> {:#x}", + dma_pusher.channel_state.bind_id, pending_acquire->address, current, + pending_acquire->value); + return true; + } + } return false; } From df31a2f2b826c2fbc9d9eddd45c79e444bbf8951 Mon Sep 17 00:00:00 2001 From: Brandon Bunnelle Date: Fri, 6 Nov 2026 15:48:59 -0500 Subject: [PATCH 6/6] puller: rescue semaphores in milliseconds, not at the 2 s warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gating both rescues on the warning flag turned boot into one ring round per two seconds (the value passes the target immediately but the channel parks until the diagnostic threshold). Passage satisfaction now needs 10 ms of grace and ring completion 100 ms — above legitimate frame pacing, 20-200x faster than the freeze it prevents. The 2 s warning stays diagnostic. Co-Authored-By: Claude Code --- src/video_core/engines/puller.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 32cb977ddf..74f1511877 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -155,9 +155,13 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { // and with the emulator's looser channel ordering the writers can already be past an // AcquireEqual target by the time the waiting channel reaches it (2026-11-06 hang: // channels wanting 0x48 while the word read 0xac) — hardware orders the channel against - // the writers, we cannot. Once the wait has outlived the warning threshold, take the - // greater-or-equal reading the game's protocol implies instead of parking forever. - if (pending_acquire->reported && + // the writers, we cannot. A short grace period (10 ms) covers a write racing the read; + // after that the greater-or-equal reading the game's protocol implies is taken. This + // fires constantly during boot (the ring passes targets by whole rounds there), so the + // grace must stay far below the 2 s warning — gating it on the warning turned boot into + // one round per two seconds. + if (std::chrono::steady_clock::now() - pending_acquire->since >= + std::chrono::milliseconds{10} && static_cast(pending_acquire->operation) == GpuSemaphoreOperation::AcquireEqual && current > pending_acquire->value) { @@ -174,7 +178,10 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { // 0x2656a — all 19). When every carrier channel is itself parked behind such an // acquire, the fence chain holds no future release and nothing ever recovers. Complete // the round ourselves: write the target the game already queued, notify the waiters. - if (pending_acquire->reported && + // 100 ms: above normal frame pacing (a legitimate round completes within a frame or + // two), far below the seconds a real stall would cost. + if (std::chrono::steady_clock::now() - pending_acquire->since >= + std::chrono::milliseconds{100} && static_cast(pending_acquire->operation) == GpuSemaphoreOperation::AcquireEqual && pending_acquire->value > current && pending_acquire->value - current <= 0x40) {