diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp index 8d83509961..e0454a95de 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp @@ -90,8 +90,22 @@ Id GuestLane(EmitContext& ctx, Id index) { return ctx.OpBitwiseAnd(ctx.U32[1], index, ctx.Const(31U)); } +// The AMD proprietary Vulkan driver crashes while creating a pipeline that contains +// OpGroupNonUniformShuffle in a fragment shader (0xC0000005 at amdvlk64.dll+0xaff2e7 on +// 32.0.21045.x). The broadcast fallback below produces the same value without the shuffle. +bool UseShuffleFallback(EmitContext& ctx) { + if (ctx.profile.has_broken_spirv_subgroup_shuffle) { + return true; + } + static const bool fragment_fallback = [] { + const char* value = std::getenv("EDEN_NX2_FRAGMENT_SHUFFLE_FALLBACK"); + return value != nullptr && value[0] != '0'; + }(); + return fragment_fallback && ctx.stage == Stage::Fragment; +} + Id ShuffleAbsolute(EmitContext& ctx, Id value, Id src_thread_id) { - if (!ctx.profile.has_broken_spirv_subgroup_shuffle) { + if (!UseShuffleFallback(ctx)) { return ctx.OpGroupNonUniformShuffle(ctx.U32[1], SubgroupScope(ctx), value, src_thread_id); } Id result{ctx.u32_zero_value}; @@ -105,7 +119,7 @@ Id ShuffleAbsolute(EmitContext& ctx, Id value, Id src_thread_id) { } Id ShuffleRelative(EmitContext& ctx, Id value, Id delta, Id src_thread_id, spv::Op op) { - if (!ctx.profile.support_shuffle_relative) { + if (!ctx.profile.support_shuffle_relative || UseShuffleFallback(ctx)) { return ShuffleAbsolute(ctx, value, HostThreadId(ctx, src_thread_id)); } const Id result{ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, delta)}; diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp index f8f966a6b5..cd53046014 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp @@ -178,7 +178,11 @@ void DefineGenericOutput(EmitContext& ctx, size_t index, std::optional invo if (xfb_varying) { ctx.Decorate(id, spv::Decoration::XfbBuffer, xfb_varying->buffer); ctx.Decorate(id, spv::Decoration::XfbStride, xfb_varying->stride); - ctx.Decorate(id, spv::Decoration::Offset, xfb_varying->offset); + // A varying that starts inside a vector (attribute % 4 != 0) is emitted as its own + // variable with a Component decoration, so its byte offset is the vector's offset plus + // the components it skips. + ctx.Decorate(id, spv::Decoration::Offset, + xfb_varying->offset + static_cast(element) * 4U); if (ctx.stage == Stage::Geometry && xfb_varying->stream != 0) { ctx.Decorate(id, spv::Decoration::Stream, xfb_varying->stream); } diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h index 66748f2bcd..a8c0330299 100644 --- a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h +++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h @@ -62,79 +62,261 @@ struct RegisterPressure { Instruction instruction; std::vector prefix, suffix; bool valid = true; + const char* reason = "native program uses lowering scratch registers"; }; - -// Native registers at or above `reserved` (the lowering's uniform mirrors and scratch) live in -// bank 1. For one instruction, each aligned quartet of them it uses is copied into a low quartet -// the instruction doesn't use (whose own values wait in bank 2), and copied back afterwards. -// The copies are SSA renames: they cost nothing in the host shader. +// Preserve high guest registers in SSA while borrowing aligned low quartets +// for one native instruction. The expanded instruction owns its temporary bank. inline RegisterPressure PrepareRegisterPressure(const Instruction& input, u32 reserved) { RegisterPressure result{input}; - struct Operand { u32 bit, count; }; + if (reserved < 4 || reserved > 246 || (reserved & 3)) { + result.valid = false; result.reason = "native scratch: reserved boundary"; + return result; + } + struct Operand { + u32 bit, count; + }; std::vector operands; const auto operand = [&](u32 bit, u32 count = 1) { operands.push_back({bit, count}); }; const u32 opcode = input.Bits(0, 12); switch (opcode) { - case 0x802: case 0xa02: case 0xb08: case 0xd08: operand(16); break; - case 0x804: operand(24); break; - case 0x805: operand(16, 2); break; - case 0x308: operand(16); operand(32); break; - case 0x433: case 0x633: operand(16); operand(24); break; - case 0x233: operand(16); operand(24); operand(32); break; - case 0x812: case 0x824: case 0x831: case 0x424: case 0x624: - operand(16); operand(24); operand(64); break; - case 0x230: case 0x231: case 0x211: case 0x224: - operand(16); operand(24); operand(32); operand(64); break; - case 0x819: case 0xa11: - operand(16); operand(24); operand(64); break; - case 0x225: case 0x625: case 0x825: - operand(16, 2); operand(24); - if (opcode == 0x225) { operand(32); operand(64, 2); } - if (opcode == 0x625) operand(64); - if (opcode == 0x825) operand(64, 2); + case 0xa02: + case 0xc3e: + case 0x802: + case 0xb08: + case 0xd08: + case 0x908: + case 0x919: + operand(16); + break; + case 0xa0c: + case 0x804: + case 0x80b: + case 0x80c: + operand(24); + break; + case 0x20c: + case 0x20b: + operand(24); + operand(32); + break; + case 0x807: + case 0x808: + case 0x923: + case 0x820: + case 0xa20: + case 0x421: + operand(16); + operand(24); + break; + case 0x207: + case 0x208: + case 0x220: + case 0x221: + case 0x209: + operand(16); + operand(24); + operand(32); + break; + case 0x805: + operand(16, 2); + break; + case 0xc17: + case 0x817: + operand(16); + operand(24); + break; + case 0xa24: + case 0xa23: + case 0x623: + case 0x423: + operand(16); + operand(24); + operand(64); + break; + case 0x621: + operand(16); + operand(24); + break; + case 0x321: + operand(16, input.Bits(74, 2) + 1); + operand(24); + operand(32); + break; + case 0x322: + operand(24); + operand(32, input.Bits(74, 2) + 1); + break; + case 0x927: + operand(24); + operand(32); + break; + case 0x224: + case 0x223: + operand(16); + operand(24); + operand(32); + operand(64); + break; + case 0x81a: + operand(16); + operand(24); + break; + case 0x308: + case 0x202: + case 0x243: + case 0x245: + case 0x305: + case 0x306: + case 0x307: + operand(16); + operand(32); + break; + case 0xb05: + case 0xd05: + operand(16); + break; + case 0x59f: + operand(24, 2); + operand(32); + break; + case 0x98e: + operand(24, input.Bits(72, 1) ? 2 : 1); + operand(32, input.Bits(73, 3) == 2 || input.Bits(73, 3) == 5 ? 2 : 1); + break; + case 0x433: + case 0x633: + operand(16); + operand(24); + break; + case 0x233: + operand(16); + operand(24); + operand(32); + break; + case 0x810: + case 0xa10: + case 0xc10: + case 0x812: + case 0x824: + case 0x823: + case 0x831: + case 0x424: + case 0x624: + operand(16); + operand(24); + operand(64); + break; + case 0x230: + case 0x231: + operand(16); + operand(24); + operand(32); + operand(64); + break; + case 0x419: + case 0x819: + operand(16); + operand(24); + operand(64); + break; + case 0x210: + case 0x211: + case 0x212: + operand(16); + operand(24); + operand(32); + operand(64); + break; + case 0xa11: + operand(16); + operand(24); + operand(64); + break; + case 0x225: + case 0x227: + case 0x625: + case 0x825: + operand(16, opcode == 0x227 ? 1 : 2); + operand(24); + if (opcode == 0x225 || opcode == 0x227) { + operand(32); + operand(64, 2); + } + if (opcode == 0x625) + operand(64); + if (opcode == 0x825) + operand(64, 2); break; case 0x83b: - operand(16, 1U << input.Bits(72, 2)); operand(24); break; - case 0x23c: { - const bool deep = input.Bits(75, 1) != 0; // .16816 - operand(16, 2); operand(24, deep ? 4 : 2); operand(32, deep ? 2 : 1); operand(64, 2); + operand(16, 1U << input.Bits(72, 2)); + operand(24); + break; + case 0x23c: + operand(16, 2); + operand(24, input.Bits(75, 1) ? 4 : 2); + operand(32, input.Bits(75, 1) ? 2 : 1); + operand(64, 2); + break; + case 0x388: + case 0x984: + case 0x988: { + const u32 width = input.Bits(73, 3), count = width == 6 ? 4 : width == 5 ? 2 : 1; + operand(opcode == 0x984 ? 16 : 32, count); + operand(24); break; } - case 0x388: case 0x984: case 0x988: { + case 0x387: + case 0x983: + case 0x987: { const u32 width = input.Bits(73, 3), count = width == 6 ? 4 : width == 5 ? 2 : 1; - operand(opcode == 0x984 ? 16 : 32, count); operand(24); break; + operand(opcode == 0x983 ? 16 : 32, count); + operand(24); + break; } - case 0x387: case 0x983: - operand(opcode == 0x387 ? 32 : 16); operand(24); break; - case 0x981: case 0x986: { + case 0x381: + case 0x386: + case 0x981: + case 0x986: { const u32 width = input.Bits(73, 3), count = width == 6 ? 4 : width == 5 ? 2 : 1; - operand(opcode == 0x981 ? 16 : 32, count); - operand(24, input.Bits(72, 1) ? 2 : 1); break; + operand(opcode == 0x386 || opcode == 0x986 ? 32 : 16, count); + operand(24, opcode == 0x381 || opcode == 0x386 || input.Bits(90, 1) ? 2 : 1); + break; } default: - result.valid = false; return result; + result.valid = false; result.reason = "native scratch: opcode not in shadow table"; + return result; } std::array used{}; std::array groups{}; for (const auto [bit, count] : operands) { const u32 reg = input.Bits(bit, 8); - if (reg == 255) continue; - if (reg + count > 255) { result.valid = false; return result; } + if (reg == 255) + continue; + if (reg + count > 255 || + (reg < reserved && reg + count > reserved) || + (reg >= reserved && reg / 4 != (reg + count - 1) / 4)) { + result.valid = false; result.reason = "native scratch: operand range"; + return result; + } for (u32 component = 0; component < count; ++component) { used[reg + component] = true; - if (reg + component >= reserved) groups[(reg + component) / 4] = true; + if (reg + component >= reserved) + groups[(reg + component) / 4] = true; } } std::array mapped{}; for (u32 group = 0; group < groups.size(); ++group) { - if (!groups[group]) continue; + if (!groups[group]) + continue; u32 borrow = reserved; while (borrow >= 4) { borrow -= 4; - if (!used[borrow] && !used[borrow + 1] && !used[borrow + 2] && !used[borrow + 3]) break; + if (!used[borrow] && !used[borrow + 1] && !used[borrow + 2] && !used[borrow + 3]) + break; } if (used[borrow] || used[borrow + 1] || used[borrow + 2] || used[borrow + 3]) { - result.valid = false; return result; + result.valid = false; result.reason = "native scratch: no free quartet"; + return result; } mapped[group] = borrow; for (u32 component = 0; component < 4; ++component) { @@ -148,10 +330,12 @@ inline RegisterPressure PrepareRegisterPressure(const Instruction& input, u32 re } for (const auto [bit, count] : operands) { const u32 reg = input.Bits(bit, 8); - if (reg == 255 || !groups[reg / 4]) continue; + if (reg == 255 || !groups[reg / 4]) + continue; const u32 replacement = mapped[reg / 4] + reg % 4; const u32 word = bit / 32, shift = bit % 32; - result.instruction.words[word] = (result.instruction.words[word] & ~(255U << shift)) | (replacement << shift); + result.instruction.words[word] = + (result.instruction.words[word] & ~(255U << shift)) | (replacement << shift); } return result; } @@ -237,14 +421,17 @@ inline LoweredProgram LowerToMaxwellWith(std::span code, u32 RegisterPressure pressure{insn}; bool requires_shadow = false; for (u32 bit : {16U, 24U, 32U, 64U}) { + // Bits 32-39 name a register in register forms and in these opcodes: TEX/TLD, the + // stores, and the data operand of ISBEWR (0x927), RED (0x98e) and SURED (0x59f), all + // listed with operand 32 in RegisterPressure. if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66 && opcode != 0xf66 && - opcode != 0x988) continue; + opcode != 0x988 && opcode != 0x927 && opcode != 0x98e && opcode != 0x59f) continue; const u32 reg = insn.Bits(bit, 8); if (reg >= shadow_boundary && reg < 255) requires_shadow = true; } if (requires_shadow) { pressure = PrepareRegisterPressure(insn, shadow_boundary); - if (!pressure.valid) return fail(index, "native program uses lowering scratch registers"); + if (!pressure.valid) return fail(index, pressure.reason); insn = pressure.instruction; result.instructions.insert(result.instructions.end(), pressure.prefix.begin(), pressure.prefix.end()); } @@ -2168,7 +2355,9 @@ inline LoweredProgram LowerToMaxwellWith(std::span code, u32 // Bit 76: a texel offset after the coordinates (x, y in Ra, Ra+1, offsets in Rb, as // for TEX); Maxwell TLD4 takes it as AOFFI (bits 54-55) from Rb. const bool offsets = insn.Bits(76, 1) != 0; - if (insn.Bits(61, 3) != 1 || insn.Bits(77, 3) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || + if ((insn.Bits(61, 3) != 1 && insn.Bits(61, 3) != 5) || + (insn.Bits(77, 3) != 0 && insn.Bits(77, 3) != 2) || insn.Bits(81, 3) != 7 || + insn.Bits(59, 1) || (insn.Bits(61, 3) == 5 && offsets) || (offsets && (bindless || source_b == 255 || (source_a != 255 && source_a >= 245))) || insn.Bits(89, 1) || mask == 0 || (bindless ? (handle >= 246 || insn.Bits(46, 13) || !insn.Bits(91, 1)) @@ -2178,20 +2367,25 @@ inline LoweredProgram LowerToMaxwellWith(std::span code, u32 if (!bindless) result.required_texture_buffer = binding; // Two coordinates: x in Ra, y in Rb (the coordinate list splits evenly across them); // with an offset, x and y in Ra, Ra+1 and the offset in Rb. - const u32 y = !offsets ? source_b : source_a == 255 ? 255U : source_a + 1; + const bool array = insn.Bits(61, 3) == 5; + const u32 y = !offsets ? (array ? (source_a == 255 ? 255U : source_a + 1) : source_b) + : source_a == 255 ? 255U : source_a + 1; emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20)); emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{y} << 20)); + if (array) { + emit(0x5c98000000000000ULL | predicate | 252 | (u64{15} << 39) | (u64{source_b} << 20)); + } if (offsets) { emit(0x5c98000000000000ULL | predicate | 246 | (u64{15} << 39) | (u64{source_b} << 20)); } if (bindless) { emit(0x5c98000000000000ULL | predicate | 246 | (u64{15} << 39) | (u64{handle} << 20)); emit(0xdec0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{246} << 20) | - (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(87, 2)} << 38) | + (u64{array ? 3U : 2U} << 28) | (u64{mask} << 31) | (u64{insn.Bits(87, 2)} << 38) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); } else { emit(0xc800000000000000ULL | predicate | 250 | (u64{250} << 8) | - (u64{offsets ? 246U : 255U} << 20) | (u64{2} << 28) | (u64{mask} << 31) | + (u64{offsets ? 246U : 255U} << 20) | (u64{array ? 3U : 2U} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{offsets ? 1U : 0U} << 54) | (u64{insn.Bits(87, 2)} << 56)); } @@ -2590,11 +2784,12 @@ inline LoweredProgram LowerToMaxwellWith(std::span code, u32 (u64{insn.Bits(76, 2)} << 52) | (u64{insn.Bits(78, 1)} << 55)); break; } - case 0x81a: { // SGXT.U32 Rd, Ra, n: keep the low n bits (BFE.U32 at position 0) - if (insn.words[2] || insn.words[1] > 32 || dest >= 255) - return fail(index, "native sign extend mode"); + case 0x81a: { // SGXT.U32/S32 Rd, Ra, n: keep the low n bits, sign flag in bit 73. + if (insn.words[2] & ~0x200U) return fail(index, "sign extend mode w2"); + if (insn.words[1] > 32) return fail(index, "sign extend mode size"); + if (dest >= 255) return fail(index, "sign extend mode dest"); emit(base(0x3800000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | - (u64{insn.words[1] << 8} << 20)); + (u64{insn.words[1] << 8} << 20) | (u64{insn.Bits(73, 1)} << 48)); break; } case 0x299: { // USHF by a uniform amount, one register shifted (SHL/SHR, bit 75 wraps). @@ -2714,7 +2909,8 @@ inline LoweredProgram LowerToMaxwellWith(std::span code, u32 const u32 compare = insn.Bits(76, 3); const u32 destination = insn.Bits(81, 3); const u32 low = insn.Bits(68, 4); - if (insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7 || insn.Bits(74, 2) != 0 || + const u32 tail_pred = insn.Bits(87, 3) != 7 ? insn.Bits(87, 4) : destination; + if (insn.Bits(84, 3) != 7 || insn.Bits(74, 2) != 0 || compare == 0 || compare == 7 || (destination != 7 && insn.Bits(12, 3) == destination)) return fail(index, "native extended integer comparison mode"); @@ -2732,7 +2928,7 @@ inline LoweredProgram LowerToMaxwellWith(std::span code, u32 emit(isetp(Ne, low, Or)); } else { emit(isetp(Eq, low, And)); - emit(isetp(compare <= 3 ? Lt : Gt, destination, Or)); + emit(isetp(compare <= 3 ? Lt : Gt, tail_pred, Or)); } break; } diff --git a/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp b/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp index 50cff52f88..8733ba1fa5 100644 --- a/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp +++ b/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp @@ -893,6 +893,7 @@ class TranslatePass { std::vector demote_blocks; std::vector demote_conds; u32 num_epilogues{}; + u32 demotes_with_code{}; u32 branch_depth{}; for (const IR::AbstractSyntaxNode& node : syntax_list) { if (node.type == Type::If) { @@ -908,18 +909,26 @@ class TranslatePass { // Skip reordering nested demote branches. continue; } + size_t insts_before{}; for (const IR::Inst& inst : node.data.block->Instructions()) { const IR::Opcode op{inst.GetOpcode()}; if (op == IR::Opcode::DemoteToHelperInvocation) { + if (insts_before != 0) { + // The block moves as a whole, but its CFG edges and the phis of its old + // merge block still name it. The AMD proprietary driver turns that into + // black output for the whole draw (Ocarina of Time on Windows). + ++demotes_with_code; + } demote_blocks.push_back(node.data.block); break; } if (op == IR::Opcode::Epilogue) { ++num_epilogues; } + ++insts_before; } } - if (demote_blocks.size() == 0) { + if (demote_blocks.size() == 0 || demotes_with_code != 0) { return; } if (num_epilogues > 1) { diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index d9f0ac2837..d1f4949c69 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -152,6 +152,10 @@ class FenceManager { wait_cv.notify_all(); }); SignalFence(std::move(func)); + // The fence's commands can be held back by a deferred flush that only this thread + // submits. The fencing thread waits for that submission before releasing the fence, + // so waiting here without submitting deadlocks (Ocarina of Time entering gameplay). + SubmitDeferredWork(); std::unique_lock lk(wait_mutex); wait_cv.wait( lk, [&wait_finished] { return wait_finished.load(std::memory_order_relaxed); }); @@ -187,6 +191,8 @@ class FenceManager { virtual bool IsFenceSignaled(TFence& fence) const = 0; /// Waits until a fence has been signalled by the host GPU. virtual void WaitFence(TFence& fence) = 0; + /// Submits work whose submission the backend deferred (called before blocking on a fence). + virtual void SubmitDeferredWork() {} VideoCore::RasterizerInterface& rasterizer; Tegra::GPU& gpu; diff --git a/src/video_core/renderer_vulkan/vk_fence_manager.cpp b/src/video_core/renderer_vulkan/vk_fence_manager.cpp index b22bc32e9e..43e8b2b5cd 100644 --- a/src/video_core/renderer_vulkan/vk_fence_manager.cpp +++ b/src/video_core/renderer_vulkan/vk_fence_manager.cpp @@ -74,4 +74,8 @@ void FenceManager::WaitFence(Fence& fence) { fence->Wait(); } +void FenceManager::SubmitDeferredWork() { + scheduler.FlushDeferred(); +} + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_fence_manager.h b/src/video_core/renderer_vulkan/vk_fence_manager.h index 3365735743..606cfd65b9 100644 --- a/src/video_core/renderer_vulkan/vk_fence_manager.h +++ b/src/video_core/renderer_vulkan/vk_fence_manager.h @@ -62,6 +62,7 @@ class FenceManager final : public GenericFenceManager { void QueueFence(Fence& fence) override; bool IsFenceSignaled(Fence& fence) const override; void WaitFence(Fence& fence) override; + void SubmitDeferredWork() override; private: Scheduler& scheduler; diff --git a/src/video_core/transform_feedback.cpp b/src/video_core/transform_feedback.cpp index 53d29c08e2..c80457e336 100644 --- a/src/video_core/transform_feedback.cpp +++ b/src/video_core/transform_feedback.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include "common/alignment.h" @@ -67,6 +68,11 @@ std::pair, u32> MakeTransformF }; std::array xfb{}; u32 count{0}; + // Diagnostic switch: drop transform feedback entirely, to prove whether a stall on the GPU + // fence comes from the unaligned varying layout below. + if (std::getenv("EDEN_NX2_NO_TFB") != nullptr) { + return {xfb, 0}; + } for (size_t buffer = 0; buffer < state.layouts.size(); ++buffer) { const auto& locations = state.varyings[buffer]; const auto& layout = state.layouts[buffer];