Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 32 additions & 24 deletions src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,14 @@ Id StorageAtomicU32x2(EmitContext& ctx, const IR::Value& binding, const IR::Valu
ctx.OpStore(pointer, result);
return original_value;
}

Id GlobalAtomic(EmitContext& ctx, EmitContext::GlobalAtomic kind, Id type, Id address, Id value) {
const Id func{ctx.global_atomic_funcs[static_cast<size_t>(kind)]};
if (!Sirit::ValidId(func)) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(type, func, address, value);
}
} // Anonymous namespace

Id EmitSharedAtomicIAdd32(EmitContext& ctx, Id offset, Id value) {
Expand Down Expand Up @@ -417,48 +425,48 @@ Id EmitStorageAtomicMaxF32x2(EmitContext& ctx, const IR::Value& binding, const I
return ctx.OpPackHalf2x16(ctx.U32[1], result);
}

Id EmitGlobalAtomicIAdd32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicIAdd32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::IAdd, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicSMin32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicSMin32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::SMin, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicUMin32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicUMin32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::UMin, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicSMax32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicSMax32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::SMax, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicUMax32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicUMax32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::UMax, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicInc32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicInc32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::Inc, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicDec32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicDec32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::Dec, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicAnd32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicAnd32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::And, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicOr32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicOr32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::Or, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicXor32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicXor32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::Xor, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicExchange32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicExchange32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::Exchange, ctx.U32[1], address, value);
}

Id EmitGlobalAtomicIAdd64(EmitContext&) {
Expand Down Expand Up @@ -549,8 +557,8 @@ Id EmitGlobalAtomicExchange32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}

Id EmitGlobalAtomicAddF32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
Id EmitGlobalAtomicAddF32(EmitContext& ctx, Id address, Id value) {
return GlobalAtomic(ctx, EmitContext::GlobalAtomic::AddF32, ctx.F32[1], address, value);
}

Id EmitGlobalAtomicAddF16x2(EmitContext&) {
Expand Down
24 changes: 12 additions & 12 deletions src/shader_recompiler/backend/spirv/emit_spirv_instructions.h
Original file line number Diff line number Diff line change
Expand Up @@ -422,17 +422,17 @@ Id EmitStorageAtomicMaxF16x2(EmitContext& ctx, const IR::Value& binding, const I
Id value);
Id EmitStorageAtomicMaxF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitGlobalAtomicIAdd32(EmitContext& ctx);
Id EmitGlobalAtomicSMin32(EmitContext& ctx);
Id EmitGlobalAtomicUMin32(EmitContext& ctx);
Id EmitGlobalAtomicSMax32(EmitContext& ctx);
Id EmitGlobalAtomicUMax32(EmitContext& ctx);
Id EmitGlobalAtomicInc32(EmitContext& ctx);
Id EmitGlobalAtomicDec32(EmitContext& ctx);
Id EmitGlobalAtomicAnd32(EmitContext& ctx);
Id EmitGlobalAtomicOr32(EmitContext& ctx);
Id EmitGlobalAtomicXor32(EmitContext& ctx);
Id EmitGlobalAtomicExchange32(EmitContext& ctx);
Id EmitGlobalAtomicIAdd32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicSMin32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicUMin32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicSMax32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicUMax32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicInc32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicDec32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicAnd32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicOr32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicXor32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicExchange32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicIAdd64(EmitContext& ctx);
Id EmitGlobalAtomicSMin64(EmitContext& ctx);
Id EmitGlobalAtomicUMin64(EmitContext& ctx);
Expand All @@ -455,7 +455,7 @@ Id EmitGlobalAtomicAnd32x2(EmitContext& ctx);
Id EmitGlobalAtomicOr32x2(EmitContext& ctx);
Id EmitGlobalAtomicXor32x2(EmitContext& ctx);
Id EmitGlobalAtomicExchange32x2(EmitContext& ctx);
Id EmitGlobalAtomicAddF32(EmitContext& ctx);
Id EmitGlobalAtomicAddF32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicAddF16x2(EmitContext& ctx);
Id EmitGlobalAtomicAddF32x2(EmitContext& ctx);
Id EmitGlobalAtomicMinF16x2(EmitContext& ctx);
Expand Down
83 changes: 78 additions & 5 deletions src/shader_recompiler/backend/spirv/spirv_emit_context.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include <array>
#include <bit>
#include <climits>
#include <optional>

#include <boost/container/static_vector.hpp>

Expand Down Expand Up @@ -493,7 +494,7 @@ EmitContext::EmitContext(const Profile& profile_, const RuntimeInfo& runtime_inf
DefineImages(program.info, image_binding, bindings.image_scaling_index);
DefineAttributeMemAccess(program.info);
DefineWriteStorageCasLoopFunction(program.info);
DefineGlobalMemoryFunctions(program.info);
DefineGlobalMemoryFunctions(program);
DefineRescalingInput(program.info);
DefineRenderArea(program.info);
}
Expand Down Expand Up @@ -948,7 +949,8 @@ void EmitContext::DefineWriteStorageCasLoopFunction(const Info& info) {
write_storage_cas_loop_func = func;
}

void EmitContext::DefineGlobalMemoryFunctions(const Info& info) {
void EmitContext::DefineGlobalMemoryFunctions(const IR::Program& program) {
const Info& info{program.info};
if (!info.uses_global_memory || !profile.support_int64) {
return;
}
Expand Down Expand Up @@ -977,7 +979,7 @@ void EmitContext::DefineGlobalMemoryFunctions(const Info& info) {
const Id ssbo_offset{OpUConvert(U32[1], OpISub(U64, addr, ssbo_addr))};
const Id ssbo_index{OpShiftRightLogical(U32[1], ssbo_offset, Const(shift))};
const Id ssbo_pointer{OpAccessChain(element_pointer, ssbo_id, zero, ssbo_index)};
callback(ssbo_pointer);
callback(ssbo_pointer, ssbo_id, ssbo_index);
AddLabel(else_label);
}};
if (info.native_global_bounds) {
Expand Down Expand Up @@ -1021,7 +1023,7 @@ void EmitContext::DefineGlobalMemoryFunctions(const Info& info) {
const Id func_id{OpFunction(type, spv::FunctionControlMask::MaskNone, function_type)};
const Id addr{OpFunctionParameter(U64)};
define_body(ssbo_member, addr, element_pointer, shift,
[&](Id ssbo_pointer) { OpReturnValue(OpLoad(type, ssbo_pointer)); });
[&](Id ssbo_pointer, Id, Id) { OpReturnValue(OpLoad(type, ssbo_pointer)); });
OpReturnValue(ConstantNull(type));
OpFunctionEnd();
return func_id;
Expand All @@ -1031,7 +1033,7 @@ void EmitContext::DefineGlobalMemoryFunctions(const Info& info) {
const Id func_id{OpFunction(void_id, spv::FunctionControlMask::MaskNone, function_type)};
const Id addr{OpFunctionParameter(U64)};
const Id data{OpFunctionParameter(type)};
define_body(ssbo_member, addr, element_pointer, shift, [&](Id ssbo_pointer) {
define_body(ssbo_member, addr, element_pointer, shift, [&](Id ssbo_pointer, Id, Id) {
OpStore(ssbo_pointer, data);
OpReturn();
});
Expand All @@ -1053,6 +1055,77 @@ void EmitContext::DefineGlobalMemoryFunctions(const Info& info) {
define(&StorageDefinitions::U32x2, storage_types.U32x2, U32[2], sizeof(u32[2]));
std::tie(load_global_func_u32x4, write_global_func_u32x4) =
define(&StorageDefinitions::U32x4, storage_types.U32x4, U32[4], sizeof(u32[4]));

// Atomics the storage buffer pass could not track: the same routing, with the operation on
// the routed word (INC, DEC and F32 add through their storage buffer CAS loops).
using enum GlobalAtomic;
const auto kind_of = [](IR::Opcode opcode) -> std::optional<GlobalAtomic> {
switch (opcode) {
case IR::Opcode::GlobalAtomicIAdd32: return IAdd;
case IR::Opcode::GlobalAtomicSMin32: return SMin;
case IR::Opcode::GlobalAtomicUMin32: return UMin;
case IR::Opcode::GlobalAtomicSMax32: return SMax;
case IR::Opcode::GlobalAtomicUMax32: return UMax;
case IR::Opcode::GlobalAtomicInc32: return Inc;
case IR::Opcode::GlobalAtomicDec32: return Dec;
case IR::Opcode::GlobalAtomicAnd32: return And;
case IR::Opcode::GlobalAtomicOr32: return Or;
case IR::Opcode::GlobalAtomicXor32: return Xor;
case IR::Opcode::GlobalAtomicExchange32: return Exchange;
case IR::Opcode::GlobalAtomicAddF32: return AddF32;
default: return std::nullopt;
}
};
std::array<bool, static_cast<size_t>(Count)> used{};
for (const IR::Block* const block : program.blocks) {
for (const IR::Inst& inst : block->Instructions()) {
if (const auto kind = kind_of(inst.GetOpcode())) {
used[static_cast<size_t>(*kind)] = true;
}
}
}
const Id scope{Const(static_cast<u32>(spv::Scope::Device))};
for (size_t index = 0; index < used.size(); ++index) {
if (!used[index]) {
continue;
}
const auto kind = static_cast<GlobalAtomic>(index);
const Id type{kind == AddF32 ? F32[1] : U32[1]};
const Id func_id{OpFunction(type, spv::FunctionControlMask::MaskNone,
TypeFunction(type, U64, type))};
const Id addr{OpFunctionParameter(U64)};
const Id value{OpFunctionParameter(type)};
define_body(&StorageDefinitions::U32, addr, storage_types.U32.element, 2,
[&](Id pointer, Id ssbo, Id word) {
const auto atomic = [&](Id (Sirit::Module::*op)(Id, Id, Id, Id, Id)) {
return (this->*op)(U32[1], pointer, scope, zero, value);
};
switch (kind) {
case IAdd: OpReturnValue(atomic(&Sirit::Module::OpAtomicIAdd)); break;
case SMin: OpReturnValue(atomic(&Sirit::Module::OpAtomicSMin)); break;
case UMin: OpReturnValue(atomic(&Sirit::Module::OpAtomicUMin)); break;
case SMax: OpReturnValue(atomic(&Sirit::Module::OpAtomicSMax)); break;
case UMax: OpReturnValue(atomic(&Sirit::Module::OpAtomicUMax)); break;
case And: OpReturnValue(atomic(&Sirit::Module::OpAtomicAnd)); break;
case Or: OpReturnValue(atomic(&Sirit::Module::OpAtomicOr)); break;
case Xor: OpReturnValue(atomic(&Sirit::Module::OpAtomicXor)); break;
case Exchange: OpReturnValue(atomic(&Sirit::Module::OpAtomicExchange)); break;
case Inc:
OpReturnValue(OpFunctionCall(U32[1], increment_cas_ssbo, word, value, ssbo));
break;
case Dec:
OpReturnValue(OpFunctionCall(U32[1], decrement_cas_ssbo, word, value, ssbo));
break;
case AddF32:
OpReturnValue(OpFunctionCall(F32[1], f32_add_cas, word, value, ssbo));
break;
case Count: break;
}
});
OpReturnValue(ConstantNull(type));
OpFunctionEnd();
global_atomic_funcs[index] = func_id;
}
}

void EmitContext::DefineRescalingInput(const Info& info) {
Expand Down
7 changes: 6 additions & 1 deletion src/shader_recompiler/backend/spirv/spirv_emit_context.h
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,11 @@ class EmitContext final : public Sirit::Module {
Id write_global_func_u32{};
Id write_global_func_u32x2{};
Id write_global_func_u32x4{};
// 32-bit global atomics on untracked addresses, routed to the storage buffer holding them.
enum class GlobalAtomic : size_t {
IAdd, SMin, UMin, SMax, UMax, Inc, Dec, And, Or, Xor, Exchange, AddF32, Count,
};
std::array<Id, static_cast<size_t>(GlobalAtomic::Count)> global_atomic_funcs{};

bool need_input_position_indirect{};
Id input_position{};
Expand Down Expand Up @@ -420,7 +425,7 @@ class EmitContext final : public Sirit::Module {
void DefineImages(const Info& info, u32& binding, u32& scaling_index);
void DefineAttributeMemAccess(const Info& info);
void DefineWriteStorageCasLoopFunction(const Info& info);
void DefineGlobalMemoryFunctions(const Info& info);
void DefineGlobalMemoryFunctions(const IR::Program& program);
void DefineRescalingInput(const Info& info);
void DefineRescalingInputPushConstant();
void DefineRescalingInputUniformConstant();
Expand Down
Loading