Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 23 additions & 4 deletions llvm/include/llvm/CodeGen/TargetInstrInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -1225,14 +1225,33 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// can be passed elsewhere. The \p Flags is used to set appropriate machine
/// flags on the spill instruction e.g. FrameSetup flag on a callee saved
/// register spill instruction, part of prologue, during the frame lowering.
virtual void storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const {
///
/// \p Lanes limits the spill to those lanes of \p SrcReg (RC lane numbering,
/// not a byte offset); the default LaneBitmask::getAll() spills the whole
/// register. A target opting in via canStoreRegToStackSlotWithLanes may get a
/// narrower mask and must store exactly those lanes, preserving the rest of
/// the slot. It must not widen the store, nor report the partial store as
/// complete via isStoreToStackSlot.
virtual void
storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
Register SrcReg, bool isKill, int FrameIndex,
const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags,
LaneBitmask Lanes = LaneBitmask::getAll()) const {
llvm_unreachable("Target didn't implement "
"TargetInstrInfo::storeRegToStackSlot!");
}

/// Whether the target can spill exactly Lanes of RC while preserving the
/// other lanes of the original stack slot. Lanes uses RC's lane numbering;
/// it is not a byte offset. In particular, a target must not round this mask
/// up to a wider store. A target which opts in receives the narrower mask
/// through storeRegToStackSlot's \p Lanes parameter.
virtual bool canStoreRegToStackSlotWithLanes(const TargetRegisterClass *RC,
LaneBitmask Lanes) const {
return false;
}

/// Load the specified register of the given register class from the specified
/// stack frame index. The load instruction is to be added to the given
/// machine basic block before the specified machine instruction. If \p
Expand Down
236 changes: 214 additions & 22 deletions llvm/lib/CodeGen/InlineSpiller.cpp

Large diffs are not rendered by default.

10 changes: 4 additions & 6 deletions llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6433,12 +6433,10 @@ static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
.addMemOperand(MMO);
}

void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
Register SrcReg, bool isKill, int FI,
const TargetRegisterClass *RC,
Register VReg,
MachineInstr::MIFlag Flags) const {
void AArch64InstrInfo::storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
bool isKill, int FI, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags, LaneBitmask Lanes) const {
MachineFunction &MF = *MBB.getParent();
MachineFrameInfo &MFI = MF.getFrameInfo();

Expand Down
10 changes: 6 additions & 4 deletions llvm/lib/Target/AArch64/AArch64InstrInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -374,10 +374,12 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
bool KillSrc, bool RenamableDest = false,
bool RenamableSrc = false) const override;

void storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
void
storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register SrcReg, bool isKill, int FrameIndex,
const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags,
LaneBitmask Lanes = LaneBitmask::getAll()) const override;

void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Expand Down
63 changes: 60 additions & 3 deletions llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1821,11 +1821,68 @@ void SIInstrInfo::storeRegToStackSlotImpl(
void SIInstrInfo::storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags) const {
MachineInstr::MIFlag Flags, LaneBitmask Lanes) const {
// A narrower-than-full mask requests a partial spill that preserves the
// other words of the shared slot. Otherwise emit the ordinary full-width
// store, including when this class does not support partial spills.
if (Lanes != LaneBitmask::getAll() &&
canStoreRegToStackSlotWithLanes(RC, Lanes)) {
MachineFunction &MF = *MBB.getParent();
SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
MachineFrameInfo &FrameInfo = MF.getFrameInfo();
if (FrameInfo.getStackID(FrameIndex) == TargetStackID::SGPRSpill)
MFI.setHasSpilledSGPRs();
const DebugLoc &DL = MBB.findDebugLoc(MI);

auto Parts = RI.getRegSplitParts(RC, 4);
for (unsigned Word = 0; Word != Parts.size(); ++Word) {
unsigned SubReg = Parts[Word];
LaneBitmask WordLanes = RI.getSubRegIndexLaneMask(SubReg);
if ((Lanes & WordLanes).none())
continue;
Lanes &= ~WordLanes;
MachinePointerInfo PtrInfo =
MachinePointerInfo::getFixedStack(MF, FrameIndex, Word * 4);
auto *StoreMMO = MF.getMachineMemOperand(
PtrInfo, MachineMemOperand::MOStore, 4,
commonAlignment(FrameInfo.getObjectAlign(FrameIndex), Word * 4));
// The scratch fallback without an EXEC-save register preserves the other
// slot words by reading them into its already-scavenged VGPR.
auto *LoadMMO = MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIndex),
MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex),
FrameInfo.getObjectAlign(FrameIndex));
BuildMI(MBB, MI, DL, get(AMDGPU::SI_SPILL_S32_SAVE_PARTIAL))
.addReg(SrcReg, getKillRegState(isKill && Lanes.none()), SubReg)
.addFrameIndex(FrameIndex)
.addImm(Word)
.addMemOperand(StoreMMO)
.addMemOperand(LoadMMO)
.addReg(MFI.getStackPtrOffsetReg(), RegState::Implicit);
}
return;
}

storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg, Flags,
false);
}

bool SIInstrInfo::canStoreRegToStackSlotWithLanes(const TargetRegisterClass *RC,
LaneBitmask Lanes) const {
if (!RI.isSGPRClass(RC) || RI.getSpillSize(*RC) <= 4)
return false;

// SGPR tuples split into 32-bit words. Never turn a partial-word mask into
// a full-word store: such masks need a different preservation mechanism.
for (int16_t SubReg : RI.getRegSplitParts(RC, 4)) {
LaneBitmask Word = RI.getSubRegIndexLaneMask(SubReg);
if ((Lanes & Word).any() && (Lanes & Word) != Word)
return false;
Lanes &= ~Word;
}
return Lanes.none();
}

void SIInstrInfo::storeRegToStackSlotCFI(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
Register SrcReg, bool isKill,
Expand Down Expand Up @@ -10293,7 +10350,7 @@ Register SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
int &FrameIndex,
TypeSize &MemBytes) const {
if (!MI.mayLoad())
if (!MI.mayLoad() || MI.getOpcode() == AMDGPU::SI_SPILL_S32_SAVE_PARTIAL)
return Register();

if (isMUBUF(MI) || isVGPRSpill(MI))
Expand All @@ -10308,7 +10365,7 @@ Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
Register SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
int &FrameIndex,
TypeSize &MemBytes) const {
if (!MI.mayStore())
if (!MI.mayStore() || MI.getOpcode() == AMDGPU::SI_SPILL_S32_SAVE_PARTIAL)
return Register();

if (isMUBUF(MI) || isVGPRSpill(MI))
Expand Down
13 changes: 9 additions & 4 deletions llvm/lib/Target/AMDGPU/SIInstrInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -349,10 +349,15 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
unsigned Size,
const SIMachineFunctionInfo &MFI) const;

void storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
void
storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
Register SrcReg, bool isKill, int FrameIndex,
const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags,
LaneBitmask Lanes = LaneBitmask::getAll()) const override;

bool canStoreRegToStackSlotWithLanes(const TargetRegisterClass *RC,
LaneBitmask Lanes) const override;

void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
Expand Down
8 changes: 8 additions & 0 deletions llvm/lib/Target/AMDGPU/SIInstructions.td
Original file line number Diff line number Diff line change
Expand Up @@ -1217,6 +1217,14 @@ defm SI_SPILL_S384 : SI_SPILL_SGPR <SReg_384>;
defm SI_SPILL_S512 : SI_SPILL_SGPR <SReg_512>;
defm SI_SPILL_S1024 : SI_SPILL_SGPR <SReg_1024>;

// Save one word in its original position in a tuple's shared spill slot.
// Scratch lowering may read the other words to preserve them when no SGPR is
// available to save EXEC. The data operand only reads this one source word.
let UseNamedOperandTable = 1, Spill = 1, SALU = 1, Uses = [EXEC],
mayStore = 1, mayLoad = 1 in
def SI_SPILL_S32_SAVE_PARTIAL : PseudoInstSI<
(outs), (ins SReg_32:$data, i32imm:$addr, i32imm:$offset)>;

let Spill = 1, VALU = 1, isConvergent = 1 in {
def SI_SPILL_S32_TO_VGPR : PseudoInstSI <(outs VGPR_32:$vdst),
(ins SReg_32:$src0, i32imm:$src1, VGPR_32:$vdst_in)> {
Expand Down
26 changes: 23 additions & 3 deletions llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ struct SGPRSpillBuilder {
MachineBasicBlock::iterator MI;
ArrayRef<int16_t> SplitParts;
unsigned NumSubRegs;
unsigned FirstLane = 0;
bool IsPartial = false;
bool IsKill;
const DebugLoc &DL;

Expand Down Expand Up @@ -151,6 +153,11 @@ struct SGPRSpillBuilder {
const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(SuperReg);
SplitParts = TRI.getRegSplitParts(RC, EltSize);
NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
IsPartial = MI->getOpcode() == AMDGPU::SI_SPILL_S32_SAVE_PARTIAL;
if (IsPartial) {
FirstLane = MI->getOperand(2).getImm();
assert(NumSubRegs == 1 && FirstLane < 32 && "Invalid partial SGPR spill");
}

if (IsWave32) {
ExecReg = AMDGPU::EXEC_LO;
Expand All @@ -172,6 +179,7 @@ struct SGPRSpillBuilder {
Data.PerVGPR = IsWave32 ? 32 : 64;
Data.NumVGPRs = (NumSubRegs + (Data.PerVGPR - 1)) / Data.PerVGPR;
Data.VGPRLanes = (1LL << std::min(Data.PerVGPR, NumSubRegs)) - 1LL;
Data.VGPRLanes <<= FirstLane;
return Data;
}

Expand Down Expand Up @@ -1313,6 +1321,7 @@ static unsigned getNumSubRegsForSpillOp(const MachineInstr &MI,
case AMDGPU::SI_SPILL_AV64_CFI_SAVE:
case AMDGPU::SI_SPILL_AV64_RESTORE:
return 2;
case AMDGPU::SI_SPILL_S32_SAVE_PARTIAL:
case AMDGPU::SI_SPILL_S32_SAVE:
case AMDGPU::SI_SPILL_S32_CFI_SAVE:
case AMDGPU::SI_SPILL_S32_RESTORE:
Expand Down Expand Up @@ -2187,7 +2196,7 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, int Index,
// VGPR lanes (mapped from spill stack slot) may be shared for SGPR
// spills of different sizes. This accounts for number of VGPR lanes alloted
// equal to the largest SGPR being spilled in them.
assert(SB.NumSubRegs <= VGPRSpills.size() &&
assert(SB.FirstLane + SB.NumSubRegs <= VGPRSpills.size() &&
"Num of SGPRs spilled should be less than or equal to num of "
"the VGPR lanes.");

Expand All @@ -2196,7 +2205,7 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, int Index,
SB.NumSubRegs == 1
? SB.SuperReg
: Register(getSubReg(SB.SuperReg, SB.SplitParts[i]));
SpilledReg Spill = VGPRSpills[i];
SpilledReg Spill = VGPRSpills[SB.FirstLane + i];

bool IsFirstSubreg = i == 0;
bool IsLastSubreg = i == SB.NumSubRegs - 1;
Expand Down Expand Up @@ -2258,6 +2267,15 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, int Index,
for (unsigned Offset = 0; Offset < PVD.NumVGPRs; ++Offset) {
RegState TmpVGPRFlags = RegState::Undef;

// Without a register to save EXEC, readWriteTmpVGPR transfers every
// wave lane. Preserve the other slot words in the temporary VGPR before
// replacing the selected word. This uses the existing emergency-spill
// machinery, rather than allocating a second SGPR tuple.
if (SB.IsPartial && !SB.SavedExecReg) {
SB.readWriteTmpVGPR(Offset, /*IsLoad=*/true);
TmpVGPRFlags = {};
}

// Write sub registers into the VGPR
for (unsigned i = Offset * PVD.PerVGPR,
e = std::min((Offset + 1) * PVD.PerVGPR, SB.NumSubRegs);
Expand All @@ -2271,7 +2289,7 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, int Index,
BuildMI(*SB.MBB, MI, SB.DL,
SB.TII.get(AMDGPU::SI_SPILL_S32_TO_VGPR), SB.TmpVGPR)
.addReg(SubReg, SubKillState)
.addImm(i % PVD.PerVGPR)
.addImm((SB.FirstLane + i) % PVD.PerVGPR)
.addReg(SB.TmpVGPR, TmpVGPRFlags);
TmpVGPRFlags = {};

Expand Down Expand Up @@ -2506,6 +2524,7 @@ bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex(
case AMDGPU::SI_SPILL_S96_SAVE:
case AMDGPU::SI_SPILL_S64_SAVE:
case AMDGPU::SI_SPILL_S32_SAVE:
case AMDGPU::SI_SPILL_S32_SAVE_PARTIAL:
return spillSGPR(MI, FI, RS, Indexes, LIS, true, SpillToPhysVGPRLane,
NeedsCFI);
case AMDGPU::SI_SPILL_S1024_RESTORE:
Expand Down Expand Up @@ -2602,6 +2621,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
case AMDGPU::SI_SPILL_S128_SAVE:
case AMDGPU::SI_SPILL_S96_SAVE:
case AMDGPU::SI_SPILL_S64_SAVE:
case AMDGPU::SI_SPILL_S32_SAVE_PARTIAL:
case AMDGPU::SI_SPILL_S32_SAVE: {
return spillSGPR(MI, Index, RS, nullptr, nullptr,
FrameInfo.getStackID(Index) == TargetStackID::SGPRSpill,
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Target/ARC/ARCInstrInfo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ void ARCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
void ARCInstrInfo::storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator I, Register SrcReg,
bool IsKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags) const {
MachineInstr::MIFlag Flags, LaneBitmask Lanes) const {
DebugLoc DL = MBB.findDebugLoc(I);
MachineFunction &MF = *MBB.getParent();
MachineFrameInfo &MFI = MF.getFrameInfo();
Expand Down
10 changes: 6 additions & 4 deletions llvm/lib/Target/ARC/ARCInstrInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,12 @@ class ARCInstrInfo : public ARCGenInstrInfo {
bool KillSrc, bool RenamableDest = false,
bool RenamableSrc = false) const override;

void storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
bool IsKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
void
storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
Register SrcReg, bool IsKill, int FrameIndex,
const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags,
LaneBitmask Lanes = LaneBitmask::getAll()) const override;

void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
Expand Down
10 changes: 4 additions & 6 deletions llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -959,12 +959,10 @@ const MachineInstrBuilder &ARMBaseInstrInfo::AddDReg(MachineInstrBuilder &MIB,
return MIB.addReg(Reg, State, SubIdx);
}

void ARMBaseInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register SrcReg, bool isKill, int FI,
const TargetRegisterClass *RC,
Register VReg,
MachineInstr::MIFlag Flags) const {
void ARMBaseInstrInfo::storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator I, Register SrcReg,
bool isKill, int FI, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags, LaneBitmask Lanes) const {
MachineFunction &MF = *MBB.getParent();
MachineFrameInfo &MFI = MF.getFrameInfo();
Align Alignment = MFI.getObjectAlign(FI);
Expand Down
10 changes: 6 additions & 4 deletions llvm/lib/Target/ARM/ARMBaseInstrInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -229,10 +229,12 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
bool KillSrc, bool RenamableDest = false,
bool RenamableSrc = false) const override;

void storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
void
storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register SrcReg, bool isKill, int FrameIndex,
const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags,
LaneBitmask Lanes = LaneBitmask::getAll()) const override;

void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Expand Down
10 changes: 4 additions & 6 deletions llvm/lib/Target/ARM/Thumb1InstrInfo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,10 @@ void Thumb1InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
}
}

void Thumb1InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I,
Register SrcReg, bool isKill, int FI,
const TargetRegisterClass *RC,
Register VReg,
MachineInstr::MIFlag Flags) const {
void Thumb1InstrInfo::storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator I, Register SrcReg,
bool isKill, int FI, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags, LaneBitmask Lanes) const {
assert((RC == &ARM::tGPRRegClass ||
(SrcReg.isPhysical() && isARMLowRegister(SrcReg))) &&
"Unknown regclass!");
Expand Down
10 changes: 6 additions & 4 deletions llvm/lib/Target/ARM/Thumb1InstrInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,12 @@ class Thumb1InstrInfo : public ARMBaseInstrInfo {
const DebugLoc &DL, Register DestReg, Register SrcReg,
bool KillSrc, bool RenamableDest = false,
bool RenamableSrc = false) const override;
void storeRegToStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
void
storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Register SrcReg, bool isKill, int FrameIndex,
const TargetRegisterClass *RC, Register VReg,
MachineInstr::MIFlag Flags = MachineInstr::NoFlags,
LaneBitmask Lanes = LaneBitmask::getAll()) const override;

void loadRegFromStackSlot(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
Expand Down
Loading
Loading