X86: Repair Go ABI state at ABI0 boundaries

llvm/lib/MC/GoObjObjectWriter.cpp

@@ -2369,8 +2369,6 @@ uint64_t GoObjObjectWriter::writeObject() {
           "GoObj private constants with relocations are not supported");
 
     int64_t Addend = getGoObjRelocAddend(Reloc);
-    GoObjSymRef TargetSymRef = GetTargetSymRef(Reloc, Addend);
-
     uint16_t RelocType = checkedUint16(Reloc.Type, "relocation type");
     if (Source.Symbol) {
       if (const auto *Overrides =
@@ -2392,6 +2390,16 @@ uint64_t GoObjObjectWriter::writeObject() {
         RelocType |= GoObj::R_WEAK;
     }
 
+    // Native x86 Go objects intentionally leave the internal-linking TLS
+    // relocation target empty. The linker resolves R_TLS_LE against its
+    // synthetic runtime.tlsg symbol and supplies that symbol itself when it
+    // translates the relocation for external ELF linking.
+    const Triple::ArchType Arch = Asm->getContext().getTargetTriple().getArch();
+    const bool IsX86TLSLE = (Arch == Triple::x86 || Arch == Triple::x86_64) &&
+                            (RelocType & ~GoObj::R_WEAK) == GoObj::R_TLS_LE;
+    GoObjSymRef TargetSymRef =
+        IsX86TLSLE ? GoObjSymRef{} : GetTargetSymRef(Reloc, Addend);
+
     Source.Relocations.push_back(
         {static_cast<uint32_t>(LocalOffset), Reloc.Size, RelocType, Addend,
          TargetSymRef.PkgIdx, TargetSymRef.SymIdx, std::nullopt});

llvm/lib/Target/X86/CMakeLists.txt

@@ -37,6 +37,7 @@ set(sources
   X86CodeGenPassBuilder.cpp
   X86DomainReassignment.cpp
   X86GlobalBaseReg.cpp
+  X86GoABI.cpp
   X86LowerTileCopy.cpp
   X86LowerAMXType.cpp
   X86LowerAMXIntrinsics.cpp

llvm/lib/Target/X86/X86.h

@@ -84,6 +84,16 @@ class X86InsertVZeroUpperPass
 };
 
 FunctionPass *createX86InsertVZeroUpperLegacyPass();
+
+/// This pass restores x86-64 Go ABIInternal's reserved register state at ABI0
+/// boundaries after register allocation and frame lowering.
+class X86GoABIPass : public OptionalPassInfoMixin<X86GoABIPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+
+FunctionPass *createX86GoABILegacyPass();
 /// This pass inserts ENDBR instructions before indirect jump/call
 /// destinations as part of CET IBT mechanism.
 class X86IndirectBranchTrackingPass
@@ -480,6 +490,7 @@ void initializeCompressEVEXLegacyPass(PassRegistry &);
 void initializeX86FixupBWInstLegacyPass(PassRegistry &);
 void initializeFixupLEAsLegacyPass(PassRegistry &);
 void initializeX86ArgumentStackSlotLegacyPass(PassRegistry &);
+void initializeX86GoABILegacyPass(PassRegistry &);
 void initializeX86AsmPrinterPass(PassRegistry &);
 void initializeX86FixupInstTuningLegacyPass(PassRegistry &);
 void initializeX86FixupVectorConstantsLegacyPass(PassRegistry &);

llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp

@@ -198,6 +198,7 @@ void X86CodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) const {
   }
   addMachineFunctionPass(X86CompressEVEXPass(), PMW);
   addMachineFunctionPass(X86InsertX87WaitPass(), PMW);
+  addMachineFunctionPass(X86GoABIPass(), PMW);
 }
 
 void X86CodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) const {

llvm/lib/Target/X86/X86FrameLowering.cpp

@@ -3121,6 +3121,19 @@ StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
   bool IsWin64Prologue = MF.getTarget().getMCAsmInfo().usesWindowsCFI();
   int64_t FPDelta = 0;
 
+  // Go keeps a frame-pointer chain for profiling and traceback, but its local
+  // frame slots are addressed from SP.  This is more than a code-generation
+  // preference: runtime.gogo can resume a suspended frame after restoring SP
+  // while deliberately clearing BP.  A BP-relative local would then become
+  // inaccessible even though the Go frame is otherwise valid.  Go frames have
+  // a reserved call frame, so SP remains a stable base for ordinary locals.
+  if (goabi::isGoCallingConv(MF.getFunction().getCallingConv()) && !IsFixed &&
+      !TRI->hasStackRealignment(MF) && !TRI->hasBasePointer(MF) &&
+      hasReservedCallFrame(MF)) {
+    FrameReg = TRI->getStackRegister();
+    return StackOffset::getFixed(Offset + StackSize);
+  }
+
   // In an x86 interrupt, remove the offset we added to account for the return
   // address from any stack object allocated in the caller's frame. Interrupts
   // do not have a standard return address. Fixed objects in the current frame,

llvm/lib/Target/X86/X86GoABI.cpp

@@ -0,0 +1,163 @@
+//===- X86GoABI.cpp - Repair reserved Go ABI state ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// R14 and XMM15 are reserved in every x86-64 Go function. ABIInternal requires
+// them to contain g and zero respectively, while ABI0 does not establish or
+// preserve those values. This late pass repairs the reserved state at ABI
+// boundaries after register allocation, frame lowering, and scheduling.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/X86BaseInfo.h"
+#include "X86.h"
+#include "X86InstrInfo.h"
+#include "X86RegisterInfo.h"
+#include "X86Subtarget.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/BinaryFormat/GoObj.h"
+#include "llvm/CodeGen/GoCallingConv.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/CodeGen/StackMaps.h"
+#include "llvm/IR/CallingConv.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/Pass.h"
+
+#include <iterator>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "x86-go-abi"
+
+namespace {
+
+class X86GoABILegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  X86GoABILegacy() : MachineFunctionPass(ID) {}
+
+  StringRef getPassName() const override { return "X86 Go ABI state repair"; }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties().setNoVRegs();
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+
+} // end anonymous namespace
+
+char X86GoABILegacy::ID = 0;
+
+INITIALIZE_PASS(X86GoABILegacy, DEBUG_TYPE, "X86 Go ABI state repair", false,
+                false)
+
+FunctionPass *llvm::createX86GoABILegacyPass() { return new X86GoABILegacy(); }
+
+static StringRef getDirectCalleeName(const MachineInstr &MI,
+                                     const X86InstrInfo &TII) {
+  const MachineOperand &Callee = MI.getOpcode() == TargetOpcode::STATEPOINT
+                                     ? StatepointOpers(&MI).getCallTarget()
+                                     : TII.getCalleeOperand(MI);
+  if (Callee.isGlobal())
+    return Callee.getGlobal()->getName();
+  if (Callee.isSymbol())
+    return Callee.getSymbolName();
+  if (Callee.isMCSymbol())
+    return Callee.getMCSymbol()->getName();
+  return {};
+}
+
+static bool isOrdinaryGoCall(const MachineInstr &MI,
+                             const MachineFunction &MF) {
+  if (MI.getOpcode() == TargetOpcode::STATEPOINT)
+    return goabi::isGoCallingConv(StatepointOpers(&MI).getCallingConv());
+
+  const X86RegisterInfo &TRI =
+      *MF.getSubtarget<X86Subtarget>().getRegisterInfo();
+  const uint32_t *GoMask =
+      TRI.getCallPreservedMask(MF, CallingConv::GoABIInternal);
+  return llvm::any_of(MI.operands(), [GoMask](const MachineOperand &MO) {
+    return MO.isRegMask() && MO.getRegMask() == GoMask;
+  });
+}
+
+static void emitABIInternalState(MachineBasicBlock &MBB,
+                                 MachineBasicBlock::iterator Pos,
+                                 const DebugLoc &DL, const X86InstrInfo &TII) {
+  BuildMI(MBB, Pos, DL, TII.get(X86::XORPSrr), X86::XMM15)
+      .addReg(X86::XMM15, RegState::Undef)
+      .addReg(X86::XMM15, RegState::Undef);
+  BuildMI(MBB, Pos, DL, TII.get(X86::MOV64rm), X86::R14)
+      .addReg(X86::NoRegister)
+      .addImm(1)
+      .addReg(X86::NoRegister)
+      .addExternalSymbol("runtime.tlsg", X86II::MO_TPOFF)
+      .addReg(X86::FS);
+}
+
+static bool repairGoABIState(MachineFunction &MF) {
+  CallingConv::ID CallerCC = MF.getFunction().getCallingConv();
+  if (!goabi::isGoCallingConv(CallerCC))
+    return false;
+
+  const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
+  if (!ST.is64Bit())
+    return false;
+
+  assert(MF.getRegInfo().isReserved(X86::R14) &&
+         MF.getRegInfo().isReserved(X86::XMM15) &&
+         "Go ABI state registers must be reserved");
+
+  const X86InstrInfo &TII = *ST.getInstrInfo();
+  bool Changed = false;
+  for (MachineBasicBlock &MBB : MF) {
+    for (auto I = MBB.begin(); I != MBB.end(); ++I) {
+      MachineInstr &MI = *I;
+      if (!MI.isCall())
+        continue;
+      if (!isOrdinaryGoCall(MI, MF))
+        continue;
+      StringRef CalleeName = getDirectCalleeName(MI, TII);
+
+      bool CalleeIsABI0 = CalleeName.ends_with(GoObj::ABI0SymbolSuffix);
+      bool RepairBefore = goabi::isGoABI0CallingConv(CallerCC) && !CalleeIsABI0;
+      bool RepairAfter =
+          goabi::isGoABIInternalCallingConv(CallerCC) && CalleeIsABI0;
+      if (RepairBefore) {
+        emitABIInternalState(MBB, I, MI.getDebugLoc(), TII);
+        Changed = true;
+      } else if (RepairAfter) {
+        emitABIInternalState(MBB, std::next(I), MI.getDebugLoc(), TII);
+        Changed = true;
+      }
+    }
+  }
+  return Changed;
+}
+
+bool X86GoABILegacy::runOnMachineFunction(MachineFunction &MF) {
+  return repairGoABIState(MF);
+}
+
+PreservedAnalyses X86GoABIPass::run(MachineFunction &MF,
+                                    MachineFunctionAnalysisManager &MFAM) {
+  return repairGoABIState(MF) ? getMachineFunctionPassPreservedAnalyses()
+                                    .preserveSet<CFGAnalyses>()
+                              : PreservedAnalyses::all();
+}

llvm/lib/Target/X86/X86PassRegistry.def

@@ -49,6 +49,7 @@ MACHINE_FUNCTION_PASS("x86-fixup-vector-constants", X86FixupVectorConstantsPass(
 MACHINE_FUNCTION_PASS("x86-flags-copy-lowering", X86FlagsCopyLoweringPass())
 MACHINE_FUNCTION_PASS("x86-fp-stackifier", X86FPStackifierPass())
 MACHINE_FUNCTION_PASS("x86-global-base-reg", X86GlobalBaseRegPass())
+MACHINE_FUNCTION_PASS("x86-go-abi", X86GoABIPass())
 MACHINE_FUNCTION_PASS("x86-indirect-branch-tracking", X86IndirectBranchTrackingPass())
 MACHINE_FUNCTION_PASS("x86-insert-vzeroupper", X86InsertVZeroUpperPass())
 MACHINE_FUNCTION_PASS("x86-insert-x87-wait", X86InsertX87WaitPass())

llvm/lib/Target/X86/X86TargetMachine.cpp

@@ -101,6 +101,7 @@ extern "C" LLVM_C_ABI void LLVMInitializeX86Target() {
   initializeX86ReturnThunksLegacyPass(PR);
   initializeX86DAGToDAGISelLegacyPass(PR);
   initializeX86ArgumentStackSlotLegacyPass(PR);
+  initializeX86GoABILegacyPass(PR);
   initializeX86AsmPrinterPass(PR);
   initializeX86FixupInstTuningLegacyPass(PR);
   initializeX86FixupVectorConstantsLegacyPass(PR);
@@ -573,6 +574,7 @@ void X86PassConfig::addPreEmitPass() {
   }
   addPass(createX86CompressEVEXLegacyPass());
   addPass(createX86InsertX87WaitLegacyPass());
+  addPass(createX86GoABILegacyPass());
 }
 
 void X86PassConfig::addPreEmitPass2() {