diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 37c9c783465d6..1e22799504d03 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1921,6 +1921,21 @@ def int_experimental_patchpoint : Intrinsic<[llvm_any_ty], //===------------------------ Garbage Collection Intrinsics ---------------===// // These are documented in docs/Statepoint.rst +// Acquire Entries pointer-sized slots from the current P's Go write-barrier +// buffer. This is deliberately an intrinsic rather than an ordinary call: +// the runtime entry point uses a private ABI and returns the buffer pointer in +// a target-specific fixed register. +// +// Do not narrow this to inaccessible memory. In addition to updating runtime +// state, the operation orders the caller's stores to the returned buffer with +// the surrounding write-barrier sequence. Likewise, do not give it the +// default nosync/nofree/willreturn attributes; the slow path may flush the +// buffer through the runtime. The entry count is the number of pointer-sized +// slots and currently follows Go's 1..8 batching limit. +def int_go_gc_write_barrier : Intrinsic< + [llvm_ptr_ty], [llvm_i32_ty], + [IntrNoCallback, ImmArg>, Range, 1, 9>]>; + def int_experimental_gc_statepoint : Intrinsic<[llvm_token_ty], [llvm_i64_ty, llvm_i32_ty, llvm_anyptr_ty, llvm_i32_ty, diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp index 53f7fdaab5522..cc9068c6aec38 100644 --- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp +++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp @@ -19,6 +19,7 @@ #include "AArch64Subtarget.h" #include "MCTargetDesc/AArch64AddressingModes.h" #include "Utils/AArch64BaseInfo.h" +#include "llvm/BinaryFormat/GoObj.h" #include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineConstantPool.h" @@ -29,6 +30,7 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DebugLoc.h" +#include "llvm/MC/MCContext.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/Pass.h" #include "llvm/Support/CodeGen.h" @@ -1326,6 +1328,34 @@ bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB, default: break; + case AArch64::GO_GC_WRITE_BARRIER: { + static constexpr const char *WriteBarrierNames[] = { + "runtime.gcWriteBarrier1", "runtime.gcWriteBarrier2", + "runtime.gcWriteBarrier3", "runtime.gcWriteBarrier4", + "runtime.gcWriteBarrier5", "runtime.gcWriteBarrier6", + "runtime.gcWriteBarrier7", "runtime.gcWriteBarrier8", + }; + int64_t Entries = MI.getOperand(0).getImm(); + if (Entries < 1 || Entries > 8) + report_fatal_error("Go write barrier entry count must be in [1, 8]"); + + MachineInstrBuilder Call = + BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::BL)); + const char *WriteBarrierName = WriteBarrierNames[Entries - 1]; + MachineFunction &MF = *MBB.getParent(); + if (MF.getTarget().getTargetTriple().isOSBinFormatGoObj()) { + MCContext &Ctx = MF.getContext(); + MCSymbol *Callee = Ctx.getOrCreateSymbol(WriteBarrierName); + Ctx.setGoObjSymbolABI(Callee, GoObj::SymABIInternal); + Call.addSym(Callee); + } else { + Call.addExternalSymbol(WriteBarrierName); + } + transferImpOps(MI, Call, Call); + MI.eraseFromParent(); + return true; + } + case AArch64::BSPv8i8: case AArch64::BSPv16i8: { Register DstReg = MI.getOperand(0).getReg(); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 51fb2e1d1f9f0..0532218514007 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -3777,6 +3777,31 @@ def BL : CallImm<1, "bl", [(AArch64call tglobaladdr:$addr)]>; } // isCall def : Pat<(AArch64call texternalsym:$func), (BL texternalsym:$func)>; +// Go's write-barrier buffer entry points use a private ABI: the selected +// runtime.gcWriteBarrierN wrapper returns the buffer pointer in X25, preserves +// the allocatable general-purpose registers except X27, and may clobber the +// linker temporaries, condition flags, LR, and all floating-point/vector +// registers. +// Keep this as a call-like pseudo through register allocation so that the +// fixed-register and clobber constraints are visible without treating the IR +// intrinsic as an ordinary call/statepoint. +let isCall = 1, isCodeGenOnly = 1, hasSideEffects = 1, + mayLoad = 1, mayStore = 1, Uses = [SP], + // Keep the fixed result first so InstrEmitter associates the intrinsic's + // value with X25 rather than with one of the private call's clobbers. + Defs = [X25, X16, X17, X27, LR, NZCV, + Q0, Q1, Q2, Q3, Q4, Q5, Q6, Q7, + Q8, Q9, Q10, Q11, Q12, Q13, Q14, Q15, + Q16, Q17, Q18, Q19, Q20, Q21, Q22, Q23, + Q24, Q25, Q26, Q27, Q28, Q29, Q30, Q31] in +def GO_GC_WRITE_BARRIER + : Pseudo<(outs), (ins i32imm:$entries), + [(set X25, + (int_go_gc_write_barrier (i32 timm:$entries)))]>, + Sched<[WriteBr]> { + let Size = 4; +} + //===----------------------------------------------------------------------===// // Exception generation instructions. //===----------------------------------------------------------------------===// diff --git a/llvm/test/CodeGen/AArch64/go-gc-write-barrier.ll b/llvm/test/CodeGen/AArch64/go-gc-write-barrier.ll new file mode 100644 index 0000000000000..d35fccce04c2a --- /dev/null +++ b/llvm/test/CodeGen/AArch64/go-gc-write-barrier.ll @@ -0,0 +1,61 @@ +; RUN: opt -S -passes='default' %s | FileCheck %s --check-prefix=OPT +; RUN: opt -S -passes=rewrite-statepoints-for-gc %s | FileCheck %s --check-prefix=STATEPOINT +; RUN: llc -mtriple=arm64-apple-macosx -verify-machineinstrs -stop-before=aarch64-expand-pseudo -o - %s | FileCheck %s --check-prefix=PSEUDO +; RUN: llc -mtriple=arm64-apple-macosx -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ASM +; RUN: llc -O0 -mtriple=arm64-apple-macosx -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ASM +; RUN: llc -mtriple=aarch64-apple-darwin-goobj -goobj-package-path=main \ +; RUN: -verify-machineinstrs -filetype=obj -o %t.o %s +; RUN: %python %S/../../MC/GoObj/Inputs/dump-goobj.py %t.o | \ +; RUN: FileCheck %s --check-prefix=OBJ + +declare ptr @llvm.go.gc.write.barrier(i32 immarg) + +define goabiinternal ptr @acquire_one() gc "statepoint-example" { +; OPT-LABEL: define goabiinternal ptr @acquire_one() +; OPT: call ptr @llvm.go.gc.write.barrier(i32 1) +; +; STATEPOINT-LABEL: define goabiinternal ptr @acquire_one() +; STATEPOINT: call ptr @llvm.go.gc.write.barrier(i32 1) +; STATEPOINT-NOT: gc.statepoint +; +; PSEUDO-LABEL: name: acquire_one +; PSEUDO: GO_GC_WRITE_BARRIER 1 +; PSEUDO-SAME: implicit-def $x25 +; PSEUDO-SAME: implicit-def {{(dead )?}}$x27 +; PSEUDO-SAME: implicit $sp +; +; ASM-LABEL: _acquire_one: +; ASM: bl _runtime.gcWriteBarrier1 +; ASM-NEXT: mov x0, x25 + %buf = call ptr @llvm.go.gc.write.barrier(i32 1) + ret ptr %buf +} + +define goabiinternal ptr @acquire_eight() gc "statepoint-example" { +; PSEUDO-LABEL: name: acquire_eight +; PSEUDO: GO_GC_WRITE_BARRIER 8 +; +; ASM-LABEL: _acquire_eight: +; ASM: bl _runtime.gcWriteBarrier8 +; ASM-NEXT: mov x0, x25 + %buf = call ptr @llvm.go.gc.write.barrier(i32 8) + ret ptr %buf +} + +define goabiinternal void @store_one(ptr %value) gc "statepoint-example" { +; PSEUDO-LABEL: name: store_one +; PSEUDO: GO_GC_WRITE_BARRIER 1 +; +; ASM-LABEL: _store_one: +; ASM: bl _runtime.gcWriteBarrier1 +; ASM-NEXT: str x0, [x25] + %buf = call ptr @llvm.go.gc.write.barrier(i32 1) + store ptr %value, ptr %buf + ret void +} + +; OBJ: nonpkgref {{[0-9]+}}: runtime.gcWriteBarrier1 abi=1 type=0 size=0 +; OBJ: nonpkgref {{[0-9]+}}: runtime.gcWriteBarrier8 abi=1 type=0 size=0 +; OBJ-NOT: runtime.gcWriteBarrier{{[18]}} abi=0 +; OBJ: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=9 add=0 target=runtime.gcWriteBarrier1 +; OBJ: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=9 add=0 target=runtime.gcWriteBarrier8