AArch64: lower Go write barrier intrinsics

llvm/include/llvm/IR/Intrinsics.td

@@ -1921,6 +1921,21 @@ def int_experimental_patchpoint : Intrinsic<[llvm_any_ty],
 //===------------------------ Garbage Collection Intrinsics ---------------===//
 // These are documented in docs/Statepoint.rst
 
+// Acquire Entries pointer-sized slots from the current P's Go write-barrier
+// buffer.  This is deliberately an intrinsic rather than an ordinary call:
+// the runtime entry point uses a private ABI and returns the buffer pointer in
+// a target-specific fixed register.
+//
+// Do not narrow this to inaccessible memory.  In addition to updating runtime
+// state, the operation orders the caller's stores to the returned buffer with
+// the surrounding write-barrier sequence.  Likewise, do not give it the
+// default nosync/nofree/willreturn attributes; the slow path may flush the
+// buffer through the runtime.  The entry count is the number of pointer-sized
+// slots and currently follows Go's 1..8 batching limit.
+def int_go_gc_write_barrier : Intrinsic<
+    [llvm_ptr_ty], [llvm_i32_ty],
+    [IntrNoCallback, ImmArg<ArgIndex<0>>, Range<ArgIndex<0>, 1, 9>]>;
+
 def int_experimental_gc_statepoint : Intrinsic<[llvm_token_ty],
                                [llvm_i64_ty, llvm_i32_ty,
                                 llvm_anyptr_ty, llvm_i32_ty,

llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp

@@ -19,6 +19,7 @@
 #include "AArch64Subtarget.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "Utils/AArch64BaseInfo.h"
+#include "llvm/BinaryFormat/GoObj.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
@@ -29,6 +30,7 @@
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DebugLoc.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CodeGen.h"
@@ -1326,6 +1328,34 @@ bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
   default:
     break;
 
+  case AArch64::GO_GC_WRITE_BARRIER: {
+    static constexpr const char *WriteBarrierNames[] = {
+        "runtime.gcWriteBarrier1", "runtime.gcWriteBarrier2",
+        "runtime.gcWriteBarrier3", "runtime.gcWriteBarrier4",
+        "runtime.gcWriteBarrier5", "runtime.gcWriteBarrier6",
+        "runtime.gcWriteBarrier7", "runtime.gcWriteBarrier8",
+    };
+    int64_t Entries = MI.getOperand(0).getImm();
+    if (Entries < 1 || Entries > 8)
+      report_fatal_error("Go write barrier entry count must be in [1, 8]");
+
+    MachineInstrBuilder Call =
+        BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::BL));
+    const char *WriteBarrierName = WriteBarrierNames[Entries - 1];
+    MachineFunction &MF = *MBB.getParent();
+    if (MF.getTarget().getTargetTriple().isOSBinFormatGoObj()) {
+      MCContext &Ctx = MF.getContext();
+      MCSymbol *Callee = Ctx.getOrCreateSymbol(WriteBarrierName);
+      Ctx.setGoObjSymbolABI(Callee, GoObj::SymABIInternal);
+      Call.addSym(Callee);
+    } else {
+      Call.addExternalSymbol(WriteBarrierName);
+    }
+    transferImpOps(MI, Call, Call);
+    MI.eraseFromParent();
+    return true;
+  }
+
   case AArch64::BSPv8i8:
   case AArch64::BSPv16i8: {
     Register DstReg = MI.getOperand(0).getReg();

llvm/lib/Target/AArch64/AArch64InstrInfo.td

@@ -3777,6 +3777,31 @@ def BL : CallImm<1, "bl", [(AArch64call tglobaladdr:$addr)]>;
 } // isCall
 def : Pat<(AArch64call texternalsym:$func), (BL texternalsym:$func)>;
 
+// Go's write-barrier buffer entry points use a private ABI: the selected
+// runtime.gcWriteBarrierN wrapper returns the buffer pointer in X25, preserves
+// the allocatable general-purpose registers except X27, and may clobber the
+// linker temporaries, condition flags, LR, and all floating-point/vector
+// registers.
+// Keep this as a call-like pseudo through register allocation so that the
+// fixed-register and clobber constraints are visible without treating the IR
+// intrinsic as an ordinary call/statepoint.
+let isCall = 1, isCodeGenOnly = 1, hasSideEffects = 1,
+    mayLoad = 1, mayStore = 1, Uses = [SP],
+    // Keep the fixed result first so InstrEmitter associates the intrinsic's
+    // value with X25 rather than with one of the private call's clobbers.
+    Defs = [X25, X16, X17, X27, LR, NZCV,
+            Q0, Q1, Q2, Q3, Q4, Q5, Q6, Q7,
+            Q8, Q9, Q10, Q11, Q12, Q13, Q14, Q15,
+            Q16, Q17, Q18, Q19, Q20, Q21, Q22, Q23,
+            Q24, Q25, Q26, Q27, Q28, Q29, Q30, Q31] in
+def GO_GC_WRITE_BARRIER
+    : Pseudo<(outs), (ins i32imm:$entries),
+             [(set X25,
+                   (int_go_gc_write_barrier (i32 timm:$entries)))]>,
+      Sched<[WriteBr]> {
+  let Size = 4;
+}
+
 //===----------------------------------------------------------------------===//
 // Exception generation instructions.
 //===----------------------------------------------------------------------===//

llvm/test/CodeGen/AArch64/go-gc-write-barrier.ll

@@ -0,0 +1,61 @@
+; RUN: opt -S -passes='default<O2>' %s | FileCheck %s --check-prefix=OPT
+; RUN: opt -S -passes=rewrite-statepoints-for-gc %s | FileCheck %s --check-prefix=STATEPOINT
+; RUN: llc -mtriple=arm64-apple-macosx -verify-machineinstrs -stop-before=aarch64-expand-pseudo -o - %s | FileCheck %s --check-prefix=PSEUDO
+; RUN: llc -mtriple=arm64-apple-macosx -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ASM
+; RUN: llc -O0 -mtriple=arm64-apple-macosx -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=ASM
+; RUN: llc -mtriple=aarch64-apple-darwin-goobj -goobj-package-path=main \
+; RUN:   -verify-machineinstrs -filetype=obj -o %t.o %s
+; RUN: %python %S/../../MC/GoObj/Inputs/dump-goobj.py %t.o | \
+; RUN:   FileCheck %s --check-prefix=OBJ
+
+declare ptr @llvm.go.gc.write.barrier(i32 immarg)
+
+define goabiinternal ptr @acquire_one() gc "statepoint-example" {
+; OPT-LABEL: define goabiinternal ptr @acquire_one()
+; OPT: call ptr @llvm.go.gc.write.barrier(i32 1)
+;
+; STATEPOINT-LABEL: define goabiinternal ptr @acquire_one()
+; STATEPOINT: call ptr @llvm.go.gc.write.barrier(i32 1)
+; STATEPOINT-NOT: gc.statepoint
+;
+; PSEUDO-LABEL: name: acquire_one
+; PSEUDO: GO_GC_WRITE_BARRIER 1
+; PSEUDO-SAME: implicit-def $x25
+; PSEUDO-SAME: implicit-def {{(dead )?}}$x27
+; PSEUDO-SAME: implicit $sp
+;
+; ASM-LABEL: _acquire_one:
+; ASM: bl _runtime.gcWriteBarrier1
+; ASM-NEXT: mov x0, x25
+  %buf = call ptr @llvm.go.gc.write.barrier(i32 1)
+  ret ptr %buf
+}
+
+define goabiinternal ptr @acquire_eight() gc "statepoint-example" {
+; PSEUDO-LABEL: name: acquire_eight
+; PSEUDO: GO_GC_WRITE_BARRIER 8
+;
+; ASM-LABEL: _acquire_eight:
+; ASM: bl _runtime.gcWriteBarrier8
+; ASM-NEXT: mov x0, x25
+  %buf = call ptr @llvm.go.gc.write.barrier(i32 8)
+  ret ptr %buf
+}
+
+define goabiinternal void @store_one(ptr %value) gc "statepoint-example" {
+; PSEUDO-LABEL: name: store_one
+; PSEUDO: GO_GC_WRITE_BARRIER 1
+;
+; ASM-LABEL: _store_one:
+; ASM: bl _runtime.gcWriteBarrier1
+; ASM-NEXT: str x0, [x25]
+  %buf = call ptr @llvm.go.gc.write.barrier(i32 1)
+  store ptr %value, ptr %buf
+  ret void
+}
+
+; OBJ: nonpkgref {{[0-9]+}}: runtime.gcWriteBarrier1 abi=1 type=0 size=0
+; OBJ: nonpkgref {{[0-9]+}}: runtime.gcWriteBarrier8 abi=1 type=0 size=0
+; OBJ-NOT: runtime.gcWriteBarrier{{[18]}} abi=0
+; OBJ: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=9 add=0 target=runtime.gcWriteBarrier1
+; OBJ: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=9 add=0 target=runtime.gcWriteBarrier8