From 85bbe83868b4a6b7a082ad7769493e91ce93dc2a Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 2 Aug 2026 08:55:18 +0800 Subject: [PATCH] cmd/compile: expand LLVM nil checks after IR construction --- src/cmd/compile/internal/ssa/ssa2llvm.go | 15 +-- .../compile/internal/ssa/ssa2llvm_nilcheck.go | 126 ++++++++++++++---- src/cmd/go.mod | 2 +- src/cmd/go.sum | 4 +- .../github.com/goallc/go-llvm/IRBindings.cpp | 6 + .../github.com/goallc/go-llvm/IRBindings.h | 3 + .../vendor/github.com/goallc/go-llvm/ir.go | 3 + src/cmd/vendor/modules.txt | 2 +- test/codegen/llvm_nilcheck.go | 10 +- 9 files changed, 129 insertions(+), 42 deletions(-) diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index 2fe71eb734a1df..5275e3dd7d2741 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -15,7 +15,6 @@ import ( type LLVMFuncContext struct { BBs map[ID]llvm.BasicBlock - BlockEnds map[ID]llvm.BasicBlock Vs map[ID]llvm.Value Locals map[llvmLocalKey]llvmStackSlot ItabMethods map[ID]bool @@ -495,7 +494,7 @@ func (lfc *LLVMFuncContext) FinishPhi() { } var predecessors []llvm.BasicBlock for _, pred := range BB.Preds { - predecessors = append(predecessors, lfc.BlockEnds[pred.Block().ID]) + predecessors = append(predecessors, lfc.BBs[pred.Block().ID]) } lfc.Vs[v.ID].AddIncoming(incomingLVals, predecessors) } @@ -730,7 +729,7 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { } savedBlock := lfc.b.GetInsertBlock() if v.Block != nil { - lfc.b.SetInsertPointAtEnd(lfc.BlockEnds[v.Block.ID]) + lfc.b.SetInsertPointAtEnd(lfc.BBs[v.Block.ID]) } defer func() { if !savedBlock.IsNil() { @@ -1064,7 +1063,7 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { lVal = lfc.b.CreateLoad(GlobalCtxt.PointerType(0), arg0(), v.String()) lVal.SetOrdering(llvm.AtomicOrderingSequentiallyConsistent) case OpNilCheck: - lVal = lfc.explicitNilCheck(v) + lVal = lfc.emitNilCheckIntrinsic(v) case OpStore: lVal = lfc.b.CreateStore(arg1(), arg0()) case OpZero: @@ -1103,13 +1102,10 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { } func (lfc *LLVMFuncContext) CompileBlock(BB *Block) { - lfc.b.SetInsertPointAtEnd(lfc.BlockEnds[BB.ID]) + lfc.b.SetInsertPointAtEnd(lfc.BBs[BB.ID]) for _, v := range BB.Values { lfc.GenLV(v) } - // A nil check splits the LLVM block while retaining the original Go SSA - // block identity. Emit the Go block terminator from its current tail. - lfc.b.SetInsertPointAtEnd(lfc.BlockEnds[BB.ID]) switch BB.Kind { case BlockRet: if lfc.ResultCount == 0 { @@ -1159,7 +1155,6 @@ func LLVMCompile(f *Func) { cc := llvmCallConv(f.OwnAux.ABI().Which()) FCtxt := &LLVMFuncContext{ BBs: map[ID]llvm.BasicBlock{}, - BlockEnds: map[ID]llvm.BasicBlock{}, Vs: map[ID]llvm.Value{}, Locals: map[llvmLocalKey]llvmStackSlot{}, ItabMethods: map[ID]bool{}, @@ -1188,7 +1183,6 @@ func LLVMCompile(f *Func) { } for _, BB := range f.Blocks { FCtxt.BBs[BB.ID] = GlobalCtxt.AddBasicBlock(FCtxt.LF, BB.String()) - FCtxt.BlockEnds[BB.ID] = FCtxt.BBs[BB.ID] for _, v := range BB.Values { if (v.Op == OpInterCall || v.Op == OpInterLECall) && len(v.Args) != 0 { code := v.Args[0] @@ -1283,6 +1277,7 @@ func LLVMCompile(f *Func) { FCtxt.CompileBlock(BB) } FCtxt.FinishPhi() + FCtxt.expandNilCheckIntrinsics() FCtxt.MappingName() err := llvm.VerifyFunction(FCtxt.LF, llvm.PrintMessageAction) diff --git a/src/cmd/compile/internal/ssa/ssa2llvm_nilcheck.go b/src/cmd/compile/internal/ssa/ssa2llvm_nilcheck.go index 5d4e22af34835c..a82383b98dda15 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm_nilcheck.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm_nilcheck.go @@ -11,6 +11,7 @@ import ( ) const llvmPanicmemName = "runtime.panicmem" +const llvmNilCheckIntrinsicName = "llvm.goallc.nilcheck" // llvmPanicmem returns the compiler-owned Go object symbol and the only LLVM // signature accepted for runtime.panicmem. Unlike ordinary compiler-generated @@ -28,7 +29,11 @@ func llvmPanicmem() (llvm.Value, llvmFuncSignature) { return getOrInsertLLVMFunction(sym.Name, sig, goABIInternalCallConv), sig } -func (lfc *LLVMFuncContext) explicitNilCheck(v *Value) llvm.Value { +// emitNilCheckIntrinsic preserves the Go nil-check side effect without +// changing the CFG while ordinary SSA values and block terminators are being +// translated. expandNilCheckIntrinsics removes every marker before LLVM IR is +// verified, optimized, or emitted. +func (lfc *LLVMFuncContext) emitNilCheckIntrinsic(v *Value) llvm.Value { if len(v.Args) != 2 || !v.Args[1].Type.IsMemory() { v.Fatalf("NilCheck has invalid arguments") } @@ -45,36 +50,105 @@ func (lfc *LLVMFuncContext) explicitNilCheck(v *Value) llvm.Value { default: v.Fatalf("NilCheck address has unsupported LLVM type") } + sig := llvm.FunctionType(GlobalCtxt.VoidType(), []llvm.Type{checked.Type()}, false) + intrinsic := getOrInsertLLVMIntrinsic(llvmNilCheckIntrinsicName, sig) + lfc.b.CreateCall(sig, intrinsic, []llvm.Value{checked}, "") + return p +} - current := lfc.BlockEnds[v.Block.ID] - if current.IsNil() || lfc.b.GetInsertBlock() != current { - v.Fatalf("NilCheck is not emitted at the current LLVM block tail") +func (lfc *LLVMFuncContext) nilCheckMarkers(intrinsic llvm.Value) []llvm.Value { + var markers []llvm.Value + for _, bb := range lfc.F.Blocks { + for inst := lfc.BBs[bb.ID].FirstInstruction(); !inst.IsNil(); inst = llvm.NextInstruction(inst) { + if inst.IsACallInst().IsNil() || inst.CalledValue() != intrinsic { + continue + } + markers = append(markers, inst) + } + } + return markers +} + +// replacePhiPredecessor repairs successor phi nodes after a nil-check marker +// splits an already complete LLVM block. +func (lfc *LLVMFuncContext) replacePhiPredecessor(from, to llvm.BasicBlock) { + for _, bb := range lfc.F.Blocks { + for _, v := range bb.Values { + if v.Op != OpPhi || v.Type.IsMemory() { + continue + } + lfc.Vs[v.ID].ReplaceIncomingBlock(from, to) + } } - isNil := lfc.b.CreateICmp(llvm.IntEQ, checked, llvm.ConstNull(checked.Type()), v.String()+".isnil") - panicBlock := GlobalCtxt.AddBasicBlock(lfc.LF, v.String()+".nil") - continueBlock := GlobalCtxt.AddBasicBlock(lfc.LF, v.String()+".notnil") - lfc.b.CreateCondBr(isNil, panicBlock, continueBlock) - - lfc.b.SetInsertPointAtEnd(panicBlock) - panicmem, sig := llvmPanicmem() - call := lfc.b.CreateCall(sig.Type, panicmem, nil, "") - call.SetInstructionCallConv(goABIInternalCallConv) - // panicmem enters the runtime panic path and may reach GC while this frame is - // suspended, so it must remain an ordinary non-leaf call for statepoint and - // stack-map construction. - // runtime.panicmem ends in the Go panic builtin. A recovered panic resumes - // in the caller's deferred recovery path, never after this call. Retain an - // explicit edge to the continuation because the LLVM declaration does not - // claim noreturn. This first implementation deliberately accepts the - // conservative live set on that artificial edge; a future noreturn form - // must also keep the call return PC inside a valid frame/PCSP range. - lfc.b.CreateBr(continueBlock) +} + +// expandNilCheckIntrinsics lowers all temporary nil-check markers after the +// original LLVM CFG and phi nodes are complete. +func (lfc *LLVMFuncContext) expandNilCheckIntrinsics() { + intrinsic := CurrentModule.NamedFunction(llvmNilCheckIntrinsicName) + if intrinsic.IsNil() { + return + } + if intrinsic.BasicBlocksCount() != 0 { + lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "LLVM nil-check intrinsic has a definition") + } + expectedType := llvm.FunctionType(GlobalCtxt.VoidType(), []llvm.Type{GlobalCtxt.PointerType(0)}, false) + if intrinsic.GlobalValueType() != expectedType { + lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "LLVM nil-check intrinsic has an invalid type") + } + + markers := lfc.nilCheckMarkers(intrinsic) + b := GlobalCtxt.NewBuilder() + defer b.Dispose() + + for _, call := range markers { + if call.OperandsCount() != 2 || call.Operand(0).Type() != GlobalCtxt.PointerType(0) { + lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "LLVM nil-check intrinsic call has invalid operands") + } + before := call.InstructionParent() + if before.IsNil() || llvm.NextInstruction(call).IsNil() { + lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "LLVM nil-check intrinsic is not followed by a block terminator") + } + + panicBlock := GlobalCtxt.AddBasicBlock(lfc.LF, "nilcheck.nil") + continueBlock := GlobalCtxt.AddBasicBlock(lfc.LF, "nilcheck.notnil") + b.SetInsertPointAtEnd(continueBlock) + for inst := llvm.NextInstruction(call); !inst.IsNil(); { + next := llvm.NextInstruction(inst) + inst.RemoveFromParentAsInstruction() + b.Insert(inst) + inst = next + } + + checked := call.Operand(0) + call.EraseFromParentAsInstruction() + b.SetInsertPointAtEnd(before) + isNil := b.CreateICmp(llvm.IntEQ, checked, llvm.ConstNull(checked.Type()), "nilcheck.isnil") + b.CreateCondBr(isNil, panicBlock, continueBlock) + + b.SetInsertPointAtEnd(panicBlock) + panicmem, sig := llvmPanicmem() + panicCall := b.CreateCall(sig.Type, panicmem, nil, "") + panicCall.SetInstructionCallConv(goABIInternalCallConv) + // panicmem enters the runtime panic path and may reach GC while this + // frame is suspended, so it remains an ordinary non-leaf call for + // statepoint and stack-map construction. + // + // A recovered panic resumes in the caller, never after this call. Keep + // the conservative continuation edge until a future noreturn form can + // also guarantee a valid call return PC and PCSP range. + b.CreateBr(continueBlock) + + lfc.replacePhiPredecessor(before, continueBlock) + } + + if !intrinsic.FirstUse().IsNil() { + lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "unexpanded LLVM nil-check intrinsic use") + } + intrinsic.EraseFromParentAsFunction() // TODO(goallc): A later target-aware optimization may fold this explicit // branch and panic call into an implicit faulting nil check. It must first // prove Go panic ordering and recover semantics, a target-valid fault // offset, memory dependence, and the target's fault classification rules. - lfc.BlockEnds[v.Block.ID] = continueBlock - lfc.b.SetInsertPointAtEnd(continueBlock) - return p } diff --git a/src/cmd/go.mod b/src/cmd/go.mod index 2893846e18a1de..b7950049445d37 100644 --- a/src/cmd/go.mod +++ b/src/cmd/go.mod @@ -3,7 +3,7 @@ module cmd go 1.27 require ( - github.com/goallc/go-llvm v0.0.0-20260728032034-006181a193aa + github.com/goallc/go-llvm v0.0.0-20260802014221-e0a85f0a18bc github.com/google/pprof v0.0.0-20260507013755-92041b743c96 golang.org/x/arch v0.27.1-0.20260521044007-9c1a596a2c97 golang.org/x/build v0.0.0-20260522210304-d55d0041b921 diff --git a/src/cmd/go.sum b/src/cmd/go.sum index 1b6436eb737cd1..eb02792871ea3e 100644 --- a/src/cmd/go.sum +++ b/src/cmd/go.sum @@ -1,5 +1,5 @@ -github.com/goallc/go-llvm v0.0.0-20260728032034-006181a193aa h1:6RuEE6LD/tbgCuFTpeCqzu9FSD2y0L8ch1zkhZD25V8= -github.com/goallc/go-llvm v0.0.0-20260728032034-006181a193aa/go.mod h1:s70tU8u5zDo1LwIPqBxh6Qgol/tNt1kmn9caPfsTz8A= +github.com/goallc/go-llvm v0.0.0-20260802014221-e0a85f0a18bc h1:d8GeynggX+JI+ntwrs/9PyA3QYqYtNar6XqPSDEdoAE= +github.com/goallc/go-llvm v0.0.0-20260802014221-e0a85f0a18bc/go.mod h1:s70tU8u5zDo1LwIPqBxh6Qgol/tNt1kmn9caPfsTz8A= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20260507013755-92041b743c96 h1:YDDnaZ9afWajDboPMt9Vikqca/yWAX7KAxVzb4lJU1M= diff --git a/src/cmd/vendor/github.com/goallc/go-llvm/IRBindings.cpp b/src/cmd/vendor/github.com/goallc/go-llvm/IRBindings.cpp index be0a46636efe87..1a80406dbcd059 100644 --- a/src/cmd/vendor/github.com/goallc/go-llvm/IRBindings.cpp +++ b/src/cmd/vendor/github.com/goallc/go-llvm/IRBindings.cpp @@ -16,6 +16,7 @@ #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" @@ -50,6 +51,11 @@ void LLVMSetMetadata2(LLVMValueRef Inst, unsigned KindID, LLVMMetadataRef MD) { unwrap(Inst)->setMetadata(KindID, N); } +void LLVMGoReplaceIncomingBlock(LLVMValueRef Phi, LLVMBasicBlockRef Old, + LLVMBasicBlockRef New) { + unwrap(Phi)->replaceIncomingBlockWith(unwrap(Old), unwrap(New)); +} + void LLVMGoSetCurrentDebugLocation(LLVMBuilderRef Bref, unsigned Line, unsigned Col, LLVMMetadataRef Scope, LLVMMetadataRef InlinedAt) { diff --git a/src/cmd/vendor/github.com/goallc/go-llvm/IRBindings.h b/src/cmd/vendor/github.com/goallc/go-llvm/IRBindings.h index 80552dec2f6875..b14fca57705ac1 100644 --- a/src/cmd/vendor/github.com/goallc/go-llvm/IRBindings.h +++ b/src/cmd/vendor/github.com/goallc/go-llvm/IRBindings.h @@ -43,6 +43,9 @@ void LLVMAddNamedMetadataOperand2(LLVMModuleRef M, const char *name, LLVMMetadataRef Val); void LLVMSetMetadata2(LLVMValueRef Inst, unsigned KindID, LLVMMetadataRef MD); +void LLVMGoReplaceIncomingBlock(LLVMValueRef Phi, LLVMBasicBlockRef Old, + LLVMBasicBlockRef New); + void LLVMGoSetCurrentDebugLocation(LLVMBuilderRef Bref, unsigned Line, unsigned Col, LLVMMetadataRef Scope, LLVMMetadataRef InlinedAt); diff --git a/src/cmd/vendor/github.com/goallc/go-llvm/ir.go b/src/cmd/vendor/github.com/goallc/go-llvm/ir.go index d770b9dfd12100..d9f786b290ab9a 100644 --- a/src/cmd/vendor/github.com/goallc/go-llvm/ir.go +++ b/src/cmd/vendor/github.com/goallc/go-llvm/ir.go @@ -1260,6 +1260,9 @@ func (v Value) IncomingBlock(i int) (bb BasicBlock) { bb.C = C.LLVMGetIncomingBlock(v.C, C.unsigned(i)) return } +func (v Value) ReplaceIncomingBlock(old, new BasicBlock) { + C.LLVMGoReplaceIncomingBlock(v.C, old.C, new.C) +} // Operations on inline assembly func InlineAsm(t Type, asmString, constraints string, hasSideEffects, isAlignStack bool, dialect InlineAsmDialect, canThrow bool) (rv Value) { diff --git a/src/cmd/vendor/modules.txt b/src/cmd/vendor/modules.txt index f3dc49aab1f4ad..86e95c395fb8ac 100644 --- a/src/cmd/vendor/modules.txt +++ b/src/cmd/vendor/modules.txt @@ -1,4 +1,4 @@ -# github.com/goallc/go-llvm v0.0.0-20260728032034-006181a193aa +# github.com/goallc/go-llvm v0.0.0-20260802014221-e0a85f0a18bc ## explicit; go 1.24 github.com/goallc/go-llvm # github.com/google/pprof v0.0.0-20260507013755-92041b743c96 diff --git a/test/codegen/llvm_nilcheck.go b/test/codegen/llvm_nilcheck.go index fd0f6057a8dc22..779309073874d9 100644 --- a/test/codegen/llvm_nilcheck.go +++ b/test/codegen/llvm_nilcheck.go @@ -11,24 +11,27 @@ package codegen // predecessor used by the join phi. // // LLVM-LABEL: define goabiinternal i64 @codegen.llvmExplicitNilcheckPhi(ptr %p, i8 %take) +// LLVM-NOT: llvm.goallc.nilcheck // LLVM: phi i64 // LLVM: icmp eq ptr %p, null // LLVM: call goabiinternal void @runtime.panicmem() // LLVM: declare goabiinternal void @runtime.panicmem() // // LLVM-LABEL: define goabiinternal i64 @codegen.llvmExplicitNilcheckTwice(ptr %p, ptr %q) +// LLVM-NOT: llvm.goallc.nilcheck // LLVM: icmp eq ptr %p, null // LLVM: call goabiinternal void @runtime.panicmem() -// LLVM: br label %[[FIRST_CONT:.*\.notnil]] +// LLVM: br label %[[FIRST_CONT:nilcheck\.notnil[0-9]*]] // LLVM: [[FIRST_CONT]]: // LLVM: load i64, ptr %p // LLVM: icmp eq ptr %q, null // LLVM: call goabiinternal void @runtime.panicmem() -// LLVM: br label %[[SECOND_CONT:.*\.notnil]] +// LLVM: br label %[[SECOND_CONT:nilcheck\.notnil[0-9]*]] // LLVM: [[SECOND_CONT]]: // LLVM: load i64, ptr %q // // LLVM-LABEL: define goabiinternal i64 @codegen.llvmExplicitNilcheck(ptr %p) +// LLVM-NOT: llvm.goallc.nilcheck // LLVM: [[ISNIL:%.*]] = icmp eq ptr %p, null // LLVM-NEXT: br i1 [[ISNIL]], label %[[NIL:.*\.nil]], label %[[CONT:.*\.notnil]] // LLVM: [[NIL]]: @@ -40,8 +43,10 @@ package codegen // LLVM-NOT: !annotation // LLVM: load i64, ptr %p // LLVM-NOT: "gc-leaf-function" +// LLVM-NOT: llvm.goallc.nilcheck // // LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmExplicitNilcheck( +// LLVM-OPT-NOT: llvm.goallc.nilcheck // LLVM-OPT: icmp eq ptr %p, null // LLVM-OPT: br i1 {{%.*}}, label %[[OPTNIL:.*]], label %[[OPTCONT:.*]] // LLVM-OPT: [[OPTNIL]]: @@ -52,6 +57,7 @@ package codegen // LLVM-OPT-NOT: !goallc.nilcheck // LLVM-OPT-NOT: !annotation // LLVM-OPT: load i64, ptr %p +// LLVM-OPT-NOT: llvm.goallc.nilcheck func llvmExplicitNilcheck(p *int) int { return *p }