cmd/compile: add first-phase defer lowering for LLVM

src/cmd/compile/internal/ssa/llvmdata.go

@@ -520,6 +520,24 @@ func setGoObjFunctionFlags(fn llvm.Value, s *obj.LSym) {
 	}))
 }
 
+// FuncIDWrapper is part of recover's semantic stack walk: a deferred-call
+// wrapper must not count as an extra frame between gopanic and recover. Carry
+// both FuncID and FuncFlag into LLVM so the GoObj writer can reproduce the
+// native compiler's FuncInfo record.
+func setGoObjFunctionInfo(fn llvm.Value, s *obj.LSym) {
+	info := s.Func()
+	if info == nil {
+		base.Fatalf("missing Go function info for %s", s.Name)
+	}
+	if info.FuncID == 0 && info.FuncFlag == 0 {
+		return
+	}
+	fn.SetGlobalMetadata(GlobalCtxt.MDKindID("goobj.func.info"), GlobalCtxt.MDNode([]llvm.Metadata{
+		llvm.ConstInt(GlobalCtxt.Int8Type(), uint64(info.FuncID), false).ConstantAsMetadata(),
+		llvm.ConstInt(GlobalCtxt.Int8Type(), uint64(info.FuncFlag), false).ConstantAsMetadata(),
+	}))
+}
+
 // LLVM can express the address relationship but not GoObj's 32-bit section
 // offsets. Record the object-format-specific relocation type explicitly;
 // weakness remains orthogonal in !goobj.weak_relocs.

src/cmd/compile/internal/ssa/ssa2llvm.go

@@ -61,6 +61,7 @@ const goGCLeafFunctionAttr = "gc-leaf-function"
 const goStackGrowthStatepointAttr = "go-stack-growth-statepoint"
 const goAsyncUnsafeAttr = "go-async-unsafe"
 const goWriteBarrierIntrinsic = "llvm.go.gc.write.barrier"
+const goDeferEdgeIntrinsic = "llvm.go.defer.edge"
 const goSourceAddressTakenMD = "goallc.source_addrtaken"
 const llvmFramePointerAttr = "frame-pointer"
 const llvmFramePointerNonLeaf = "non-leaf"
@@ -262,6 +263,14 @@ func getOrInsertLLVMIntrinsic(name string, typ llvm.Type) llvm.Value {
 	return fn
 }
 
+func getLLVMIntrinsicDeclaration(name string) llvm.Value {
+	id := llvm.LookupIntrinsicID(name)
+	if id == 0 {
+		base.Fatalf("unknown LLVM intrinsic %s", name)
+	}
+	return llvm.GetIntrinsicDeclaration(CurrentModule, id, nil)
+}
+
 func (lfc *LLVMFuncContext) llvmLifetimeStart(slot llvmStackSlot) {
 	sig := llvm.FunctionType(
 		GlobalCtxt.VoidType(),
@@ -1194,13 +1203,20 @@ func llvmStaticCallSignature(v *Value, aux *AuxCall, sig llvmFuncSignature) llvm
 		return sig
 	}
 	wantArgs := int64(0)
+	pointerArgs := int64(0)
 	switch aux.Fn {
-	case ir.Syms.Newproc:
+	case ir.Syms.Newproc, ir.Syms.Deferproc, ir.Syms.DeferprocStack:
 		wantArgs = 1
+		pointerArgs = 1
+	case ir.Syms.Deferprocat:
+		wantArgs = 2
+		pointerArgs = 1
 	case ir.Syms.WBZero:
 		wantArgs = 2
+		pointerArgs = wantArgs
 	case ir.Syms.WBMove:
 		wantArgs = 3
+		pointerArgs = wantArgs
 	default:
 		return sig
 	}
@@ -1210,16 +1226,27 @@ func llvmStaticCallSignature(v *Value, aux *AuxCall, sig llvmFuncSignature) llvm
 	if aux.NArgs() != wantArgs || aux.NResults() != 0 {
 		v.Fatalf("%s has unexpected raw call signature: %d arguments, %d results", aux.Fn.Name, aux.NArgs(), aux.NResults())
 	}
-	if aux.Fn == ir.Syms.Newproc && (len(v.Args) != 2 || v.Args[0].Type == nil || !v.Args[0].Type.IsPtrShaped()) {
-		v.Fatalf("runtime.newproc argument is not pointer-shaped")
-	}
-	for i := int64(0); i < aux.NArgs(); i++ {
+	for i := int64(0); i < pointerArgs; i++ {
+		if int(i) >= len(v.Args)-1 || v.Args[i].Type == nil {
+			v.Fatalf("argument %d to %s is not pointer-shaped", i, aux.Fn.Name)
+		}
+		pointerShaped := v.Args[i].Type.IsPtrShaped()
+		// Write-barrier calls carry the type descriptor as Addr<uintptr> in
+		// Go SSA because AuxCall uses uintptr for its physical ABI assignment.
+		// OpAddr still lowers directly to an LLVM pointer, just like the
+		// pointer-shaped destination and source operands.
+		writeBarrierTypeAddr := i == 0 &&
+			(aux.Fn == ir.Syms.WBZero || aux.Fn == ir.Syms.WBMove) &&
+			v.Args[i].Op == OpAddr && v.Args[i].Type.IsUintptr()
+		if !pointerShaped && !writeBarrierTypeAddr {
+			v.Fatalf("argument %d to %s is not pointer-shaped", i, aux.Fn.Name)
+		}
 		if typ := aux.TypeOfArg(i); typ == nil || !typ.IsUintptr() {
 			v.Fatalf("argument %d to %s is not raw uintptr", i, aux.Fn.Name)
 		}
 	}
 	params := append([]llvm.Type(nil), sig.Type.ParamTypes()...)
-	for i := range params {
+	for i := int64(0); i < pointerArgs; i++ {
 		params[i] = GlobalCtxt.PointerType(0)
 	}
 	sig.Type = llvm.FunctionType(sig.ReturnType, params, false)
@@ -2027,6 +2054,17 @@ func (lfc *LLVMFuncContext) CompileBlock(BB *Block, values []*Value) {
 	case BlockIf:
 		cond := lfc.llvmCondition(lfc.GenLV(BB.Controls[0]), BB.String()+".cond")
 		lfc.b.CreateCondBr(cond, lfc.BBs[BB.Succs[0].Block().ID], lfc.BBs[BB.Succs[1].Block().ID])
+	case BlockDefer:
+		if len(BB.Succs) != 2 || BB.NumControls() != 1 || !BB.Controls[0].Type.IsMemory() {
+			BB.Func.fe.Fatalf(BB.Pos, "invalid LLVM defer block %s", BB)
+		}
+		deferEdge := getLLVMIntrinsicDeclaration(goDeferEdgeIntrinsic)
+		lfc.b.CreateCallBr(
+			deferEdge.GlobalValueType(), deferEdge, nil,
+			lfc.BBs[BB.Succs[0].Block().ID],
+			[]llvm.BasicBlock{lfc.BBs[BB.Succs[1].Block().ID]},
+			"",
+		)
 	case BlockPlain:
 		lfc.b.CreateBr(lfc.BBs[BB.Succs[0].Block().ID])
 	case BlockExit:
@@ -2132,6 +2170,7 @@ func LLVMCompile(f *Func) {
 	}
 	FCtxt.LF.SetGC(goGCStrategy)
 	setGoObjFunctionFlags(FCtxt.LF, f.OwnAux.Fn)
+	setGoObjFunctionInfo(FCtxt.LF, f.OwnAux.Fn)
 	inParams := f.OwnAux.ABIInfo().InParams()
 	if got, want := len(inParams), int(f.OwnAux.NArgs()); got != want {
 		f.fe.Fatalf(f.Entry.Pos, "LLVM parameter metadata count %d does not match signature count %d for %s", got, want, f.Name)

src/cmd/compile/internal/ssagen/ssa.go

@@ -395,6 +395,11 @@ func buildssa(fn *ir.Func, worker int, isPgoHot bool) *ssa.Func {
 
 	s.hasOpenDefers = base.Flag.N == 0 && s.hasdefer && !s.curfn.OpenCodedDeferDisallowed()
 	switch {
+	case base.Flag.EnableLLVM:
+		// The first LLVM defer implementation models the runtime registration
+		// calls and their recovery edges directly. Open-coded defers additionally
+		// require bitmap and FUNCDATA lowering, so keep them on the classic path.
+		s.hasOpenDefers = false
 	case base.Debug.NoOpenDefer != 0:
 		s.hasOpenDefers = false
 	case s.hasOpenDefers && (base.Ctxt.Flag_shared || base.Ctxt.Flag_dynlink) && base.Ctxt.Arch.Name == "386":

src/cmd/llvmplugin/CMakeLists.txt

@@ -82,6 +82,34 @@ if(BUILD_TESTING)
       "slot.relocated"
   )
 
+  add_test(
+    NAME GoALLCStatepoints.DeferEdge
+    COMMAND
+      "${GOALLC_LLC_EXECUTABLE}"
+      "-load-pass-plugin=$<TARGET_FILE:GoALLCStatepoints>"
+      -goallc-pass-plugin-emit-ir
+      -filetype=null
+      -o -
+      "${CMAKE_CURRENT_SOURCE_DIR}/testdata/defer-edge.ll"
+  )
+  set_tests_properties(GoALLCStatepoints.DeferEdge PROPERTIES
+    PASS_REGULAR_EXPRESSION
+      "callbr void @llvm.go.defer.edge"
+    FAIL_REGULAR_EXPRESSION
+      "GoALLC statepoints do not yet support invoke or callbr;call goabiinternal void @runtime.deferproc"
+  )
+
+  add_test(
+    NAME GoALLCStatepoints.DeferEdgeGoObj
+    COMMAND
+      "${GOALLC_LLC_EXECUTABLE}"
+      "-load-pass-plugin=$<TARGET_FILE:GoALLCStatepoints>"
+      -verify-machineinstrs
+      -filetype=obj
+      -o "${CMAKE_CURRENT_BINARY_DIR}/defer-edge.goobj"
+      "${CMAKE_CURRENT_SOURCE_DIR}/testdata/defer-edge.ll"
+  )
+
   add_test(
     NAME GoALLCStatepoints.FrontendMarkersPreserved
     COMMAND
@@ -282,6 +310,18 @@ if(BUILD_TESTING)
         "GOALLC_OBJVIEW_EXECUTABLE does not exist: "
         "${GOALLC_OBJVIEW_EXECUTABLE}")
     endif()
+    add_test(
+      NAME GoALLCStatepoints.DeferEdgeObjView
+      COMMAND
+        "${Python3_EXECUTABLE}"
+        "${CMAKE_CURRENT_SOURCE_DIR}/testdata/check-defer-edge.py"
+        --objview "${GOALLC_OBJVIEW_EXECUTABLE}"
+        --object "${CMAKE_CURRENT_BINARY_DIR}/defer-edge.goobj"
+    )
+    set_tests_properties(GoALLCStatepoints.DeferEdgeObjView PROPERTIES
+      DEPENDS GoALLCStatepoints.DeferEdgeGoObj
+    )
+
     add_test(
       NAME GoALLCStatepoints.MultipleCallsObjView
       COMMAND

src/cmd/llvmplugin/testdata/check-defer-edge.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import subprocess
+import sys
+
+
+FUNCTION = "defer_edge"
+WRAPPER = "defer_wrapper"
+
+
+def fail(message):
+    raise RuntimeError(message)
+
+
+def only(items, predicate, description):
+    matches = [item for item in items if predicate(item)]
+    if len(matches) != 1:
+        fail(f"found {len(matches)} {description}, want 1")
+    return matches[0]
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--objview", required=True)
+    parser.add_argument("--object", required=True)
+    args = parser.parse_args()
+
+    result = subprocess.run(
+        [args.objview, "-format=json", args.object],
+        check=True,
+        stdout=subprocess.PIPE,
+        text=True,
+    )
+    document = json.loads(result.stdout)
+    objects = [
+        member["go_object"]
+        for member in document["members"]
+        if member.get("go_object") is not None
+    ]
+    obj = only(objects, lambda item: True, "Go objects")
+    function = only(
+        obj["symbols"],
+        lambda item: item["name"] == FUNCTION,
+        f"{FUNCTION} symbols",
+    )
+    metadata = function.get("function")
+    if metadata is None:
+        fail(f"{FUNCTION} has no function metadata")
+
+    wrapper = only(
+        obj["symbols"],
+        lambda item: item["name"] == WRAPPER,
+        f"{WRAPPER} symbols",
+    )
+    wrapper_metadata = wrapper.get("function")
+    if wrapper_metadata is None or wrapper_metadata.get("info") is None:
+        fail(f"{WRAPPER} has no function info")
+    wrapper_info = wrapper_metadata["info"]
+    if wrapper_info["func_id"] != 23 or wrapper_info["func_flags"] != 0:
+        fail(f"{WRAPPER} lost GoObj wrapper identity: {wrapper_info}")
+
+    references = {item["index"]: item["name"] for item in obj["references"]}
+
+    def target_name(relocation):
+        target = relocation["target"]
+        if "name" in target:
+            return target["name"]
+        return references.get(target["sym_index"])
+
+    deferreturn = only(
+        function["relocations"],
+        lambda item: item["type"] == "R_CALL"
+        and target_name(item) == "runtime.deferreturn",
+        "direct runtime.deferreturn relocations",
+    )
+    query = only(
+        metadata["stack_map_queries"],
+        lambda item: item["call_offset"] == deferreturn["offset"],
+        "runtime.deferreturn stack-map queries",
+    )
+    if query["relocation_type"] != "R_CALL":
+        fail(f"deferreturn is not represented as a direct call: {query}")
+    if query["stack_map_index"] < 0:
+        fail(f"deferreturn has no valid GC stack map: {query}")
+
+    print(
+        f"{FUNCTION}: runtime.deferreturn R_CALL at {deferreturn['offset']}, "
+        f"stack map {query['stack_map_index']}; {WRAPPER}: FuncID 23, flags 0"
+    )
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (KeyError, RuntimeError, subprocess.CalledProcessError, json.JSONDecodeError) as error:
+        print(f"check-defer-edge: {error}", file=sys.stderr)
+        sys.exit(1)

src/cmd/llvmplugin/testdata/defer-edge.ll

@@ -0,0 +1,27 @@
+target triple = "x86_64-unknown-linux-goobj"
+
+declare goabiinternal void @runtime.deferproc()
+declare goabiinternal void @runtime.deferreturn()
+declare void @llvm.go.defer.edge()
+
+define goabiinternal void @defer_edge() #0 gc "goallc" {
+entry:
+  call goabiinternal void @runtime.deferproc()
+  callbr void @llvm.go.defer.edge() to label %normal [label %recover]
+
+normal:
+  ret void
+
+recover:
+  call goabiinternal void @runtime.deferreturn()
+  ret void
+}
+
+define goabiinternal void @defer_wrapper() !goobj.func.info !0 {
+entry:
+  ret void
+}
+
+attributes #0 = { "go-stack-growth-statepoint" }
+
+!0 = !{i8 23, i8 0}