coro: compact frames and native allocation classes

cl/_testgo/allocinloop/in.go

@@ -32,8 +32,9 @@ func Test() {
 }
 
 // CHECK-LABEL: define ptr @"main.main$coro"(ptr %0, ptr %1){{.*}} {
-// CHECK: call ptr @"main.Test$coro"
-// CHECK: call i1 @__llgo_coro_await_prepare_inline_v4
+// CHECK-NEXT: _llgo_0:
+// CHECK-NEXT: %[[HANDLE:[0-9]+]] = call ptr @"main.Test$coro"(ptr %0, ptr %1)
+// CHECK-NEXT: ret ptr %[[HANDLE]]
 func main() {
 	Test()
 }

cl/coro_abi.go

@@ -146,7 +146,6 @@ const (
 	coroCompletePrepareHookV2                  = "__llgo_coro_complete_prepare_v2"
 	coroFrameFreeHookV1                        = "__llgo_coro_frame_free_v1"
 	coroDescriptorPrefixV1                     = "__llgo_coro_frame_descriptor_v1."
-	coroBorrowedFrameMetadataWordsV2           = 20
 )
 
 const (
@@ -163,9 +162,7 @@ const (
 	coroHeaderResultSlot
 	coroHeaderSuspendReason
 	coroHeaderLifecycle
-	coroHeaderStateID
 	coroHeaderLine
-	coroHeaderFlags
 )
 
 const (
@@ -197,7 +194,6 @@ const coroPreemptInstructionBudget = 64
 // storage whose address never escapes; the ordinary SROA/mem2reg pipeline can
 // therefore keep it in SSA registers on a non-suspending loop edge. Every
 // activation resets it, so it is not live across a scheduler-visible suspend.
-// StateID remains exclusively the published resume-state identity.
 const coroPreemptCheckpointStride uint64 = 2048
 
 type coroPhysicalABI struct {
@@ -258,14 +254,22 @@ type coroBodyContext struct {
 	completePrepare        llssa.Expr
 	terminalStatus         llssa.Expr
 	preemptCountdown       llssa.Expr
-	nextState              uint32
-	terminalState          uint32
-	needsPreempt           bool
-	instructions           int
-	frameRetention         *coroFrameRetentionProof
-	critical               *coroCriticalProof
-	terminalResultAllocs   map[*ssa.Alloc]llssa.Expr
-	sourceBlockPollFresh   bool
+	// outcomeScratch is the one frame-local status/interface record shared by
+	// every fully consumed managed child transaction. Source execution cannot
+	// overlap two calls in one physical frame: a coroutine child suspends its
+	// parent until the old outcome is consumed, while an outcome-plain child
+	// returns synchronously. Keeping one directly addressed record avoids one
+	// permanent CoroSplit field per call site without adding a runtime lookup or
+	// dynamic lifetime protocol.
+	outcomeScratch       llssa.Expr
+	nextState            uint32
+	terminalState        uint32
+	needsPreempt         bool
+	instructions         int
+	frameRetention       *coroFrameRetentionProof
+	critical             *coroCriticalProof
+	terminalResultAllocs map[*ssa.Alloc]llssa.Expr
+	sourceBlockPollFresh bool
 }
 
 func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *types.Signature) coroPhysicalABI {
@@ -461,12 +465,20 @@ func coroHeaderType(prog llssa.Program) llssa.Type {
 		prog.VoidPtr(), // result slot
 		prog.Uint16(),  // suspend reason
 		prog.Uint16(),  // lifecycle state
-		prog.Uint32(),  // state ID
 		prog.Uint32(),  // source line
-		prog.Uint32(),  // flags
 	)
 }
 
+func coroBorrowedFrameMetadataWordsV2(prog llssa.Program) int64 {
+	pointerSize := prog.PointerSize()
+	if pointerSize != 4 && pointerSize != 8 {
+		panic("coroutine frame metadata requires a 32-bit or 64-bit pointer target")
+	}
+	// Mirrors runtime/internal/coro.BorrowedFrameStorageV2. The native Frame is
+	// twelve words; pointer-32 needs one extra word for its uint32 status field.
+	return int64(12 + 4/pointerSize)
+}
+
 func (p *context) beginCoroBody(
 	b llssa.Builder,
 	abi coroPhysicalABI,
@@ -488,11 +500,11 @@ func (p *context) beginCoroBody(
 	headerType := coroHeaderType(prog)
 	header := b.AllocaT(headerType)
 	borrowedFrameMetadataType := p.type_(
-		types.NewArray(types.Typ[types.Uintptr], coroBorrowedFrameMetadataWordsV2),
+		types.NewArray(types.Typ[types.Uintptr], coroBorrowedFrameMetadataWordsV2(prog)),
 		llssa.InGo,
 	)
 	// Dynamic ramps never consume this fallback storage. Leave it uninitialized
-	// here so every ordinary coroutine creation does not pay a 20-word memset;
+	// here so every ordinary coroutine creation does not pay a metadata memset;
 	// PublishFrameV2 initializes the complete private Frame only when LLVM has
 	// actually selected the allocation-elided path (storage == nil).
 	borrowedFrameMetadata := b.AllocaT(borrowedFrameMetadataType)
@@ -533,10 +545,8 @@ func (p *context) beginCoroBody(
 		// managed calls and receive their ordinary Return outcomes.
 		body.terminalStatus = b.AllocaT(prog.Uint32())
 		b.Store(body.terminalStatus, prog.IntVal(coroAwaitCompletionReturn, prog.Uint32()))
-		// This address is compiler-private and never reaches a runtime call. It
-		// deliberately differs from Header.StateID: that externally visible
-		// field aliases runtime validation calls and therefore forces a
-		// load/store on every otherwise plain loop edge.
+		// This address is compiler-private and never reaches a runtime call, so
+		// ordinary SROA can keep it in SSA registers on non-suspending edges.
 		body.preemptCountdown = b.AllocaT(prog.Uint32())
 	}
 	if abi.runDecisionTakeZeroHook != "" {
@@ -820,12 +830,11 @@ func coroPanicTraceReplaceSignature() *types.Signature {
 func (c *coroBodyContext) publishState(
 	b llssa.Builder,
 	reason, lifecycle uint64,
-	stateID, line uint32,
+	_ uint32, line uint32,
 ) {
 	prog := b.Prog
 	b.Store(b.FieldAddr(c.header, coroHeaderSuspendReason), prog.IntVal(reason, prog.Uint16()))
 	b.Store(b.FieldAddr(c.header, coroHeaderLifecycle), prog.IntVal(lifecycle, prog.Uint16()))
-	b.Store(b.FieldAddr(c.header, coroHeaderStateID), prog.IntVal(uint64(stateID), prog.Uint32()))
 	b.Store(b.FieldAddr(c.header, coroHeaderLine), prog.IntVal(uint64(line), prog.Uint32()))
 }
 
@@ -1121,7 +1130,6 @@ func (c *coroBodyContext) panicWithLine(
 	prog := b.Prog
 	b.Store(b.FieldAddr(c.header, coroHeaderSuspendReason), prog.IntVal(coroSuspendPanic, prog.Uint16()))
 	b.Store(b.FieldAddr(c.header, coroHeaderLifecycle), prog.IntVal(coroLifecycleFinalSuspended, prog.Uint16()))
-	b.Store(b.FieldAddr(c.header, coroHeaderStateID), prog.IntVal(uint64(c.terminalStateID()), prog.Uint32()))
 	b.Store(b.FieldAddr(c.header, coroHeaderLine), line)
 	b.Call(
 		c.panicPrepare,
@@ -1176,6 +1184,10 @@ func (p *context) compileCoroPhysicalBody(b llssa.Builder, fn *ssa.Function, abi
 	if err != nil {
 		panic(fmt.Errorf("load frozen coroutine physical plan: %w", err))
 	}
+	if physicalPlan.tailForward != nil {
+		p.compileCoroTailForwardPhysicalBody(b, fn, physicalPlan, sourceParamBase)
+		return
+	}
 	frameRetention := physicalPlan.frameRetention
 	critical := physicalPlan.critical
 	cleanupPlan := physicalPlan.cleanup
@@ -1300,6 +1312,55 @@ func (p *context) compileCoroPhysicalBody(b llssa.Builder, fn *ssa.Function, abi
 	emission.completeManagedPhysicalBody(bodyCapability)
 }
 
+// compileCoroTailForwardPhysicalBody emits a stable coroutine entry ramp that
+// owns no coroutine frame. The frozen physical plan guarantees that its source
+// body is exactly one static call followed by an unchanged return, so passing
+// the task and result slot straight through preserves every terminal and
+// suspension protocol while avoiding a redundant parent await transaction.
+func (p *context) compileCoroTailForwardPhysicalBody(
+	b llssa.Builder,
+	function *ssa.Function,
+	physical *coroPhysicalFunctionPlan,
+	sourceParamBase int,
+) {
+	if p == nil || b == nil || b.Func != p.fn || function == nil || physical == nil ||
+		physical.function != function || physical.tailForward == nil ||
+		p.compilation == nil || p.immutablePlan() == nil || sourceParamBase < 2 {
+		panic("coroutine tail-forward emission requires one exact frozen physical plan")
+	}
+	forward := physical.tailForward
+	if err := forward.validate(function, p.immutablePlan()); err != nil {
+		panic(fmt.Errorf("validate frozen coroutine tail-forward plan: %w", err))
+	}
+	if sourceParamBase != 2 {
+		panic("coroutine tail-forward source unexpectedly has a closure environment")
+	}
+
+	b.SetBlock(p.fn.Block(0))
+	entry := p.mustFunctionSymbol(forward.target)
+	if entry.plan.ID != forward.targetID || !entry.usesCoroPhysicalABI() {
+		panic("coroutine tail-forward target no longer resolves to its frozen physical entry")
+	}
+	target, _, kind := p.compileFunctionEntry(entry)
+	if kind != goFunc || target == nil {
+		panic("coroutine tail-forward target did not resolve to a Go coroutine entry")
+	}
+	args := make([]llssa.Expr, 0, len(forward.args)+2)
+	args = append(args, p.fn.PhysicalParam(0), p.fn.PhysicalParam(1))
+	for _, argument := range forward.args {
+		if argument.sourceParameter >= 0 {
+			args = append(args, p.fn.PhysicalParam(sourceParamBase+argument.sourceParameter))
+			continue
+		}
+		args = append(args, p.compileValueAs(b, argument.constant, argument.targetType))
+	}
+	handle := b.Call(target.Expr, args...)
+	if handle.Type == nil || !types.Identical(handle.RawType(), types.Typ[types.UnsafePointer]) {
+		panic("coroutine tail-forward target returned a non-handle value")
+	}
+	b.Return(handle)
+}
+
 // validateCoroExactSyntheticForwarder proves the complete SSA body shared by
 // compiler-owned spawn carriers. The caller remains responsible for proving
 // why this exact value may be wrapped; this helper proves only that the

cl/coro_abi_test.go

@@ -316,6 +316,73 @@ func TestCoroChildAwaitPhysicalABIV1Presplit(t *testing.T) {
 	}
 }
 
+func TestCoroExactTailForwardReusesTargetHandle(t *testing.T) {
+	const source = `package foo
+func Child(value uint32, delta int) {}
+func Parent(value uint32) { Child(value, -1) }
+`
+	prog, ssaPkg, files, universe, plan := prepareCoroChildAwaitPhysicalABISource(t, nil, source)
+	defer prog.Dispose()
+	compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe}
+	enableCoroChildAwaitCompilation(compilation)
+	pkg, _, err := NewPackageExWithEmbedOptions(
+		prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{},
+		PackageOptions{Compilation: compilation},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	module := pkg.Module()
+	defer module.Dispose()
+	if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil {
+		t.Fatalf("verify tail-forward coroutine: %v\n%s", err, module.String())
+	}
+
+	parentSSA, childSSA := ssaPkg.Func("Parent"), ssaPkg.Func("Child")
+	owner := universe.ownerOf(parentSSA)
+	physical, err := universe.coroProgramIR.physicalFunctionPlan(parentSSA, owner)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if physical.tailForward == nil || physical.tailForward.target != childSSA ||
+		len(physical.tailForward.args) != 2 ||
+		physical.tailForward.args[0].sourceParameter != 0 ||
+		physical.tailForward.args[1].constant == nil ||
+		physical.tailForward.args[1].constant.Int64() != -1 {
+		t.Fatalf("tail-forward physical plan = %+v; want Parent(value)->Child(value,-1)", physical.tailForward)
+	}
+
+	parent := requireCoroPhysicalFunction(t, module, "foo.Parent")
+	child := requireCoroPhysicalFunction(t, module, "foo.Child")
+	parentIR, childIR := parent.String(), child.String()
+	if strings.Contains(parentIR, "llvm.coro.") ||
+		strings.Contains(parentIR, coroFrameAllocHookV1) ||
+		strings.Contains(parentIR, coroAwaitPrepareInlineHookV4) {
+		t.Fatalf("tail-forward ramp retained a coroutine frame or await transaction:\n%s", parentIR)
+	}
+	if !regexp.MustCompile(`call ptr @"?foo\.Child\$coro"?\(ptr [^,]+, ptr [^,]+, i32 [^,]+, i64 -1\)`).MatchString(parentIR) {
+		t.Fatalf("tail-forward ramp did not pass task/result/parameter/constant directly:\n%s", parentIR)
+	}
+	if strings.Count(parentIR, "call ptr") != 1 || !strings.Contains(parentIR, "ret ptr") {
+		t.Fatalf("tail-forward ramp is not one target call plus handle return:\n%s", parentIR)
+	}
+	if !strings.Contains(childIR, "llvm.coro.begin") ||
+		!strings.Contains(childIR, coroFrameAllocHookV1) {
+		t.Fatalf("tail-forward target lost its physical coroutine body:\n%s", childIR)
+	}
+
+	runCoroABITestPipeline(t, prog, module)
+	if module.NamedFunction("foo.Parent$coro.resume").IsNil() == false ||
+		module.NamedFunction("foo.Parent$coro.destroy").IsNil() == false {
+		t.Fatalf("frame-free tail-forward ramp acquired split resume/destroy entries:\n%s", module.String())
+	}
+	for _, suffix := range []string{".resume", ".destroy"} {
+		if module.NamedFunction("foo.Child$coro" + suffix).IsNil() {
+			t.Fatalf("tail-forward target lost split %s entry:\n%s", suffix, module.String())
+		}
+	}
+}
+
 func TestCoroNamedFunctionTypeDirectAwait(t *testing.T) {
 	const source = `package foo
 type Task func(uint32) uint32
@@ -801,6 +868,13 @@ func TestCoroPhysicalValueTransportABIV1NativeAndWasm(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
+			parentPhysical, err := universe.coroProgramIR.physicalFunctionPlan(parent, universe.ownerOf(parent))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if parentPhysical.tailForward == nil || parentPhysical.tailForward.target != child {
+				t.Fatalf("Parent physical tail forward = %+v; want exact Child target", parentPhysical.tailForward)
+			}
 			module := pkg.Module()
 			defer module.Dispose()
 			if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil {
@@ -823,12 +897,17 @@ func TestCoroPhysicalValueTransportABIV1NativeAndWasm(t *testing.T) {
 
 			runCoroABITestPipeline(t, prog, module)
 			post := module.String()
-			for _, function := range []string{"foo.Child$coro", "foo.Parent$coro", "foo.Pair$coro"} {
+			for _, function := range []string{"foo.Child$coro", "foo.Pair$coro"} {
 				for _, suffix := range []string{".resume", ".destroy"} {
 					if module.NamedFunction(function + suffix).IsNil() {
 						t.Fatalf("CoroSplit did not create %s%s:\n%s", function, suffix, post)
 					}
 				}
+				for _, suffix := range []string{".resume", ".destroy"} {
+					if !module.NamedFunction("foo.Parent$coro" + suffix).IsNil() {
+						t.Fatalf("frame-free Parent tail-forward acquired %s:\n%s", suffix, post)
+					}
+				}
 			}
 			assertCoroResultSlotFields(t, "Pair after CoroSplit", module.NamedFunction("foo.Pair$coro.resume").String(), uintptrIR)
 			if !regexp.MustCompile(`store %foo\.Payload [^,]+, ptr `).MatchString(module.NamedFunction("foo.Child$coro.resume").String()) {
@@ -2171,10 +2250,9 @@ func assertCoroV0HeaderStateZero(t *testing.T, body string) {
 	}{
 		{index: coroHeaderSuspendReason, type_: "i16", name: "suspend reason"},
 		{index: coroHeaderLifecycle, type_: "i16", name: "lifecycle"},
-		{index: coroHeaderStateID, type_: "i32", name: "state ID"},
 	} {
 		addresses := regexp.MustCompile(
-			`(?m)^\s*(%[-a-zA-Z$._0-9]+) = getelementptr[^\n{]* \{ ptr, ptr, ptr, ptr, ptr, i16, i16, i32, i32, i32 \}, ptr [^,]+, i32 0, i32 `+strconv.Itoa(field.index)+`\s*$`,
+			`(?m)^\s*(%[-a-zA-Z$._0-9]+) = getelementptr[^\n{]* \{ ptr, ptr, ptr, ptr, ptr, i16, i16, i32 \}, ptr [^,]+, i32 0, i32 `+strconv.Itoa(field.index)+`\s*$`,
 		).FindAllStringSubmatch(body, -1)
 		if len(addresses) == 0 {
 			t.Fatalf("v0 coroutine has no header %s store:\n%s", field.name, body)
@@ -2710,9 +2788,9 @@ func assertCoroV1Completion(t *testing.T, name, body string) {
 		t.Fatalf("%s does not prepare completion before final suspend:\n%s", name, body)
 	}
 	segment := body[:complete]
-	state := regexp.MustCompile(`(?s)store i16 2,.*store i16 4,.*store i32 [1-9][0-9]*,`)
+	state := regexp.MustCompile(`(?s)store i16 2,.*store i16 4,.*store i32 0,`)
 	if !state.MatchString(segment) {
-		t.Fatalf("%s does not publish final reason/lifecycle/stateID before completion preparation:\n%s", name, body)
+		t.Fatalf("%s does not publish final reason/lifecycle and clear its source line before completion preparation:\n%s", name, body)
 	}
 }
 
@@ -2739,9 +2817,9 @@ func assertCoroStaticChildAwait(t *testing.T, parent string) {
 	if !parentLink.MatchString(prefix) {
 		t.Fatalf("Parent does not store its handle into child.parent before handoff:\n%s", prefix)
 	}
-	state := regexp.MustCompile(`(?s)store i16 1,.*store i16 3,.*store i32 1,`)
+	state := regexp.MustCompile(`(?s)store i16 1,.*store i16 3,.*store i32 [1-9][0-9]*,`)
 	if !state.MatchString(prefix) {
-		t.Fatalf("Parent does not publish Call/Suspended/stateID=1 before await_prepare:\n%s", prefix)
+		t.Fatalf("Parent does not publish Call/Suspended/source-line before await_prepare:\n%s", prefix)
 	}
 	awaitSuspend := strings.Index(parent[await:], "call i8 @llvm.coro.suspend")
 	if awaitSuspend < 0 {
@@ -2769,9 +2847,9 @@ func assertCoroStaticChildAwait(t *testing.T, parent string) {
 		`.*store i32 .*load i32.*switch i32`).MatchString(parent[await:]) {
 		t.Fatalf("Parent shared fast/resumed continuation does not carry the fused or slow child outcome into its status switch:\n%s", parent)
 	}
-	completionState := regexp.MustCompile(`(?s)store i16 2,.*store i16 4,.*store i32 2,`)
+	completionState := regexp.MustCompile(`(?s)store i16 2,.*store i16 4,.*store i32 0,`)
 	if !completionState.MatchString(parent[childCall[0]:]) {
-		t.Fatalf("Parent does not publish FrameComplete/FinalSuspended/stateID=2 after await:\n%s", parent)
+		t.Fatalf("Parent does not publish FrameComplete/FinalSuspended and clear its source line after await:\n%s", parent)
 	}
 }