diff --git a/cl/_testgo/allocinloop/in.go b/cl/_testgo/allocinloop/in.go index 499fb6d1f6..b9c0fdf85a 100644 --- a/cl/_testgo/allocinloop/in.go +++ b/cl/_testgo/allocinloop/in.go @@ -32,8 +32,9 @@ func Test() { } // CHECK-LABEL: define ptr @"main.main$coro"(ptr %0, ptr %1){{.*}} { -// CHECK: call ptr @"main.Test$coro" -// CHECK: call i1 @__llgo_coro_await_prepare_inline_v4 +// CHECK-NEXT: _llgo_0: +// CHECK-NEXT: %[[HANDLE:[0-9]+]] = call ptr @"main.Test$coro"(ptr %0, ptr %1) +// CHECK-NEXT: ret ptr %[[HANDLE]] func main() { Test() } diff --git a/cl/coro_abi.go b/cl/coro_abi.go index e1c2d20673..db4830da2a 100644 --- a/cl/coro_abi.go +++ b/cl/coro_abi.go @@ -146,7 +146,6 @@ const ( coroCompletePrepareHookV2 = "__llgo_coro_complete_prepare_v2" coroFrameFreeHookV1 = "__llgo_coro_frame_free_v1" coroDescriptorPrefixV1 = "__llgo_coro_frame_descriptor_v1." - coroBorrowedFrameMetadataWordsV2 = 20 ) const ( @@ -163,9 +162,7 @@ const ( coroHeaderResultSlot coroHeaderSuspendReason coroHeaderLifecycle - coroHeaderStateID coroHeaderLine - coroHeaderFlags ) const ( @@ -197,7 +194,6 @@ const coroPreemptInstructionBudget = 64 // storage whose address never escapes; the ordinary SROA/mem2reg pipeline can // therefore keep it in SSA registers on a non-suspending loop edge. Every // activation resets it, so it is not live across a scheduler-visible suspend. -// StateID remains exclusively the published resume-state identity. const coroPreemptCheckpointStride uint64 = 2048 type coroPhysicalABI struct { @@ -258,14 +254,22 @@ type coroBodyContext struct { completePrepare llssa.Expr terminalStatus llssa.Expr preemptCountdown llssa.Expr - nextState uint32 - terminalState uint32 - needsPreempt bool - instructions int - frameRetention *coroFrameRetentionProof - critical *coroCriticalProof - terminalResultAllocs map[*ssa.Alloc]llssa.Expr - sourceBlockPollFresh bool + // outcomeScratch is the one frame-local status/interface record shared by + // every fully consumed managed child transaction. Source execution cannot + // overlap two calls in one physical frame: a coroutine child suspends its + // parent until the old outcome is consumed, while an outcome-plain child + // returns synchronously. Keeping one directly addressed record avoids one + // permanent CoroSplit field per call site without adding a runtime lookup or + // dynamic lifetime protocol. + outcomeScratch llssa.Expr + nextState uint32 + terminalState uint32 + needsPreempt bool + instructions int + frameRetention *coroFrameRetentionProof + critical *coroCriticalProof + terminalResultAllocs map[*ssa.Alloc]llssa.Expr + sourceBlockPollFresh bool } func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *types.Signature) coroPhysicalABI { @@ -461,12 +465,20 @@ func coroHeaderType(prog llssa.Program) llssa.Type { prog.VoidPtr(), // result slot prog.Uint16(), // suspend reason prog.Uint16(), // lifecycle state - prog.Uint32(), // state ID prog.Uint32(), // source line - prog.Uint32(), // flags ) } +func coroBorrowedFrameMetadataWordsV2(prog llssa.Program) int64 { + pointerSize := prog.PointerSize() + if pointerSize != 4 && pointerSize != 8 { + panic("coroutine frame metadata requires a 32-bit or 64-bit pointer target") + } + // Mirrors runtime/internal/coro.BorrowedFrameStorageV2. The native Frame is + // twelve words; pointer-32 needs one extra word for its uint32 status field. + return int64(12 + 4/pointerSize) +} + func (p *context) beginCoroBody( b llssa.Builder, abi coroPhysicalABI, @@ -488,11 +500,11 @@ func (p *context) beginCoroBody( headerType := coroHeaderType(prog) header := b.AllocaT(headerType) borrowedFrameMetadataType := p.type_( - types.NewArray(types.Typ[types.Uintptr], coroBorrowedFrameMetadataWordsV2), + types.NewArray(types.Typ[types.Uintptr], coroBorrowedFrameMetadataWordsV2(prog)), llssa.InGo, ) // Dynamic ramps never consume this fallback storage. Leave it uninitialized - // here so every ordinary coroutine creation does not pay a 20-word memset; + // here so every ordinary coroutine creation does not pay a metadata memset; // PublishFrameV2 initializes the complete private Frame only when LLVM has // actually selected the allocation-elided path (storage == nil). borrowedFrameMetadata := b.AllocaT(borrowedFrameMetadataType) @@ -533,10 +545,8 @@ func (p *context) beginCoroBody( // managed calls and receive their ordinary Return outcomes. body.terminalStatus = b.AllocaT(prog.Uint32()) b.Store(body.terminalStatus, prog.IntVal(coroAwaitCompletionReturn, prog.Uint32())) - // This address is compiler-private and never reaches a runtime call. It - // deliberately differs from Header.StateID: that externally visible - // field aliases runtime validation calls and therefore forces a - // load/store on every otherwise plain loop edge. + // This address is compiler-private and never reaches a runtime call, so + // ordinary SROA can keep it in SSA registers on non-suspending edges. body.preemptCountdown = b.AllocaT(prog.Uint32()) } if abi.runDecisionTakeZeroHook != "" { @@ -820,12 +830,11 @@ func coroPanicTraceReplaceSignature() *types.Signature { func (c *coroBodyContext) publishState( b llssa.Builder, reason, lifecycle uint64, - stateID, line uint32, + _ uint32, line uint32, ) { prog := b.Prog b.Store(b.FieldAddr(c.header, coroHeaderSuspendReason), prog.IntVal(reason, prog.Uint16())) b.Store(b.FieldAddr(c.header, coroHeaderLifecycle), prog.IntVal(lifecycle, prog.Uint16())) - b.Store(b.FieldAddr(c.header, coroHeaderStateID), prog.IntVal(uint64(stateID), prog.Uint32())) b.Store(b.FieldAddr(c.header, coroHeaderLine), prog.IntVal(uint64(line), prog.Uint32())) } @@ -1121,7 +1130,6 @@ func (c *coroBodyContext) panicWithLine( prog := b.Prog b.Store(b.FieldAddr(c.header, coroHeaderSuspendReason), prog.IntVal(coroSuspendPanic, prog.Uint16())) b.Store(b.FieldAddr(c.header, coroHeaderLifecycle), prog.IntVal(coroLifecycleFinalSuspended, prog.Uint16())) - b.Store(b.FieldAddr(c.header, coroHeaderStateID), prog.IntVal(uint64(c.terminalStateID()), prog.Uint32())) b.Store(b.FieldAddr(c.header, coroHeaderLine), line) b.Call( c.panicPrepare, @@ -1176,6 +1184,10 @@ func (p *context) compileCoroPhysicalBody(b llssa.Builder, fn *ssa.Function, abi if err != nil { panic(fmt.Errorf("load frozen coroutine physical plan: %w", err)) } + if physicalPlan.tailForward != nil { + p.compileCoroTailForwardPhysicalBody(b, fn, physicalPlan, sourceParamBase) + return + } frameRetention := physicalPlan.frameRetention critical := physicalPlan.critical cleanupPlan := physicalPlan.cleanup @@ -1300,6 +1312,55 @@ func (p *context) compileCoroPhysicalBody(b llssa.Builder, fn *ssa.Function, abi emission.completeManagedPhysicalBody(bodyCapability) } +// compileCoroTailForwardPhysicalBody emits a stable coroutine entry ramp that +// owns no coroutine frame. The frozen physical plan guarantees that its source +// body is exactly one static call followed by an unchanged return, so passing +// the task and result slot straight through preserves every terminal and +// suspension protocol while avoiding a redundant parent await transaction. +func (p *context) compileCoroTailForwardPhysicalBody( + b llssa.Builder, + function *ssa.Function, + physical *coroPhysicalFunctionPlan, + sourceParamBase int, +) { + if p == nil || b == nil || b.Func != p.fn || function == nil || physical == nil || + physical.function != function || physical.tailForward == nil || + p.compilation == nil || p.immutablePlan() == nil || sourceParamBase < 2 { + panic("coroutine tail-forward emission requires one exact frozen physical plan") + } + forward := physical.tailForward + if err := forward.validate(function, p.immutablePlan()); err != nil { + panic(fmt.Errorf("validate frozen coroutine tail-forward plan: %w", err)) + } + if sourceParamBase != 2 { + panic("coroutine tail-forward source unexpectedly has a closure environment") + } + + b.SetBlock(p.fn.Block(0)) + entry := p.mustFunctionSymbol(forward.target) + if entry.plan.ID != forward.targetID || !entry.usesCoroPhysicalABI() { + panic("coroutine tail-forward target no longer resolves to its frozen physical entry") + } + target, _, kind := p.compileFunctionEntry(entry) + if kind != goFunc || target == nil { + panic("coroutine tail-forward target did not resolve to a Go coroutine entry") + } + args := make([]llssa.Expr, 0, len(forward.args)+2) + args = append(args, p.fn.PhysicalParam(0), p.fn.PhysicalParam(1)) + for _, argument := range forward.args { + if argument.sourceParameter >= 0 { + args = append(args, p.fn.PhysicalParam(sourceParamBase+argument.sourceParameter)) + continue + } + args = append(args, p.compileValueAs(b, argument.constant, argument.targetType)) + } + handle := b.Call(target.Expr, args...) + if handle.Type == nil || !types.Identical(handle.RawType(), types.Typ[types.UnsafePointer]) { + panic("coroutine tail-forward target returned a non-handle value") + } + b.Return(handle) +} + // validateCoroExactSyntheticForwarder proves the complete SSA body shared by // compiler-owned spawn carriers. The caller remains responsible for proving // why this exact value may be wrapped; this helper proves only that the diff --git a/cl/coro_abi_test.go b/cl/coro_abi_test.go index 91c01c6323..bc146163a2 100644 --- a/cl/coro_abi_test.go +++ b/cl/coro_abi_test.go @@ -316,6 +316,73 @@ func TestCoroChildAwaitPhysicalABIV1Presplit(t *testing.T) { } } +func TestCoroExactTailForwardReusesTargetHandle(t *testing.T) { + const source = `package foo +func Child(value uint32, delta int) {} +func Parent(value uint32) { Child(value, -1) } +` + prog, ssaPkg, files, universe, plan := prepareCoroChildAwaitPhysicalABISource(t, nil, source) + defer prog.Dispose() + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroChildAwaitCompilation(compilation) + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + t.Fatal(err) + } + module := pkg.Module() + defer module.Dispose() + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify tail-forward coroutine: %v\n%s", err, module.String()) + } + + parentSSA, childSSA := ssaPkg.Func("Parent"), ssaPkg.Func("Child") + owner := universe.ownerOf(parentSSA) + physical, err := universe.coroProgramIR.physicalFunctionPlan(parentSSA, owner) + if err != nil { + t.Fatal(err) + } + if physical.tailForward == nil || physical.tailForward.target != childSSA || + len(physical.tailForward.args) != 2 || + physical.tailForward.args[0].sourceParameter != 0 || + physical.tailForward.args[1].constant == nil || + physical.tailForward.args[1].constant.Int64() != -1 { + t.Fatalf("tail-forward physical plan = %+v; want Parent(value)->Child(value,-1)", physical.tailForward) + } + + parent := requireCoroPhysicalFunction(t, module, "foo.Parent") + child := requireCoroPhysicalFunction(t, module, "foo.Child") + parentIR, childIR := parent.String(), child.String() + if strings.Contains(parentIR, "llvm.coro.") || + strings.Contains(parentIR, coroFrameAllocHookV1) || + strings.Contains(parentIR, coroAwaitPrepareInlineHookV4) { + t.Fatalf("tail-forward ramp retained a coroutine frame or await transaction:\n%s", parentIR) + } + if !regexp.MustCompile(`call ptr @"?foo\.Child\$coro"?\(ptr [^,]+, ptr [^,]+, i32 [^,]+, i64 -1\)`).MatchString(parentIR) { + t.Fatalf("tail-forward ramp did not pass task/result/parameter/constant directly:\n%s", parentIR) + } + if strings.Count(parentIR, "call ptr") != 1 || !strings.Contains(parentIR, "ret ptr") { + t.Fatalf("tail-forward ramp is not one target call plus handle return:\n%s", parentIR) + } + if !strings.Contains(childIR, "llvm.coro.begin") || + !strings.Contains(childIR, coroFrameAllocHookV1) { + t.Fatalf("tail-forward target lost its physical coroutine body:\n%s", childIR) + } + + runCoroABITestPipeline(t, prog, module) + if module.NamedFunction("foo.Parent$coro.resume").IsNil() == false || + module.NamedFunction("foo.Parent$coro.destroy").IsNil() == false { + t.Fatalf("frame-free tail-forward ramp acquired split resume/destroy entries:\n%s", module.String()) + } + for _, suffix := range []string{".resume", ".destroy"} { + if module.NamedFunction("foo.Child$coro" + suffix).IsNil() { + t.Fatalf("tail-forward target lost split %s entry:\n%s", suffix, module.String()) + } + } +} + func TestCoroNamedFunctionTypeDirectAwait(t *testing.T) { const source = `package foo type Task func(uint32) uint32 @@ -801,6 +868,13 @@ func TestCoroPhysicalValueTransportABIV1NativeAndWasm(t *testing.T) { if err != nil { t.Fatal(err) } + parentPhysical, err := universe.coroProgramIR.physicalFunctionPlan(parent, universe.ownerOf(parent)) + if err != nil { + t.Fatal(err) + } + if parentPhysical.tailForward == nil || parentPhysical.tailForward.target != child { + t.Fatalf("Parent physical tail forward = %+v; want exact Child target", parentPhysical.tailForward) + } module := pkg.Module() defer module.Dispose() if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { @@ -823,12 +897,17 @@ func TestCoroPhysicalValueTransportABIV1NativeAndWasm(t *testing.T) { runCoroABITestPipeline(t, prog, module) post := module.String() - for _, function := range []string{"foo.Child$coro", "foo.Parent$coro", "foo.Pair$coro"} { + for _, function := range []string{"foo.Child$coro", "foo.Pair$coro"} { for _, suffix := range []string{".resume", ".destroy"} { if module.NamedFunction(function + suffix).IsNil() { t.Fatalf("CoroSplit did not create %s%s:\n%s", function, suffix, post) } } + for _, suffix := range []string{".resume", ".destroy"} { + if !module.NamedFunction("foo.Parent$coro" + suffix).IsNil() { + t.Fatalf("frame-free Parent tail-forward acquired %s:\n%s", suffix, post) + } + } } assertCoroResultSlotFields(t, "Pair after CoroSplit", module.NamedFunction("foo.Pair$coro.resume").String(), uintptrIR) if !regexp.MustCompile(`store %foo\.Payload [^,]+, ptr `).MatchString(module.NamedFunction("foo.Child$coro.resume").String()) { @@ -2171,10 +2250,9 @@ func assertCoroV0HeaderStateZero(t *testing.T, body string) { }{ {index: coroHeaderSuspendReason, type_: "i16", name: "suspend reason"}, {index: coroHeaderLifecycle, type_: "i16", name: "lifecycle"}, - {index: coroHeaderStateID, type_: "i32", name: "state ID"}, } { addresses := regexp.MustCompile( - `(?m)^\s*(%[-a-zA-Z$._0-9]+) = getelementptr[^\n{]* \{ ptr, ptr, ptr, ptr, ptr, i16, i16, i32, i32, i32 \}, ptr [^,]+, i32 0, i32 `+strconv.Itoa(field.index)+`\s*$`, + `(?m)^\s*(%[-a-zA-Z$._0-9]+) = getelementptr[^\n{]* \{ ptr, ptr, ptr, ptr, ptr, i16, i16, i32 \}, ptr [^,]+, i32 0, i32 `+strconv.Itoa(field.index)+`\s*$`, ).FindAllStringSubmatch(body, -1) if len(addresses) == 0 { t.Fatalf("v0 coroutine has no header %s store:\n%s", field.name, body) @@ -2710,9 +2788,9 @@ func assertCoroV1Completion(t *testing.T, name, body string) { t.Fatalf("%s does not prepare completion before final suspend:\n%s", name, body) } segment := body[:complete] - state := regexp.MustCompile(`(?s)store i16 2,.*store i16 4,.*store i32 [1-9][0-9]*,`) + state := regexp.MustCompile(`(?s)store i16 2,.*store i16 4,.*store i32 0,`) if !state.MatchString(segment) { - t.Fatalf("%s does not publish final reason/lifecycle/stateID before completion preparation:\n%s", name, body) + t.Fatalf("%s does not publish final reason/lifecycle and clear its source line before completion preparation:\n%s", name, body) } } @@ -2739,9 +2817,9 @@ func assertCoroStaticChildAwait(t *testing.T, parent string) { if !parentLink.MatchString(prefix) { t.Fatalf("Parent does not store its handle into child.parent before handoff:\n%s", prefix) } - state := regexp.MustCompile(`(?s)store i16 1,.*store i16 3,.*store i32 1,`) + state := regexp.MustCompile(`(?s)store i16 1,.*store i16 3,.*store i32 [1-9][0-9]*,`) if !state.MatchString(prefix) { - t.Fatalf("Parent does not publish Call/Suspended/stateID=1 before await_prepare:\n%s", prefix) + t.Fatalf("Parent does not publish Call/Suspended/source-line before await_prepare:\n%s", prefix) } awaitSuspend := strings.Index(parent[await:], "call i8 @llvm.coro.suspend") if awaitSuspend < 0 { @@ -2769,9 +2847,9 @@ func assertCoroStaticChildAwait(t *testing.T, parent string) { `.*store i32 .*load i32.*switch i32`).MatchString(parent[await:]) { t.Fatalf("Parent shared fast/resumed continuation does not carry the fused or slow child outcome into its status switch:\n%s", parent) } - completionState := regexp.MustCompile(`(?s)store i16 2,.*store i16 4,.*store i32 2,`) + completionState := regexp.MustCompile(`(?s)store i16 2,.*store i16 4,.*store i32 0,`) if !completionState.MatchString(parent[childCall[0]:]) { - t.Fatalf("Parent does not publish FrameComplete/FinalSuspended/stateID=2 after await:\n%s", parent) + t.Fatalf("Parent does not publish FrameComplete/FinalSuspended and clear its source line after await:\n%s", parent) } } diff --git a/cl/coro_await.go b/cl/coro_await.go index 1008d2f9b8..166bd85c1e 100644 --- a/cl/coro_await.go +++ b/cl/coro_await.go @@ -684,7 +684,7 @@ func (p *context) compileCoroPatchInitAwait(b llssa.Builder) { if kind != goFunc { panic("patch original initializer outcome target did not resolve to a Go entry") } - completion := p.structuredOutcomeAlloca(outcomePlainCompletionType(p.prog), true) + completion := p.structuredOutcomeScratch() resultType := p.prog.Type(newOutcomePlainPhysicalABI(original.Signature).resultSlotType, llssa.InGo) resultSlot := p.structuredOutcomeAlloca(resultType, false) b.Call(calleeFn.Expr, @@ -747,6 +747,31 @@ func (p *context) structuredOutcomeAlloca(typ llssa.Type, zeroed bool) llssa.Exp return alloc.AllocaT(typ) } +// structuredOutcomeScratch returns the one status/interface record owned by +// the active physical body. Every consumer completes before source execution +// can begin another transaction in the same frame: outcome-plain calls are +// synchronous, and a coroutine child keeps its parent suspended until the +// scheduler-owned completion has been consumed. The record therefore has a +// function-wide non-overlapping lifetime even when several source call sites +// survive CoroSplit. +func (p *context) structuredOutcomeScratch() llssa.Expr { + if !p.hasStructuredOutcomePhysicalBody() || p.fn == nil { + panic("structured outcome scratch requires an active physical body") + } + slot, coroutine := p.activeStructuredOutcomeScratchSlot() + if slot == nil { + panic("structured outcome scratch lost its physical owner") + } + if slot.IsNil() { + if coroutine { + *slot = p.coroFrameAlloc(outcomePlainCompletionType(p.prog)) + } else { + *slot = p.structuredOutcomeAlloca(outcomePlainCompletionType(p.prog), true) + } + } + return *slot +} + func (p *context) coroFrameByteAlloca(b llssa.Builder, size int64) llssa.Expr { if size < 0 { panic("coroutine byte alloca requires a non-negative constant size") @@ -848,9 +873,10 @@ func (p *context) awaitCoroChildWithRecovery( if body.abi.awaitConsumeHook == "" { panic("coroutine child await has no outcome consume hook") } - typeWord := p.coroFrameAlloca(p.prog.VoidPtr()) - dataWord := p.coroFrameAlloca(p.prog.VoidPtr()) - statusWord := p.coroFrameAlloca(p.prog.Uint32()) + outcomeScratch := p.structuredOutcomeScratch() + typeWord := b.FieldAddr(outcomeScratch, outcomePlainCompletionTypeWord) + dataWord := b.FieldAddr(outcomeScratch, outcomePlainCompletionDataWord) + statusWord := b.FieldAddr(outcomeScratch, outcomePlainCompletionStatus) b.Store(typeWord, p.prog.Nil(p.prog.VoidPtr())) b.Store(dataWord, p.prog.Nil(p.prog.VoidPtr())) b.Store(statusWord, p.prog.IntVal(0, p.prog.Uint32())) diff --git a/cl/coro_defer_test.go b/cl/coro_defer_test.go index 47ac8086c2..0b85f043e1 100644 --- a/cl/coro_defer_test.go +++ b/cl/coro_defer_test.go @@ -902,13 +902,39 @@ func coroTestIsAwaitCompletionSwitch(terminator llvm.Value) bool { func coroTestBlockStoresValueTo(block llvm.BasicBlock, value, address llvm.Value) bool { for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { if instruction.InstructionOpcode() == llvm.Store && - instruction.Operand(0) == value && instruction.Operand(1) == address { + instruction.Operand(0) == value && coroTestSameAddress(instruction.Operand(1), address) { return true } } return false } +// coroTestSameAddress recognizes equivalent GEP expressions even when LLVM has +// emitted separate SSA instructions for the same field in shared frame storage. +func coroTestSameAddress(left, right llvm.Value) bool { + if left == right { + return true + } + if left.IsNil() || right.IsNil() || left.IsAGetElementPtrInst().IsNil() || right.IsAGetElementPtrInst().IsNil() || + left.OperandsCount() != right.OperandsCount() { + return false + } + for index := 0; index < left.OperandsCount(); index++ { + leftOperand, rightOperand := left.Operand(index), right.Operand(index) + if leftOperand == rightOperand { + continue + } + if !leftOperand.IsAConstantInt().IsNil() && !rightOperand.IsAConstantInt().IsNil() && + leftOperand.ZExtValue() == rightOperand.ZExtValue() { + continue + } + if !coroTestSameAddress(leftOperand, rightOperand) { + return false + } + } + return true +} + func coroTestBlockCanReachDirectCall(entry llvm.BasicBlock, callee string) bool { seen := make(map[llvm.BasicBlock]bool) pending := []llvm.BasicBlock{entry} diff --git a/cl/coro_emission_session.go b/cl/coro_emission_session.go index d174ffe79d..6022741679 100644 --- a/cl/coro_emission_session.go +++ b/cl/coro_emission_session.go @@ -192,6 +192,20 @@ func (p *context) hasStructuredOutcomePhysicalBody() bool { return p.activeCoroPhysicalBodyCapability() != nil } +// activeStructuredOutcomeScratchSlot exposes only the reusable completion +// record owned by the active exclusive body. Keeping the body choice here +// avoids giving the call lowerer another complete coroutine-body capability. +func (p *context) activeStructuredOutcomeScratchSlot() (slot *llssa.Expr, coroutine bool) { + body := p.activeCoroPhysicalBodyCapability() + if body == nil { + return nil, false + } + if body.coroutine != nil { + return &body.coroutine.outcomeScratch, true + } + return &body.outcome.outcomeScratch, false +} + // managedPhysicalTask is the scheduler-owned G passed to either structured // physical ABI. Outcome-plain DAG calls forward it unchanged; only a full LLVM // coroutine body may consume the coroutine and cleanup capabilities below. diff --git a/cl/coro_generic_receiver_instance_test.go b/cl/coro_generic_receiver_instance_test.go index bdf9b718c1..59875a4cdc 100644 --- a/cl/coro_generic_receiver_instance_test.go +++ b/cl/coro_generic_receiver_instance_test.go @@ -119,18 +119,28 @@ func Root(p *Pointer[int]) *int { return p.Load() } if err != nil { t.Fatal(err) } + rootPhysical, err := universe.coroProgramIR.physicalFunctionPlan(root, universe.ownerOf(root)) + if err != nil { + t.Fatal(err) + } + if rootPhysical.tailForward == nil || rootPhysical.tailForward.target != instance { + t.Fatalf("generic receiver root tail forward = %+v; want exact instance target", rootPhysical.tailForward) + } module := compiled.Module() defer module.Dispose() if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { t.Fatalf("verify generic receiver instance before CoroSplit: %v\n%s", err, module.String()) } rootIR := requireCoroPhysicalFunction(t, module, "foo.Root").String() - if !strings.Contains(rootIR, "$coro") || !strings.Contains(rootIR, "call i1 @"+coroAwaitPrepareInlineHookV4) { - t.Fatalf("generic receiver call did not use child await:\n%s", rootIR) + if !strings.Contains(rootIR, `call ptr @"foo.(*Pointer[int]).Load$coro"`) || + !strings.Contains(rootIR, "ret ptr") || strings.Contains(rootIR, "llvm.coro.") || + strings.Contains(rootIR, coroAwaitPrepareInlineHookV4) { + t.Fatalf("generic receiver call did not use the exact frame-free tail ramp:\n%s", rootIR) } runCoroABITestPipeline(t, prog, module) - if module.NamedFunction("foo.Root$coro.resume").IsNil() { - t.Fatalf("CoroSplit lost generic receiver caller resume:\n%s", module.String()) + if !module.NamedFunction("foo.Root$coro.resume").IsNil() || + module.NamedFunction("foo.(*Pointer[int]).Load$coro.resume").IsNil() { + t.Fatalf("CoroSplit did not preserve the frame-free root/physical instance split:\n%s", module.String()) } }) } diff --git a/cl/coro_outcome_plain.go b/cl/coro_outcome_plain.go index 360611d6b4..ea3ddb7236 100644 --- a/cl/coro_outcome_plain.go +++ b/cl/coro_outcome_plain.go @@ -86,10 +86,11 @@ const ( ) type outcomePlainBodyContext struct { - abi outcomePlainPhysicalABI - task llssa.Expr - resultSlot llssa.Expr - completion llssa.Expr + abi outcomePlainPhysicalABI + task llssa.Expr + resultSlot llssa.Expr + completion llssa.Expr + outcomeScratch llssa.Expr } func (body *outcomePlainBodyContext) publish( @@ -443,7 +444,7 @@ func (p *context) compileCoroStaticOutcomeTargetCallResult( // suspension. Keep it in the physical function entry: loop iterations reuse // the record after a fully synchronous call, avoiding dynamic stack growth and // exposing its fields to ordinary SROA. - completion := p.structuredOutcomeAlloca(outcomePlainCompletionType(p.prog), true) + completion := p.structuredOutcomeScratch() physicalArgs := make([]llssa.Expr, 0, len(args)+3) physicalArgs = append(physicalArgs, p.managedPhysicalTask(), diff --git a/cl/coro_outcome_plain_test.go b/cl/coro_outcome_plain_test.go index 144ccab340..6ddd79d125 100644 --- a/cl/coro_outcome_plain_test.go +++ b/cl/coro_outcome_plain_test.go @@ -72,6 +72,22 @@ func Parent(first, second uint32, choose bool, payload any, fail bool) uint32 { } ` +const coroOutcomePlainSharedScratchFixture = `package foo + +func Leaf(value uint32, payload any, fail bool) uint32 { + if fail { + panic(payload) + } + return value + 1 +} + +func Parent(first, second uint32, payload any, fail bool) uint32 { + left := Leaf(first, payload, false) + right := Leaf(second, payload, fail) + return Leaf(left+right, payload, false) +} +` + const coroOutcomePlainLargeDAGFixture = `package foo type Huge [131073]byte @@ -213,6 +229,44 @@ func TestCoroOutcomePlainStaticTwinKeepsDynamicCoroutineEntry(t *testing.T) { runCoroABITestPipeline(t, prog, module) } +func TestCoroOutcomePlainCallSitesShareOneFrameScratchNativeAndWasm32(t *testing.T) { + llssa.Initialize(llssa.InitAll) + for _, test := range []struct { + name string + target *llssa.Target + }{ + {name: "native"}, + {name: "wasm32", target: &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}}, + } { + t.Run(test.name, func(t *testing.T) { + prog, pkg, plan, ssaPkg := compileCoroOutcomePlainSource( + t, test.target, coroOutcomePlainSharedScratchFixture, "Parent", 64, + ) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + leaf := ssaPkg.Func("Leaf") + leafPlan, found := plan.FunctionPlan(leaf) + if !found || leafPlan.Emission != coro.EmitOutcomePlain { + t.Fatalf("Leaf plan = %+v, present=%t; want outcome-plain", leafPlan, found) + } + parent := requireCoroPhysicalFunction(t, module, "foo.Parent") + parentIR := parent.String() + if got := strings.Count(parentIR, "foo.Leaf$outcome"); got != 3 { + t.Fatalf("Parent outcome calls = %d, want 3:\n%s", got, parentIR) + } + if got := strings.Count(parentIR, "alloca { i32, ptr, ptr }"); got != 1 { + t.Fatalf("Parent outcome completion allocas = %d, want one shared frame scratch:\n%s", got, parentIR) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify shared outcome scratch module before CoroSplit: %v\n%s", err, module.String()) + } + runCoroABITestPipeline(t, prog, module) + }) + } +} + func TestCoroOutcomePlainLeafNativeAndWasm32(t *testing.T) { llssa.Initialize(llssa.InitAll) for _, test := range []struct { diff --git a/cl/coro_panic_test.go b/cl/coro_panic_test.go index 7c704122ae..f9c1068828 100644 --- a/cl/coro_panic_test.go +++ b/cl/coro_panic_test.go @@ -148,11 +148,11 @@ func assertCoroExplicitStatusPanicBody(t *testing.T, body string, panicSites int t.Fatalf("final coro.suspend calls = %d, want exactly one shared final suspend:\n%s", got, body) } stateAndHook := regexp.MustCompile( - `(?s)store i16 5,.*?store i16 4,.*?store i32 [1-9][0-9]*,.*?store i32 [1-9][0-9]*,.*?call void @` + regexp.QuoteMeta(coroPanicPrepareHookV1) + + `(?s)store i16 5,.*?store i16 4,.*?store i32 [1-9][0-9]*,.*?call void @` + regexp.QuoteMeta(coroPanicPrepareHookV1) + `\(ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^)]+\)`, ) if got := len(stateAndHook.FindAllStringIndex(body, -1)); got != panicSites { - t.Fatalf("Panic/FinalSuspended/stateID/source-line publication followed by the five-pointer hook = %d, want %d:\n%s", got, panicSites, body) + t.Fatalf("Panic/FinalSuspended/source-line publication followed by the five-pointer hook = %d, want %d:\n%s", got, panicSites, body) } hookBranch := regexp.MustCompile( `call void @`+regexp.QuoteMeta(coroPanicPrepareHookV1)+`\([^\n]+\)\n\s+br label (%[-a-zA-Z$._0-9]+)`, diff --git a/cl/coro_park_test.go b/cl/coro_park_test.go index 826ddf99d6..634bf12a46 100644 --- a/cl/coro_park_test.go +++ b/cl/coro_park_test.go @@ -87,11 +87,11 @@ func TestCoroParkCurrentFrameNativeAndWasm32(t *testing.T) { t.Fatalf("structured park leaked an ordinary sync helper call:\n%s", body) } stateAndHook := regexp.MustCompile( - `(?s)store i16 4,.*store i16 3,.*store i32 1,.*call void @` + regexp.QuoteMeta(coroKeyedParkHookV2) + + `(?s)store i16 4,.*store i16 3,.*store i32 [1-9][0-9]*,.*call void @` + regexp.QuoteMeta(coroKeyedParkHookV2) + `\(ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^)]+\)`, ) if !stateAndHook.MatchString(body) { - t.Fatalf("Root does not publish Park/Suspended/stateID=1 before the exact v1 hook:\n%s", body) + t.Fatalf("Root does not publish Park/Suspended/source-line before the exact v1 hook:\n%s", body) } hook := strings.Index(body, "call void @"+coroKeyedParkHookV2) parkSuspendRelative := strings.Index(body[hook:], "call i8 @llvm.coro.suspend") diff --git a/cl/coro_physical_plan.go b/cl/coro_physical_plan.go index 4a88333e0f..79287dcb71 100644 --- a/cl/coro_physical_plan.go +++ b/cl/coro_physical_plan.go @@ -431,6 +431,7 @@ func (plan coroPhysicalInstructionPlan) elidesRuntimeHelper(helper string) bool type coroPhysicalFunctionPlan struct { function *ssa.Function owner *preparedEmissionPackage + tailForward *coroPhysicalTailForwardPlan frameRetention *coroFrameRetentionProof critical *coroCriticalProof cleanup *coroStaticCleanupPlan @@ -445,6 +446,84 @@ type coroPhysicalFunctionPlan struct { instructions map[ssa.Instruction]coroPhysicalInstructionPlan } +// coroPhysicalTailForwardArgument is one frozen source-to-target parameter +// recipe for a coroutine ramp that can be represented by another ramp. Exactly +// one arm is selected: sourceParameter indexes the forwarding function's +// physical source parameters, while constant retains one immutable SSA +// literal and its exact target parameter type. +type coroPhysicalTailForwardArgument struct { + sourceParameter int + constant *ssa.Const + targetType types.Type +} + +// coroPhysicalTailForwardPlan proves that a source function contributes no +// physical state of its own: it performs one exact static coroutine call and +// returns that call's results unchanged. Its public $coro symbol remains the +// stable callable identity, but the ramp can return the target ramp's handle +// directly. Suspension, preemption, cancellation, panic outcome, and result +// publication then remain owned by the target frame instead of traversing a +// redundant parent frame and await transaction. +type coroPhysicalTailForwardPlan struct { + target *ssa.Function + targetID coro.FunctionID + args []coroPhysicalTailForwardArgument +} + +func (plan *coroPhysicalTailForwardPlan) validate( + function *ssa.Function, + whole *coro.SSAPlan, +) error { + if plan == nil || function == nil || whole == nil || plan.target == nil || + plan.target == function || plan.targetID == "" || len(plan.target.FreeVars) != 0 { + return fmt.Errorf("tail-forward plan has an incomplete function or target identity") + } + functionPlan, functionPlanned := whole.FunctionPlan(function) + targetPlan, targetPlanned := whole.FunctionPlan(plan.target) + if !functionPlanned || functionPlan.Emission != coro.EmitCoroutine || + functionPlan.ManagedEntry != coro.ManagedEntryCoroutine || functionPlan.Recursive { + return fmt.Errorf("tail-forward source is not one non-recursive coroutine primary") + } + if !targetPlanned || targetPlan.ID != plan.targetID || + targetPlan.Emission != coro.EmitCoroutine || + targetPlan.ManagedEntry != coro.ManagedEntryCoroutine || + targetPlan.Primary != coro.PrimaryCoroutine || targetPlan.FuncRep != coro.DirectCoro { + return fmt.Errorf("tail-forward target is not the frozen direct coroutine primary %q", plan.targetID) + } + sourceSig := coroPhysicalNormalizeSourceSignature(function.Signature) + targetSig := coroPhysicalNormalizeSourceSignature(plan.target.Signature) + if sourceSig == nil || targetSig == nil || sourceSig.Variadic() || targetSig.Variadic() || + sourceSig.Params().Len() != len(function.Params) || + targetSig.Params().Len() != len(plan.args) || + !types.Identical(sourceSig.Results(), targetSig.Results()) { + return fmt.Errorf("tail-forward source and target physical signatures are incompatible") + } + for index, argument := range plan.args { + parameter := argument.sourceParameter >= 0 + literal := argument.constant != nil + if parameter == literal || argument.targetType == nil || + !types.Identical(argument.targetType, targetSig.Params().At(index).Type()) { + return fmt.Errorf("tail-forward argument %d has an invalid frozen recipe", index) + } + if parameter { + if argument.sourceParameter >= sourceSig.Params().Len() || + !types.Identical( + sourceSig.Params().At(argument.sourceParameter).Type(), + argument.targetType, + ) { + return fmt.Errorf("tail-forward argument %d has an incompatible source parameter", index) + } + continue + } + if argument.constant.Type() == nil || + (!types.AssignableTo(argument.constant.Type(), argument.targetType) && + !types.Identical(types.Default(argument.constant.Type()), argument.targetType)) { + return fmt.Errorf("tail-forward argument %d has an incompatible constant", index) + } + } + return nil +} + func prepareCoroPhysicalFunctionPlan( audit *coroPhysicalPureSSAAudit, owner *preparedEmissionPackage, @@ -574,9 +653,159 @@ func prepareCoroPhysicalFunctionPlan( plan.atomicCostProof = logical.AtomicCostProof plan.atomicCertificate = logical.AtomicCostCertificate } + tailForward, err := planCoroPhysicalTailForward(plan, whole) + if err != nil { + return nil, err + } + plan.tailForward = tailForward return plan, nil } +// planCoroPhysicalTailForward recognizes the deliberately small forwarding +// language after ordinary instruction and call control recipes are frozen. It +// is not source inlining: the forwarding symbol remains externally callable +// and merely returns the exact target coroutine handle. Restricting arguments +// to unchanged parameters and literals makes target invocation independent of +// the removed frame and prevents code generation from rediscovering SSA value +// semantics after ProgramIR preflight. +func planCoroPhysicalTailForward( + physical *coroPhysicalFunctionPlan, + whole *coro.SSAPlan, +) (*coroPhysicalTailForwardPlan, error) { + if physical == nil || physical.function == nil || whole == nil || + physical.cleanup != nil || physical.critical != nil { + return nil, nil + } + function := physical.function + logical, planned := whole.FunctionPlan(function) + if !planned || logical.Emission != coro.EmitCoroutine || + logical.ManagedEntry != coro.ManagedEntryCoroutine || logical.Recursive || + len(function.Blocks) != 1 || len(function.Blocks[0].Succs) != 0 || + len(function.FreeVars) != 0 || function.Recover != nil || + function.Signature == nil || function.Signature.Variadic() || + hasNoInlineDirective(function) { + return nil, nil + } + + // DebugRef is intentionally not ignored. A debug-symbol build keeps the + // ordinary frame so its source parameter locations and physical inline + // boundary remain representable without a second debug-only recipe. + instructions := function.Blocks[0].Instrs + if len(instructions) < 2 { + return nil, nil + } + call, ok := instructions[0].(*ssa.Call) + if !ok || call.Common() == nil || call.Common().IsInvoke() || + call.Common().Method != nil || call.Common().StaticCallee() == nil { + return nil, nil + } + callPhysical, frozen := physical.instructions[call] + if !frozen || callPhysical.semantic.recipe != coro.RecipeID("cl.ssa.call.v1") || + !callPhysical.semantic.evaluated || callPhysical.control != coroPhysicalControlDirectAwait || + callPhysical.operation != coroPhysicalOperationNone || + callPhysical.outcome != coroPhysicalOutcomeNone || + callPhysical.controlFailure != "" || callPhysical.operationFailure != "" || + callPhysical.outcomeFailure != "" { + return nil, nil + } + target := callPhysical.controlTarget + if target == nil || target == function || target != call.Common().StaticCallee() || + len(target.FreeVars) != 0 || target.Signature == nil || target.Signature.Variadic() { + return nil, nil + } + callPlan, callPlanned := whole.CallPlan(call) + targetPlan, targetPlanned := whole.FunctionPlan(target) + if !callPlanned || callPlan.Kind != coro.CallDirect || callPlan.Open || + callPlan.MayBeNil || callPlan.Rep != coro.DirectCoro || len(callPlan.Targets) != 1 || + !targetPlanned || targetPlan.ID != callPhysical.controlTargetID || + callPlan.Targets[0] != targetPlan.ID || targetPlan.Emission != coro.EmitCoroutine || + targetPlan.ManagedEntry != coro.ManagedEntryCoroutine || + targetPlan.Primary != coro.PrimaryCoroutine || targetPlan.FuncRep != coro.DirectCoro { + return nil, nil + } + + sourceSig := coroPhysicalNormalizeSourceSignature(function.Signature) + targetSig := coroPhysicalNormalizeSourceSignature(target.Signature) + if sourceSig.Params().Len() != len(function.Params) || + targetSig.Params().Len() != len(call.Common().Args) || + !types.Identical(sourceSig.Results(), targetSig.Results()) { + return nil, nil + } + parameterIndex := make(map[*ssa.Parameter]int, len(function.Params)) + for index, parameter := range function.Params { + parameterIndex[parameter] = index + } + arguments := make([]coroPhysicalTailForwardArgument, len(call.Common().Args)) + for index, value := range call.Common().Args { + targetType := targetSig.Params().At(index).Type() + if parameter, ok := value.(*ssa.Parameter); ok { + sourceIndex, owned := parameterIndex[parameter] + if !owned || !types.Identical(parameter.Type(), targetType) { + return nil, nil + } + arguments[index] = coroPhysicalTailForwardArgument{ + sourceParameter: sourceIndex, + targetType: targetType, + } + continue + } + literal, ok := value.(*ssa.Const) + if !ok || literal.Type() == nil || + (!types.AssignableTo(literal.Type(), targetType) && + !types.Identical(types.Default(literal.Type()), targetType)) { + return nil, nil + } + arguments[index] = coroPhysicalTailForwardArgument{ + sourceParameter: -1, + constant: literal, + targetType: targetType, + } + } + + resultCount := sourceSig.Results().Len() + returned := make([]ssa.Value, 0, resultCount) + cursor := 1 + switch resultCount { + case 0: + case 1: + returned = append(returned, call) + default: + for index := 0; index < resultCount; index++ { + if cursor >= len(instructions)-1 { + return nil, nil + } + extract, ok := instructions[cursor].(*ssa.Extract) + if !ok || extract.Tuple != call || extract.Index != index { + return nil, nil + } + returned = append(returned, extract) + cursor++ + } + } + if cursor != len(instructions)-1 { + return nil, nil + } + ret, ok := instructions[cursor].(*ssa.Return) + if !ok || len(ret.Results) != len(returned) { + return nil, nil + } + for index := range returned { + if ret.Results[index] != returned[index] { + return nil, nil + } + } + + forward := &coroPhysicalTailForwardPlan{ + target: target, + targetID: targetPlan.ID, + args: arguments, + } + if err := forward.validate(function, whole); err != nil { + return nil, fmt.Errorf("tail-forward physical plan: %w", err) + } + return forward, nil +} + // coroPhysicalInstructionNeedsRuntimeContext is the emission-side closure of // compiler-injected helpers whose requirement is not represented by an // ordinary Go call edge. ProgramIR seeds the corresponding source operation; diff --git a/cl/coro_poll_wait_test.go b/cl/coro_poll_wait_test.go index 8fd8eeddcd..cb4e46191f 100644 --- a/cl/coro_poll_wait_test.go +++ b/cl/coro_poll_wait_test.go @@ -153,11 +153,11 @@ func TestCoroPollWaitCurrentFrameNativeAndWasm32(t *testing.T) { } } stateAndPark := regexp.MustCompile( - `(?s)store i16 4,.*store i16 3,.*store i32 1,.*call void @` + regexp.QuoteMeta(coroPollParkHookV2) + + `(?s)store i16 4,.*store i16 3,.*store i32 [1-9][0-9]*,.*call void @` + regexp.QuoteMeta(coroPollParkHookV2) + `\(ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^,]+, i(?:32|64) [^,]+, i32 [^,]+, i32 [^,]+, i64 [^)]+\)`, ) if !stateAndPark.MatchString(body) { - t.Fatalf("Root does not publish Park/Suspended/stateID=1 before Poll V2 park:\n%s", body) + t.Fatalf("Root does not publish Park/Suspended/source-line before Poll V2 park:\n%s", body) } park := strings.Index(body, "call void @"+coroPollParkHookV2) suspendRelative := strings.Index(body[park:], "call i8 @llvm.coro.suspend") diff --git a/cl/coro_timer_sleep_test.go b/cl/coro_timer_sleep_test.go index 15232d9cd7..80dc2f3684 100644 --- a/cl/coro_timer_sleep_test.go +++ b/cl/coro_timer_sleep_test.go @@ -109,11 +109,11 @@ func TestCoroTimerSleepCurrentFrameNativeAndWasm32(t *testing.T) { } } stateAndPark := regexp.MustCompile( - `(?s)store i16 4,.*store i16 3,.*store i32 1,.*call void @` + regexp.QuoteMeta(coroTimerParkHookV2) + + `(?s)store i16 4,.*store i16 3,.*store i32 [1-9][0-9]*,.*call void @` + regexp.QuoteMeta(coroTimerParkHookV2) + `\(ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^,]+, i64 [^)]+\)`, ) if !stateAndPark.MatchString(body) { - t.Fatalf("Root does not publish Park/Suspended/stateID=1 before Timer V2 park:\n%s", body) + t.Fatalf("Root does not publish Park/Suspended/source-line before Timer V2 park:\n%s", body) } park := strings.Index(body, "call void @"+coroTimerParkHookV2) suspendRelative := strings.Index(body[park:], "call i8 @llvm.coro.suspend") diff --git a/internal/build/coro_bootstrap_factory.go b/internal/build/coro_bootstrap_factory.go index 145c0e6978..402a140ac6 100644 --- a/internal/build/coro_bootstrap_factory.go +++ b/internal/build/coro_bootstrap_factory.go @@ -27,7 +27,7 @@ import ( const ( coroProgramFrameAllocHookV1 = "__llgo_coro_frame_alloc_v1" - coroProgramFramePublishHookV1 = "__llgo_coro_frame_publish_v1" + coroProgramFramePublishHookV3 = "__llgo_coro_frame_publish_v3" coroProgramAwaitPrepareHookV2 = "__llgo_coro_await_prepare_v2" coroProgramAwaitConsumeHookV1 = "__llgo_coro_await_consume_v1" coroProgramPanicPrepareHookV1 = "__llgo_coro_panic_prepare_v1" @@ -58,9 +58,7 @@ const ( coroProgramHeaderResultSlotV1 coroProgramHeaderSuspendReasonV1 coroProgramHeaderLifecycleV1 - coroProgramHeaderStateIDV1 coroProgramHeaderLineV1 - coroProgramHeaderFlagsV1 ) type coroProgramBootstrapFactoryTargetV2 struct { @@ -160,13 +158,18 @@ func emitCoroProgramBootstrapFactoryV2( descriptorPointer := b.Convert(prog.VoidPtr(), descriptor) headerType := coroProgramBootstrapHeaderTypeV1(prog) header := b.AllocaT(headerType) + frameMetadataType := prog.Type( + types.NewArray(types.Typ[types.Uintptr], coroProgramFrameMetadataWordsV2(prog)), + llssa.InGo, + ) + frameMetadata := b.AllocaT(frameMetadataType) alloc := pkg.NewFunc(coroProgramFrameAllocHookV1, newSignature( []types.Type{pointer, types.Typ[types.Uintptr], types.Typ[types.Uintptr], pointer}, []types.Type{pointer}, ), llssa.InC) - publish := pkg.NewFunc(coroProgramFramePublishHookV1, newSignature( - []types.Type{pointer, pointer, pointer, pointer}, nil, + publish := pkg.NewFunc(coroProgramFramePublishHookV3, newSignature( + []types.Type{pointer, pointer, pointer, pointer, pointer, pointer, pointer}, nil, ), llssa.InC) await := pkg.NewFunc(coroProgramAwaitPrepareHookV2, newSignature( []types.Type{pointer, pointer, pointer}, nil, @@ -197,6 +200,7 @@ func emitCoroProgramBootstrapFactoryV2( return b.Call(alloc.Expr, g, size, align, descriptorPointer) }, Free: func(b llssa.Builder, storage, size, align llssa.Expr) { + b.KeepAlive(frameMetadata) b.Call(free.Expr, g, storage, size, align, descriptorPointer) }, } @@ -207,22 +211,16 @@ func emitCoroProgramBootstrapFactoryV2( emitCoroProgramTakeNormalRunDecisionV1(b, runDecisionTake, g) }, BeforeInitialSuspend: func(b llssa.Builder, handle, storage llssa.Expr) { - values := []llssa.Expr{ + b.Call( + publish.Expr, g, - null, + handle, + b.Convert(prog.VoidPtr(), header), + storage, + b.Convert(prog.VoidPtr(), frameMetadata), descriptorPointer, - null, out, - prog.IntVal(coroProgramSuspendNoneV1, prog.Uint16()), - prog.IntVal(coroProgramLifecycleInitialV1, prog.Uint16()), - prog.IntVal(0, prog.Uint32()), - prog.IntVal(0, prog.Uint32()), - prog.IntVal(0, prog.Uint32()), - } - for index, value := range values { - b.Store(b.FieldAddr(header, index), value) - } - b.Call(publish.Expr, g, handle, b.Convert(prog.VoidPtr(), header), storage) + ) }, }) @@ -265,10 +263,8 @@ func emitCoroProgramBootstrapFactoryV2( child := b.Call(rootFactory, g, null, null) childHeader := b.CoroPromise(child, headerType) b.Store(b.FieldAddr(childHeader, coroProgramHeaderParentV1), coroBuilder.Handle()) - stateID := uint64(index + 1) b.Store(b.FieldAddr(header, coroProgramHeaderSuspendReasonV1), prog.IntVal(coroProgramSuspendCallV1, prog.Uint16())) b.Store(b.FieldAddr(header, coroProgramHeaderLifecycleV1), prog.IntVal(coroProgramLifecycleSuspendedV1, prog.Uint16())) - b.Store(b.FieldAddr(header, coroProgramHeaderStateIDV1), prog.IntVal(stateID, prog.Uint32())) b.Store(b.FieldAddr(header, coroProgramHeaderLineV1), prog.IntVal(0, prog.Uint32())) b.Call(await.Expr, g, coroBuilder.Handle(), child) coroBuilder.SuspendCurrentBlock() @@ -313,7 +309,6 @@ func emitCoroProgramBootstrapFactoryV2( b.SetBlockEx(panicked, llssa.AtEnd, false) b.Store(b.FieldAddr(header, coroProgramHeaderSuspendReasonV1), prog.IntVal(coroProgramSuspendPanicV1, prog.Uint16())) b.Store(b.FieldAddr(header, coroProgramHeaderLifecycleV1), prog.IntVal(coroProgramLifecycleFinalV1, prog.Uint16())) - b.Store(b.FieldAddr(header, coroProgramHeaderStateIDV1), prog.IntVal(uint64(len(bootstrap.Steps)+1), prog.Uint32())) b.Store(b.FieldAddr(header, coroProgramHeaderLineV1), prog.IntVal(0, prog.Uint32())) b.Call( panicPrepare.Expr, @@ -329,7 +324,6 @@ func emitCoroProgramBootstrapFactoryV2( b.Store(b.FieldAddr(header, coroProgramHeaderSuspendReasonV1), prog.IntVal(coroProgramSuspendFrameCompleteV1, prog.Uint16())) b.Store(b.FieldAddr(header, coroProgramHeaderLifecycleV1), prog.IntVal(coroProgramLifecycleFinalV1, prog.Uint16())) - b.Store(b.FieldAddr(header, coroProgramHeaderStateIDV1), prog.IntVal(uint64(len(bootstrap.Steps)+1), prog.Uint32())) b.Store(b.FieldAddr(header, coroProgramHeaderLineV1), prog.IntVal(0, prog.Uint32())) b.Call( complete.Expr, @@ -346,6 +340,14 @@ func emitCoroProgramBootstrapFactoryV2( return factory } +func coroProgramFrameMetadataWordsV2(prog llssa.Program) int64 { + pointerSize := prog.PointerSize() + if pointerSize != 4 && pointerSize != 8 { + panic("coroutine bootstrap metadata requires a 32-bit or 64-bit pointer target") + } + return int64(12 + 4/pointerSize) +} + func validateCoroProgramBootstrapFactoryV2( pkg llssa.Package, bootstrap *coroProgramBootstrapV1, targets []coroProgramBootstrapFactoryTargetV2, ) { @@ -402,8 +404,6 @@ func coroProgramBootstrapHeaderTypeV1(prog llssa.Program) llssa.Type { prog.VoidPtr(), // ResultSlot prog.Uint16(), // SuspendReason prog.Uint16(), // Lifecycle - prog.Uint32(), // StateID prog.Uint32(), // Line - prog.Uint32(), // Flags ) } diff --git a/internal/build/coro_bootstrap_factory_test.go b/internal/build/coro_bootstrap_factory_test.go index d10ebe2d9d..300025d64a 100644 --- a/internal/build/coro_bootstrap_factory_test.go +++ b/internal/build/coro_bootstrap_factory_test.go @@ -307,7 +307,7 @@ func assertCoroProgramBootstrapFactoryPresplitV2(t *testing.T, ir, uintptrIR str t.Fatalf("V2 factory without closed-static spawn emitted main-return cancellation:\n%s", body) } assertInOrder(t, body, - "call void @"+coroProgramFramePublishHookV1, + "call void @"+coroProgramFramePublishHookV3, "call i8 @llvm.coro.suspend", "call void @"+coroRunDecisionTakeSymbolV1, "store i16 2", diff --git a/internal/build/coro_directive_gate_test.go b/internal/build/coro_directive_gate_test.go index d274fc6747..848ec72a01 100644 --- a/internal/build/coro_directive_gate_test.go +++ b/internal/build/coro_directive_gate_test.go @@ -83,8 +83,11 @@ func TestCoroProductionDirectiveInventory(t *testing.T) { // its descriptor is created O_NONBLOCK, a semantic fact unavailable // from the C signature and therefore not soundly compiler-inferable. "contract": 8, - "noblock": 35, - "sync": 28, + // The final two noblock leaves are the bounded native frame-cache + // take/put operations. Their raw C signatures cannot express either the + // fixed-size critical section or callback-free ownership transfer. + "noblock": 37, + "sync": 28, } if !reflect.DeepEqual(got, want) { t.Fatalf( @@ -94,7 +97,7 @@ func TestCoroProductionDirectiveInventory(t *testing.T) { ) } sort.Strings(manifest) - const wantManifestSHA256 = "79e04f606714069df05f69f48ddce3c7108b38bc503b734370a021ab5aadc912" + const wantManifestSHA256 = "deb857baa2d1c1d9a1b223bf3af61bb998229732d80bf36c3ce8dd126fefbf5a" manifestSHA256 := fmt.Sprintf( "%x", sha256.Sum256([]byte(strings.Join(manifest, "\n"))), ) diff --git a/internal/build/coro_native_fleet_e2e_test.go b/internal/build/coro_native_fleet_e2e_test.go index 752c6f3f72..d451f76855 100644 --- a/internal/build/coro_native_fleet_e2e_test.go +++ b/internal/build/coro_native_fleet_e2e_test.go @@ -2254,14 +2254,15 @@ func buildCoroNativeFleetE2ERuntimeIsland(t *testing.T, temp string) []string { } objects = append(objects, buildCoroRuntimeIslandFaultStringStubs(t, prog, temp), + buildCoroNativeAllocationCacheObject(t, temp), buildCoroNativeWorkerCallObject(t, temp), buildCoroNativeDoorbellObject(t, temp), buildCoroNativePollObject(t, temp), buildCoroNativeFleetOwnerObject(t, temp), ) prog.Dispose() - if len(objects) != len(allowed)+5 { - t.Fatalf("native fleet runtime objects = %d, want exactly %d package objects plus fault-string, worker, doorbell, poll, and fleet-owner leaves", len(objects), len(allowed)) + if len(objects) != len(allowed)+6 { + t.Fatalf("native fleet runtime objects = %d, want exactly %d package objects plus fault-string, allocation-cache, worker, doorbell, poll, and fleet-owner leaves", len(objects), len(allowed)) } return objects } diff --git a/internal/build/coro_spawn_native_e2e_test.go b/internal/build/coro_spawn_native_e2e_test.go index b7fe62e00f..47016684a8 100644 --- a/internal/build/coro_spawn_native_e2e_test.go +++ b/internal/build/coro_spawn_native_e2e_test.go @@ -915,12 +915,13 @@ func buildCoroSpawnNativeE2ERuntimeIsland(t *testing.T, temp string) []string { } objects = append(objects, buildCoroRuntimeIslandFaultStringStubs(t, prog, temp), + buildCoroNativeAllocationCacheObject(t, temp), buildCoroNativeWorkerCallObject(t, temp), buildCoroNativeDoorbellObject(t, temp), ) prog.Dispose() - if len(objects) != len(allowed)+3 { - t.Fatalf("production coroutine runtime island objects = %d, want exactly %d package objects plus fault-string, worker, and doorbell leaves", len(objects), len(allowed)) + if len(objects) != len(allowed)+4 { + t.Fatalf("production coroutine runtime island objects = %d, want exactly %d package objects plus fault-string, allocation-cache, worker, and doorbell leaves", len(objects), len(allowed)) } return objects } diff --git a/internal/build/coro_time_sleep_e2e_test.go b/internal/build/coro_time_sleep_e2e_test.go index f4456a90b9..91d2f87d33 100644 --- a/internal/build/coro_time_sleep_e2e_test.go +++ b/internal/build/coro_time_sleep_e2e_test.go @@ -560,8 +560,14 @@ func assertCoroTimeSleepDirectCoroCall(plan *coro.SSAPlan, call ssa.CallInstruct func assertCoroTimeSleepAwaitsIR(t *testing.T, label, body string, childSymbols ...string) { t.Helper() - if len(childSymbols) == 0 || strings.Count(body, coroTimeSleepAwaitHookV1) != len(childSymbols) { - t.Fatalf("%s await handoffs = %d, want %d:\n%s", label, strings.Count(body, coroTimeSleepAwaitHookV1), len(childSymbols), body) + if len(childSymbols) == 0 { + t.Fatalf("%s has no expected coroutine child", label) + } + handoffs := strings.Count(body, coroTimeSleepAwaitHookV1) + tailForward := len(childSymbols) == 1 && handoffs == 0 && + strings.Contains(body, "ret ptr") && !strings.Contains(body, "llvm.coro.") + if handoffs != len(childSymbols) && !tailForward { + t.Fatalf("%s await handoffs = %d, want %d or one frame-free tail forward:\n%s", label, handoffs, len(childSymbols), body) } previous := -1 for _, childSymbol := range childSymbols { @@ -570,9 +576,12 @@ func assertCoroTimeSleepAwaitsIR(t *testing.T, label, body string, childSymbols t.Fatalf("%s does not call coroutine child %q exactly once in source order:\n%s", label, childSymbol, body) } await := strings.Index(body[child:], coroTimeSleepAwaitHookV1) - if await < 0 { + if await < 0 && !tailForward { t.Fatalf("%s child %q is not followed by a structured await:\n%s", label, childSymbol, body) } - previous = child + await + previous = child + if await >= 0 { + previous += await + } } } diff --git a/internal/build/coro_worker_e2e_test.go b/internal/build/coro_worker_e2e_test.go index 20c0f25167..4e4ccd4166 100644 --- a/internal/build/coro_worker_e2e_test.go +++ b/internal/build/coro_worker_e2e_test.go @@ -24,6 +24,23 @@ import ( "testing" ) +// buildCoroNativeAllocationCacheObject materializes the LLGoFiles leaf owned +// by runtime/internal/coroalloc. Source-island E2E tests emit package LLVM +// modules directly, so their ordinary package link never sees this C object. +func buildCoroNativeAllocationCacheObject(t *testing.T, temp string) string { + t.Helper() + clang, err := exec.LookPath("clang") + if err != nil { + t.Skip("clang is unavailable") + } + source := filepath.Join("..", "..", "runtime", "internal", "coroalloc", "_cache", "cache.c") + object := filepath.Join(temp, "coro-allocation-cache.o") + if output, err := exec.Command(clang, "-std=c11", "-O2", "-c", source, "-o", object).CombinedOutput(); err != nil { + t.Fatalf("compile native coroutine allocation cache leaf: %v\n%s", err, output) + } + return object +} + // buildCoroNativeWorkerCallObject materializes the LLGoFiles leaf normally // owned by runtime/internal/coroworker. Source-island E2E tests emit package // LLVM modules themselves, so the ordinary package linker never gets a chance diff --git a/runtime/coro_task_allocation_source_test.go b/runtime/coro_task_allocation_source_test.go index 221a72898d..3329c68f52 100644 --- a/runtime/coro_task_allocation_source_test.go +++ b/runtime/coro_task_allocation_source_test.go @@ -41,7 +41,7 @@ func TestCoroSpawnFusesTaskAndRuntimeContextAllocation(t *testing.T) { for _, required := range []string{ "coroalloc.AllocTask(allocationSize)", "coro.Zero(raw, allocationSize)", - "coroBindTaskAllocationRuntimeContext(child, parent)", + "coroBindTaskAllocationRuntimeContextCompiler(child, parent)", "coroalloc.FreeTask(raw, allocationSize)", } { if !strings.Contains(spawn, required) { diff --git a/runtime/internal/coro/completion.go b/runtime/internal/coro/completion.go index c3c5c9dd59..6af4ef6718 100644 --- a/runtime/internal/coro/completion.go +++ b/runtime/internal/coro/completion.go @@ -261,7 +261,7 @@ func prepareChildPanic( ) bool { if !ValidG(g) || !resumeGateTaken(g) || status != ExplicitStatusPanic || typeWord == nil || frame == nil || frame.parent == nil || frame != g.active || frame.header != header || - header == nil || header.Flags != 0 || frame.state != FrameActive || + header == nil || frame.state != FrameActive || header.G != unsafe.Pointer(g) || header.SuspendReason != uint16(SuspendPanic) || header.Lifecycle != uint16(FrameFinalSuspended) || g.state != GRunning || g.runP == nil || g.pending.kind != pendingNone || diff --git a/runtime/internal/coro/explicit_status.go b/runtime/internal/coro/explicit_status.go index 2eeb6e15d3..f15fdf5e01 100644 --- a/runtime/internal/coro/explicit_status.go +++ b/runtime/internal/coro/explicit_status.go @@ -81,15 +81,15 @@ func LoadPanicRecord(g *G) (PanicRecordSnapshot, bool) { func validPanicAncestor(g *G, frame *Frame) bool { return frame != nil && frame.owner == g && frame.handle != nil && frame.header != nil && - frame.state == FrameSuspended && frame.header.G == unsafe.Pointer(g) && frame.header.Flags == 0 && + frame.state == FrameSuspended && frame.header.G == unsafe.Pointer(g) && frame.header.SuspendReason == uint16(SuspendCall) && frame.header.Lifecycle == uint16(FrameSuspended) } // validPanicAncestry proves before publication that every continuation which // terminal panic unwinding would bypass is a plain suspended await. Version -// zero has no cleanup/recover transport, so any non-zero flags reject the -// entire operation before the active frame or an ancestor can be destroyed. +// zero has no cleanup/recover transport; the compiler must lower those shapes +// through the structured cleanup protocol before this terminal boundary. func validPanicAncestry(g *G, active *Frame) bool { if g == nil || active == nil || active.header == nil { return false @@ -109,10 +109,10 @@ func validPanicAncestry(g *G, active *Frame) bool { // the publication attempt; any malformed winner permanently poisons the record // instead of allowing execution to continue with ambiguous terminal state. // -// HeaderV1.Flags must be zero. Thus cleanup/recover/Goexit/implicit-fault -// shapes cannot be smuggled through an unversioned flag convention. An -// untyped nil panic is also rejected; the compiler must first materialize the -// Go-version-appropriate non-nil panic type word. +// Cleanup/recover/Goexit/implicit-fault shapes use their dedicated structured +// protocols rather than an open-ended header convention. An untyped nil panic +// is also rejected; the compiler must first materialize the Go-version- +// appropriate non-nil panic type word. func PrepareExplicitStatus( g *G, handle unsafe.Pointer, @@ -145,7 +145,7 @@ func PrepareExplicitStatus( preemptStore(&record.status, explicitStatusRejected) return false } - if status != ExplicitStatusPanic || typeWord == nil || handle == nil || header == nil || header.Flags != 0 || + if status != ExplicitStatusPanic || typeWord == nil || handle == nil || header == nil || g.state != GRunning || g.active == nil || g.root == nil || g.runP == nil || g.pending.kind != pendingNone || g.pending.directChannel || g.pending.from != nil || g.pending.target != nil || g.destroyTarget != nil || g.destroyRoot || g.queued || g.nextReady != nil || diff --git a/runtime/internal/coro/explicit_status_test.go b/runtime/internal/coro/explicit_status_test.go index f5b3966fd8..1a51d25d03 100644 --- a/runtime/internal/coro/explicit_status_test.go +++ b/runtime/internal/coro/explicit_status_test.go @@ -46,7 +46,6 @@ func newExplicitPanicFixture(t *testing.T, depth int) *explicitPanicFixture { for index := range frames { handle := unsafe.Pointer(new(byte)) frames[index] = newTestFrame(t, g, handle, parent) - frames[index].header.StateID = uint32(index + 1) byHandle[handle] = frames[index] parent = handle } @@ -298,8 +297,6 @@ func TestExplicitPanicRetainsDescriptorTraceDeepestToRoot(t *testing.T) { lines := []uint32{0, 20, 11} for index, frame := range fixture.frames { descriptor := unsafe.Pointer(descriptors[index]) - metadata := FrameFromStorage(frame.storage) - metadata.descriptor = descriptor frame.descriptor = descriptor frame.header.Descriptor = descriptor frame.header.Line = lines[index] @@ -368,8 +365,6 @@ func TestActiveTraceFrameUsesCurrentCompilerDescriptor(t *testing.T) { Function: "main.viaGo", File: "/src/main.go", } - metadata := FrameFromStorage(frame.storage) - metadata.descriptor = unsafe.Pointer(descriptor) frame.descriptor = unsafe.Pointer(descriptor) frame.header.Descriptor = unsafe.Pointer(descriptor) frame.header.Line = 17 @@ -483,14 +478,11 @@ func TestExplicitStatusUnsupportedShapesFailClosed(t *testing.T) { name string status ExplicitStatus typeWord bool - flags uint32 }{ {name: "normal return", status: ExplicitStatusReturn, typeWord: true}, {name: "goexit", status: ExplicitStatusGoexit, typeWord: true}, {name: "implicit fault", status: ExplicitStatusImplicitFault, typeWord: true}, {name: "explicit nil", status: ExplicitStatusPanic}, - {name: "cleanup", status: ExplicitStatusPanic, typeWord: true, flags: 1 << 0}, - {name: "recover", status: ExplicitStatusPanic, typeWord: true, flags: 1 << 1}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -498,7 +490,6 @@ func TestExplicitStatusUnsupportedShapesFailClosed(t *testing.T) { leaf := fixture.frames[0] leaf.header.SuspendReason = uint16(SuspendPanic) leaf.header.Lifecycle = uint16(FrameFinalSuspended) - leaf.header.Flags = test.flags var typeWord unsafe.Pointer if test.typeWord { typeWord = unsafe.Pointer(new(byte)) @@ -512,7 +503,6 @@ func TestExplicitStatusUnsupportedShapesFailClosed(t *testing.T) { if record, ok := LoadPanicRecord(fixture.g); ok || record != (PanicRecordSnapshot{}) { t.Fatalf("rejected explicit terminal shape published record (%+v, %t)", record, ok) } - leaf.header.Flags = 0 if PreparePanic(fixture.g, leaf.handle, leaf.header, unsafe.Pointer(new(byte)), unsafe.Pointer(new(byte))) { t.Fatal("poisoned one-shot record accepted a later supported panic") } @@ -521,49 +511,6 @@ func TestExplicitStatusUnsupportedShapesFailClosed(t *testing.T) { } } -func TestExplicitPanicRejectsUnsupportedAncestorBeforeDestroy(t *testing.T) { - fixture := newExplicitPanicFixture(t, 2) - root, leaf := fixture.frames[0], fixture.frames[1] - rootMetadata, leafMetadata := FrameFromStorage(root.storage), FrameFromStorage(leaf.storage) - root.header.Flags = 1 // cleanup/recover metadata is not representable in v0. - leaf.header.SuspendReason = uint16(SuspendPanic) - leaf.header.Lifecycle = uint16(FrameFinalSuspended) - if PreparePanic(fixture.g, leaf.handle, leaf.header, unsafe.Pointer(new(byte)), unsafe.Pointer(new(byte))) { - t.Fatal("panic with unsupported suspended ancestor was published") - } - if fixture.g.pending.kind != pendingNone || fixture.g.panicUnwind || fixture.g.destroyTarget != nil || - leafMetadata.state != FrameActive || rootMetadata.state != FrameSuspended || - leaf.header.Lifecycle != uint16(FrameFinalSuspended) || root.header.Lifecycle != uint16(FrameSuspended) { - t.Fatal("rejected ancestor cleanup mutated frame destruction state") - } - if record, ok := LoadPanicRecord(fixture.g); ok || record != (PanicRecordSnapshot{}) { - t.Fatalf("rejected ancestor cleanup published record (%+v, %t)", record, ok) - } - runtime.KeepAlive(root.memory) - runtime.KeepAlive(leaf.memory) -} - -func TestExplicitPanicRechecksAncestorBeforeDirectDestroy(t *testing.T) { - fixture := newExplicitPanicFixture(t, 2) - root := fixture.frames[0] - rootMetadata := FrameFromStorage(root.storage) - fixture.publish(t, unsafe.Pointer(new(byte)), unsafe.Pointer(new(byte))) - action := fixture.beginPanicDestroy(t) - fixture.release(t, action) - - // Model corrupted or version-skewed metadata after publication. The active - // panic frame may already be gone, but the unsupported ancestor must never - // be directly destroyed or resumed. - root.header.Flags = 1 - next, ok := Destroyed(fixture.p, fixture.g, action) - if ok || next != (Action{}) || fixture.g.destroyTarget != nil || - rootMetadata.state != FrameSuspended || root.header.Lifecycle != uint16(FrameSuspended) { - t.Fatalf("unsupported ancestor entered direct destroy: action=(%+v, %t), state=%d lifecycle=%d", - next, ok, rootMetadata.state, root.header.Lifecycle) - } - runtime.KeepAlive(root.memory) -} - func TestExplicitPanicTerminalScheduleRaceDoesNotRedestroy(t *testing.T) { tests := []struct { name string diff --git a/runtime/internal/coro/frame.go b/runtime/internal/coro/frame.go index 3601faf979..72d4ab1d61 100644 --- a/runtime/internal/coro/frame.go +++ b/runtime/internal/coro/frame.go @@ -32,9 +32,7 @@ type HeaderV1 struct { ResultSlot unsafe.Pointer SuspendReason uint16 Lifecycle uint16 - StateID uint32 Line uint32 - Flags uint32 } // FrameDescriptorV1 is the runtime prefix emitted for every physical @@ -81,7 +79,12 @@ const ( ) // FrameState values deliberately match the lifecycle field emitted by cl. -type FrameState uint16 +// FrameState has eight values and is scheduler-private. Keep it byte-sized so +// the lifecycle/context/retention flags fill one pointer-aligned word after +// panicLine instead of forcing a padding word in every physical frame. The +// compiler-facing HeaderV1 lifecycle remains uint16 and is converted at the +// validation boundary. +type FrameState uint8 const ( FrameAllocated FrameState = iota @@ -91,6 +94,10 @@ const ( FrameFinalSuspended FrameDestroyPending FrameDestroyed + // FrameTraceRetained is no longer schedulable. Its dead header word carries + // the immutable descriptor while handle carries the raw allocation base; + // this avoids two duplicate pointers in every live physical frame. + FrameTraceRetained ) // frameRuntimeContextMode is a publication-time cache of the immutable @@ -155,10 +162,11 @@ type pendingTransition struct { target *Frame } -// Frame is scheduler-owned metadata. It lives at the beginning of the same -// allocation as the aligned LLVM frame storage. A back-pointer immediately -// before storage makes the free hook independent of maps, TLS, pthreads, -// libuv, and any particular garbage collector. +// Frame is scheduler-owned metadata. Compiler-created dynamic and elided +// coroutines place it in BorrowedFrameStorageV2 inside LLVM frame storage; +// compatibility constructors may still place it at the allocation base. A +// back-pointer immediately before storage keeps lookup independent of maps, +// TLS, pthreads, libuv, and any particular garbage collector. type Frame struct { owner *G handle unsafe.Pointer @@ -175,11 +183,6 @@ type Frame struct { // direct-parking LLVM coroutine frame; ordinary frames pay only this // metadata pointer during the first migration stage. parkWait *WaitSetRecord - storage unsafe.Pointer - rawBase unsafe.Pointer - descriptor unsafe.Pointer - size uintptr - align uintptr allocationSize uintptr panicLine uint32 state FrameState @@ -202,12 +205,18 @@ type Frame struct { // Frame stored inside an elidable coroutine. It is intentionally opaque to // generated code: the compiler only reserves and forwards this pointer, while // the runtime initializes and remains the sole owner of Frame's private -// layout. Twenty pointer -// words cover both wasm32 and native layouts with expansion room; the compile- -// time assertion fails if runtime metadata ever outgrows the ABI capacity. -type BorrowedFrameStorageV2 [20]uintptr +// layout. The compile-time assertion fails if runtime metadata ever outgrows +// the ABI capacity. Twelve words exactly cover the native layout. Pointer-32 targets +// need one additional word because CompletionRecord's uint32 status cannot +// share pointer alignment padding; the compiler mirrors this formula from the +// target pointer width. Dynamic frames record only the exact allocation size; their raw +// base stays in HeaderV1 until destruction and moves into the dead handle word +// only for the rare retained-panic trace. Keeping that pointer out of every +// live Frame preserves the compact layout on native, wasm, and bare metal. +type BorrowedFrameStorageV2 [12 + 4/unsafe.Sizeof(uintptr(0))]uintptr var _ [int(unsafe.Sizeof(BorrowedFrameStorageV2{})) - int(unsafe.Sizeof(Frame{}))]byte +var _ [int(unsafe.Sizeof(Frame{})) - int(unsafe.Sizeof(BorrowedFrameStorageV2{}))]byte // ValidG reports whether g has been initialized as a coroutine task. func ValidG(g *G) bool { @@ -217,10 +226,14 @@ func ValidG(g *G) bool { // FrameAllocationSize returns the single-allocation size needed for Frame, // the storage back-pointer, alignment padding, and the LLVM coroutine frame. func FrameAllocationSize(size, align uintptr) (uintptr, bool) { + overhead := unsafe.Sizeof(Frame{}) + unsafe.Sizeof(uintptr(0)) + return frameStorageAllocationSize(size, align, overhead) +} + +func frameStorageAllocationSize(size, align, overhead uintptr) (uintptr, bool) { if align == 0 || align&(align-1) != 0 { return 0, false } - overhead := unsafe.Sizeof(Frame{}) + unsafe.Sizeof(uintptr(0)) max := ^uintptr(0) if overhead > max-(align-1) { return 0, false @@ -232,6 +245,26 @@ func FrameAllocationSize(size, align uintptr) (uintptr, bool) { return overhead + size, true } +// compilerFrameAllocationReceiptV1 is the temporary prefix immediately before +// dynamically allocated LLVM storage. Before publication it carries the raw +// allocator receipt; publication moves total into the compiler-injected Frame +// metadata and replaces the last word with FrameFromStorage's back-pointer. +// The raw base remains in HeaderV1.AllocationBase until physical destruction. +type compilerFrameAllocationReceiptV1 struct { + raw unsafe.Pointer + total uintptr +} + +// CompilerFrameAllocationSize is the compact compiler-owned allocation shape. +// Every coroutine already reserves BorrowedFrameStorageV2 inside its LLVM +// frame for allocation elision. Dynamic frames use that same storage instead +// of prepending a second Frame, leaving only this two-word allocation receipt. +func CompilerFrameAllocationSize(size, align uintptr) (uintptr, bool) { + return frameStorageAllocationSize( + size, align, unsafe.Sizeof(compilerFrameAllocationReceiptV1{}), + ) +} + // AlignedStorage locates LLVM frame storage within a combined allocation. func AlignedStorage(raw unsafe.Pointer, align uintptr) (unsafe.Pointer, bool) { if raw == nil || align == 0 || align&(align-1) != 0 { @@ -246,6 +279,10 @@ func AlignedStorage(raw unsafe.Pointer, align uintptr) (unsafe.Pointer, bool) { func alignedStorageOffset(base, align uintptr) (uintptr, bool) { offset := unsafe.Sizeof(Frame{}) + unsafe.Sizeof(uintptr(0)) + return frameStorageOffset(base, align, offset) +} + +func frameStorageOffset(base, align, offset uintptr) (uintptr, bool) { if offset > ^uintptr(0)-base { return 0, false } @@ -257,6 +294,65 @@ func alignedStorageOffset(base, align uintptr) (uintptr, bool) { return offset + padding, true } +func compilerAlignedStorage(raw unsafe.Pointer, align uintptr) (unsafe.Pointer, bool) { + if raw == nil || align == 0 || align&(align-1) != 0 { + return nil, false + } + base := uintptr(raw) + offset, ok := frameStorageOffset( + base, align, unsafe.Sizeof(compilerFrameAllocationReceiptV1{}), + ) + if !ok { + return nil, false + } + return unsafe.Add(raw, offset), true +} + +func compilerFrameReceipt(storage unsafe.Pointer) (*compilerFrameAllocationReceiptV1, bool) { + if storage == nil { + return nil, false + } + size := unsafe.Sizeof(compilerFrameAllocationReceiptV1{}) + if uintptr(storage) < size { + return nil, false + } + return (*compilerFrameAllocationReceiptV1)(unsafe.Add(storage, -int(size))), true +} + +func allocationContains(raw unsafe.Pointer, total uintptr, value unsafe.Pointer, size uintptr) bool { + if raw == nil || value == nil { + return false + } + base, address := uintptr(raw), uintptr(value) + if total > ^uintptr(0)-base || address < base || address > base+total { + return false + } + return size <= base+total-address +} + +func validFrameAllocationIdentity(frame *Frame) bool { + if frame == nil || frame.header == nil { + return false + } + if frame.borrowedStorage { + return frame.allocationSize == 0 && + frame.header.AllocationBase == unsafe.Pointer(frame) + } + return frame.allocationSize != 0 && + allocationContains( + frame.header.AllocationBase, frame.allocationSize, + unsafe.Pointer(frame), unsafe.Sizeof(Frame{}), + ) +} + +func rangesOverlap(first unsafe.Pointer, firstSize uintptr, second unsafe.Pointer, secondSize uintptr) bool { + firstAddress, secondAddress := uintptr(first), uintptr(second) + if firstSize > ^uintptr(0)-firstAddress || secondSize > ^uintptr(0)-secondAddress { + return true + } + return firstAddress < secondAddress+secondSize && secondAddress < firstAddress+firstSize +} + // Zero clears size bytes beginning at ptr without introducing a libc/runtime // dependency into the scheduler core. func Zero(ptr unsafe.Pointer, size uintptr) { @@ -281,11 +377,9 @@ func RegisterFrame(g *G, raw unsafe.Pointer, total, size, align uintptr, descrip } frame := (*Frame)(raw) frame.owner = g - frame.storage = storage - frame.rawBase = raw - frame.descriptor = descriptor - frame.size = size - frame.align = align + // FrameAllocated has no compiler header yet. Stage the descriptor in the + // header word; PublishFrame consumes it before the frame can run. + frame.header = (*HeaderV1)(descriptor) frame.allocationSize = total frame.state = FrameAllocated frame.next = g.frames @@ -295,6 +389,32 @@ func RegisterFrame(g *G, raw unsafe.Pointer, total, size, align uintptr, descrip return storage, true } +// RegisterFrameCompiler prepares a compact, zero-filled compiler allocation. +// No scheduler action can occur between this call and PublishFrameV3Compiler, +// so the Frame embedded in LLVM storage is linked only once its handle and +// header exist. The two-word receipt is then replaced by the normal storage +// back-pointer without a map or a second allocation. +func RegisterFrameCompiler( + g *G, raw unsafe.Pointer, total, size, align uintptr, descriptor unsafe.Pointer, +) (unsafe.Pointer, bool) { + want, layoutOK := CompilerFrameAllocationSize(size, align) + if !ValidG(g) || raw == nil || descriptor == nil || !layoutOK || total != want || + uintptr(raw)%unsafe.Alignof(Frame{}) != 0 { + return nil, false + } + storage, ok := compilerAlignedStorage(raw, align) + base, address := uintptr(raw), uintptr(storage) + if !ok || address < base || address-base >= total || size > total-(address-base) { + return nil, false + } + receipt, ok := compilerFrameReceipt(storage) + if !ok { + return nil, false + } + *receipt = compilerFrameAllocationReceiptV1{raw: raw, total: total} + return storage, true +} + // FrameFromStorage obtains scheduler metadata through the back-pointer stored // immediately before LLVM coroutine frame storage. func FrameFromStorage(storage unsafe.Pointer) *Frame { @@ -324,9 +444,13 @@ func PublishFrame(g *G, handle unsafe.Pointer, header *HeaderV1, storage unsafe. return false } frame := FrameFromStorage(storage) - if frame == nil || frame.owner != g || frame.storage != storage || frame.state != FrameAllocated || - frame.handle != nil || frame.header != nil || header.G != unsafe.Pointer(g) || - header.Descriptor != frame.descriptor || header.Lifecycle != uint16(FrameInitialSuspended) || + if frame == nil { + return false + } + descriptor := unsafe.Pointer(frame.header) + if frame.owner != g || frame.state != FrameAllocated || + frame.handle != nil || descriptor == nil || header.G != unsafe.Pointer(g) || + header.Descriptor != descriptor || header.Lifecycle != uint16(FrameInitialSuspended) || header.SuspendReason != uint16(SuspendNone) { return false } @@ -336,7 +460,7 @@ func PublishFrame(g *G, handle unsafe.Pointer, header *HeaderV1, storage unsafe. frame.handle = handle frame.header = header frame.state = FrameInitialSuspended - header.AllocationBase = frame.rawBase + header.AllocationBase = unsafe.Pointer(frame) return true } @@ -378,7 +502,6 @@ func PublishFrameV2( frame.owner = g frame.handle = handle frame.header = header - frame.descriptor = header.Descriptor frame.state = FrameInitialSuspended frame.runtimeContext = mode frame.borrowedStorage = true @@ -392,7 +515,7 @@ func PublishFrameV2( // publishes either dynamic storage or compiler-borrowed metadata. Keeping the // initialization in this shared helper makes coroutine ramps small: generated // code supplies only immutable descriptor/result operands and never duplicates -// the scheduler header's ten-field initialization sequence. +// the scheduler header's eight-field initialization sequence. func PublishFrameV3( g *G, handle unsafe.Pointer, header *HeaderV1, storage, metadata, descriptor, resultSlot unsafe.Pointer, @@ -410,6 +533,85 @@ func PublishFrameV3( return PublishFrameV2(g, handle, header, storage, metadata) } +// PublishFrameV3Compiler is the adjacent generated-ramp publication lane. +// RegisterFrameCompiler has just prepended a dynamic frame, or metadata names +// the compiler-owned borrowed slot for this exact handle. No scheduler action +// can run between those operations, so the new head is an O(1) uniqueness +// certificate and a second findFrame walk establishes no additional safety. +func PublishFrameV3Compiler( + g *G, handle unsafe.Pointer, header *HeaderV1, storage, metadata, + descriptor, resultSlot unsafe.Pointer, +) bool { + if !ValidG(g) || handle == nil || header == nil || metadata == nil || descriptor == nil { + return false + } + mode, ok := descriptorRuntimeContextMode(descriptor) + if !ok { + return false + } + if storage != nil { + receipt, receiptOK := compilerFrameReceipt(storage) + if !receiptOK || receipt.raw == nil || receipt.total == 0 || + uintptr(receipt.raw)%unsafe.Alignof(Frame{}) != 0 || + metadata == unsafe.Pointer(g) || metadata == unsafe.Pointer(header) || + uintptr(metadata)%unsafe.Alignof(Frame{}) != 0 || + uintptr(unsafe.Pointer(header)) < uintptr(storage) || uintptr(metadata) < uintptr(storage) || + !allocationContains(receipt.raw, receipt.total, unsafe.Pointer(receipt), unsafe.Sizeof(*receipt)) || + !allocationContains(receipt.raw, receipt.total, storage, 1) || + !allocationContains(receipt.raw, receipt.total, unsafe.Pointer(header), unsafe.Sizeof(HeaderV1{})) || + !allocationContains(receipt.raw, receipt.total, metadata, unsafe.Sizeof(Frame{})) || + rangesOverlap(unsafe.Pointer(header), unsafe.Sizeof(HeaderV1{}), metadata, unsafe.Sizeof(Frame{})) || + findFrame(g, handle) != nil { + return false + } + frame := (*Frame)(metadata) + *header = HeaderV1{ + G: unsafe.Pointer(g), + Descriptor: descriptor, + AllocationBase: receipt.raw, + ResultSlot: resultSlot, + SuspendReason: uint16(SuspendNone), + Lifecycle: uint16(FrameInitialSuspended), + } + // AllocFrame guarantees a zero-filled range. Initialize only live words; + // unlike borrowed storage this dynamic metadata cannot contain state from + // an earlier parent-frame activation. + frame.owner = g + frame.handle = handle + frame.header = header + frame.allocationSize = receipt.total + frame.state = FrameInitialSuspended + frame.runtimeContext = mode + frame.next = g.frames + g.frames = frame + receipt.total = uintptr(unsafe.Pointer(frame)) + return true + } + if metadata == unsafe.Pointer(g) || metadata == unsafe.Pointer(header) || + uintptr(metadata)%unsafe.Alignof(Frame{}) != 0 { + return false + } + *header = HeaderV1{ + G: unsafe.Pointer(g), + Descriptor: descriptor, + ResultSlot: resultSlot, + SuspendReason: uint16(SuspendNone), + Lifecycle: uint16(FrameInitialSuspended), + } + Zero(metadata, unsafe.Sizeof(Frame{})) + frame := (*Frame)(metadata) + frame.owner = g + frame.handle = handle + frame.header = header + frame.state = FrameInitialSuspended + frame.runtimeContext = mode + frame.borrowedStorage = true + frame.next = g.frames + g.frames = frame + header.AllocationBase = metadata + return true +} + // PrepareAwait records a parent-to-child handoff. It never resumes either // coroutine; only the runtime driver may perform handle operations requested // by the scheduler action protocol. @@ -552,9 +754,11 @@ func PrepareCompleteStatus(g *G, handle unsafe.Pointer, header *HeaderV1, status } // PrepareCompleteStatusCompiler handles the dominant ordinary-return suffix -// from compiler-generated code using the active-frame and parent-completion -// certificates already established by PrepareAwaitCompletionCompiler. Panic, -// Goexit, cancellation, roots, and any uncertain shape retain the complete +// from compiler-generated code using the active-frame certificate and, for a +// managed child, the parent-completion certificate already established by +// PrepareAwaitCompletionCompiler. A root has no CompletionRecord publisher; +// its exact singleton frame chain is itself the corresponding certificate. +// Panic, Goexit, cancellation, and any uncertain shape retain the complete // status validator below. func PrepareCompleteStatusCompiler(g *G, handle unsafe.Pointer, header *HeaderV1, status CompletionStatus) bool { if status == CompletionReturn && ValidG(g) && resumeGateTaken(g) && @@ -562,15 +766,22 @@ func PrepareCompleteStatusCompiler(g *G, handle unsafe.Pointer, header *HeaderV1 g.spawnChild == nil && compilerReleasableParkState(&g.park) && g.park.taskCancelPhase != taskCancelRequested { frame := g.active - if frame != nil && frame.parent != nil && frame.handle == handle && frame.header == header && + if frame != nil && frame.handle == handle && frame.header == header && frame.owner == g && frame.state == FrameActive && header.SuspendReason == uint16(SuspendFrameComplete) && header.Lifecycle == uint16(FrameFinalSuspended) { - if awaitCompletionArmedForChild(frame) && + if frame.parent != nil && awaitCompletionArmedForChild(frame) && publishAwaitCompletion(frame.parent, CompletionReturn, nil, nil) { g.pending = pendingTransition{kind: pendingComplete, from: frame} return true } + if frame.parent == nil && frame == g.root && g.frames == frame && frame.next == nil && + header.Parent == nil && frame.parkWait == nil && + emptyCompletionRecord(&frame.completion) && + g.park.taskCancelKind == TaskCancelNone { + g.pending = pendingTransition{kind: pendingComplete, from: frame} + return true + } } } return PrepareCompleteStatus(g, handle, header, status) @@ -659,14 +870,17 @@ func ReleaseFrame(g *G, storage unsafe.Pointer, size, align uintptr, descriptor if !ValidG(g) || !gPreemptEnabledAtDepthZero(g) || storage == nil { return nil, 0, false } + total, layoutOK := FrameAllocationSize(size, align) frame := FrameFromStorage(storage) - if frame == nil || frame.owner != g || frame.storage != storage || frame.size != size || - frame.align != align || frame.descriptor != descriptor || frame.state != FrameDestroyPending || - g.destroyTarget != frame || frame.header == nil || frame.parkWait != nil || + aligned, storageOK := AlignedStorage(unsafe.Pointer(frame), align) + if frame == nil || !layoutOK || !storageOK || aligned != storage || + frame.owner != g || frame.header == nil || frame.header.Descriptor != descriptor || frame.allocationSize != total || + frame.state != FrameDestroyPending || + g.destroyTarget != frame || frame.parkWait != nil || frame.header.Lifecycle != uint16(FrameDestroyPending) { return nil, 0, false } - raw, total := frame.rawBase, frame.allocationSize + raw := unsafe.Pointer(frame) if !unlinkFrame(g, frame) { return nil, 0, false } @@ -677,6 +891,40 @@ func ReleaseFrame(g *G, storage unsafe.Pointer, size, align uintptr, descriptor return raw, total, true } +// ReleaseFrameCompiler consumes the exact llvm.coro.free callback adjacent to +// one scheduler-issued destroy. The destroy target and newest frame are the +// same dynamic allocation on the ordinary managed path. Exceptional or legacy +// shapes retain ReleaseFrame's complete unlink search and validation. +func ReleaseFrameCompiler( + g *G, storage unsafe.Pointer, size, align uintptr, descriptor unsafe.Pointer, +) (metadata, raw unsafe.Pointer, total uintptr, ok bool) { + if ValidG(g) && gPreemptEnabledAtDepthZero(g) && storage != nil { + want, layoutOK := CompilerFrameAllocationSize(size, align) + frame := g.destroyTarget + var allocationBase unsafe.Pointer + if frame != nil && frame.header != nil { + allocationBase = frame.header.AllocationBase + } + aligned, storageOK := compilerAlignedStorage(allocationBase, align) + if frame != nil && layoutOK && storageOK && aligned == storage && + FrameFromStorage(storage) == frame && g.frames == frame && + frame.owner == g && frame.header != nil && frame.header.Descriptor == descriptor && frame.allocationSize == want && + frame.state == FrameDestroyPending && frame.parkWait == nil && + allocationContains(allocationBase, want, unsafe.Pointer(frame), unsafe.Sizeof(Frame{})) && + frame.header.Lifecycle == uint16(FrameDestroyPending) { + g.frames = frame.next + frame.next = nil + frame.state = FrameDestroyed + frame.panicLine = frame.header.Line + frame.header.Lifecycle = uint16(FrameDestroyed) + g.destroyTarget = nil + return unsafe.Pointer(frame), allocationBase, want, true + } + } + legacyRaw, legacyTotal, legacyOK := ReleaseFrame(g, storage, size, align, descriptor) + return legacyRaw, legacyRaw, legacyTotal, legacyOK +} + // CommitFrameDestroyV2 completes the physical destroy of an allocation-elided // static frame. A dynamic frame has already been unlinked by ReleaseFrame and // is accepted without a second mutation. The adapter invokes this immediately @@ -691,8 +939,8 @@ func CommitFrameDestroyV2(g *G, handle unsafe.Pointer) bool { // schedulable frame with the same handle would make that receipt invalid. return findFrame(g, handle) == nil } - if frame.handle != handle || !frame.borrowedStorage || frame.storage != nil || - frame.rawBase != nil || frame.allocationSize != 0 || frame.owner != g || + if frame.handle != handle || !frame.borrowedStorage || + frame.allocationSize != 0 || frame.owner != g || frame.header == nil || frame.parkWait != nil || frame.state != FrameDestroyPending || frame.header.AllocationBase != unsafe.Pointer(frame) || frame.header.Lifecycle != uint16(FrameDestroyPending) || !unlinkFrame(g, frame) { @@ -758,13 +1006,17 @@ func ActiveTraceFrame(g *G) (PanicTraceFrameSnapshot, bool) { parkResumeHeader := header != nil && header.SuspendReason == uint16(SuspendPark) && header.Lifecycle == uint16(FrameSuspended) + var descriptorPointer unsafe.Pointer + if header != nil { + descriptorPointer = header.Descriptor + } if frame.owner != g || frame.handle == nil || header == nil || - frame.descriptor == nil || frame.state != FrameActive || - header.G != unsafe.Pointer(g) || header.Descriptor != frame.descriptor || + descriptorPointer == nil || frame.state != FrameActive || + header.G != unsafe.Pointer(g) || header.Descriptor != descriptorPointer || (!activeHeader && !parkResumeHeader) { return PanicTraceFrameSnapshot{}, false } - descriptor := (*FrameDescriptorV1)(frame.descriptor) + descriptor := (*FrameDescriptorV1)(descriptorPointer) if descriptor.Version != 1 || descriptor.Flags & ^frameDescriptorAllowedFlagsV1 != 0 || len(descriptor.Function) == 0 { @@ -869,20 +1121,31 @@ func retainPanicTraceFrameMetadata( return false } if frame.borrowedStorage { - if raw != nil || total != 0 || frame.rawBase != nil || frame.allocationSize != 0 || - frame.storage != nil { + if raw != nil || total != 0 || frame.allocationSize != 0 { + return false + } + } else { + if raw == nil || total == 0 || frame.allocationSize != total { + return false + } + // Compatibility frames live at raw. Compact compiler frames live inside + // the LLVM range and carry raw in their still-live header until this + // transaction deliberately clears it. + if raw != unsafe.Pointer(frame) && + (frame.header == nil || frame.header.AllocationBase != raw || + !allocationContains(raw, total, unsafe.Pointer(frame), unsafe.Sizeof(Frame{}))) { return false } - } else if raw == nil || total == 0 || frame.rawBase != raw || - frame.allocationSize != total || raw != unsafe.Pointer(frame) { - return false } - if frame.owner != g || - frame.state != FrameDestroyed || frame.next != nil || frame.descriptor == nil || + if frame.owner != g || frame.state != FrameDestroyed || frame.next != nil || frame.header == nil || !emptyCompletionRecord(&frame.completion) { return false } - descriptor := (*FrameDescriptorV1)(frame.descriptor) + descriptorPointer := frame.header.Descriptor + if descriptorPointer == nil { + return false + } + descriptor := (*FrameDescriptorV1)(descriptorPointer) if descriptor.Version != 1 || descriptor.Flags & ^frameDescriptorAllowedFlagsV1 != 0 || len(descriptor.Function) == 0 { @@ -900,7 +1163,17 @@ func retainPanicTraceFrameMetadata( return false } } - frame.header = nil + if !frame.borrowedStorage { + // The LLVM handle is dead and the frame is no longer schedulable. Reuse + // its handle word for the physical allocation base so compact metadata + // embedded inside LLVM storage can later release the whole allocation + // without adding a permanent pointer to every live frame. + frame.handle = raw + } + // The compiler header dies with the LLVM frame. Preserve only its immutable + // descriptor in the same pointer word for diagnostic trace iteration. + frame.header = (*HeaderV1)(descriptorPointer) + frame.state = FrameTraceRetained frame.retainPanicTrace = false if emptyPanicTrace(g) { frame.completion = CompletionRecord{ @@ -958,7 +1231,15 @@ func RetainPendingPanicTraceFrame(g *G, raw unsafe.Pointer, total uintptr) bool if !ValidG(g) || raw == nil { return false } - frame := (*Frame)(raw) + return retainPendingPanicTraceFrame(g, (*Frame)(raw), raw, total) +} + +func retainPendingPanicTraceFrame( + g *G, frame *Frame, raw unsafe.Pointer, total uintptr, +) bool { + if !ValidG(g) || frame == nil || raw == nil { + return false + } if !frame.retainPanicTrace || frame.parent == nil || frame.parent.completion.status != CompletionPanic || frame.parent.completion.child != frame.handle || @@ -966,7 +1247,21 @@ func RetainPendingPanicTraceFrame(g *G, raw unsafe.Pointer, total uintptr) bool return false } record := frame.parent.completion - return retainPanicTraceFrame(g, raw, total, record.typeWord, record.dataWord) + return retainPanicTraceFrameMetadata( + g, frame, raw, total, record.typeWord, record.dataWord, + ) +} + +// RetainPendingPanicTraceFrameCompiler retains metadata injected into the +// LLVM frame while preserving the distinct physical allocation base returned +// by the compact compiler allocator. +func RetainPendingPanicTraceFrameCompiler( + g *G, metadata, raw unsafe.Pointer, total uintptr, +) bool { + if metadata == nil { + return false + } + return retainPendingPanicTraceFrame(g, (*Frame)(metadata), raw, total) } // RetainPanicTraceFrame transfers one terminal command-root frame allocation @@ -982,6 +1277,21 @@ func RetainPanicTraceFrame(g *G, raw unsafe.Pointer, total uintptr) bool { return retainPanicTraceFrame(g, raw, total, record.typeWord, record.dataWord) } +// RetainPanicTraceFrameCompiler is the compact-allocation form of +// RetainPanicTraceFrame. metadata may live inside raw rather than at its base. +func RetainPanicTraceFrameCompiler( + g *G, metadata, raw unsafe.Pointer, total uintptr, +) bool { + if !ValidG(g) || metadata == nil || !g.panicUnwind || + !publishedPanicRecord(&g.panicRecord) { + return false + } + record := &g.panicRecord + return retainPanicTraceFrameMetadata( + g, (*Frame)(metadata), raw, total, record.typeWord, record.dataWord, + ) +} + // PanicTraceDiscardPending reports the transient adapter-owned drain state. // It is true only between a successful recovery/replacement transaction and // the synchronous runtime hook that releases the detached allocations. @@ -1004,15 +1314,15 @@ func TakeDiscardedPanicTraceFrame(g *G) (raw unsafe.Pointer, total uintptr, ok b return nil, 0, false } frame := g.panicTraceHead - if frame.owner != g || frame.state != FrameDestroyed || - frame.header != nil || frame.descriptor == nil { + if frame.owner != g || frame.state != FrameTraceRetained || + frame.header == nil { return nil, 0, false } if frame.borrowedStorage { - if frame.rawBase != nil || frame.allocationSize != 0 || frame.storage != nil { + if frame.allocationSize != 0 { return nil, 0, false } - } else if frame.rawBase != unsafe.Pointer(frame) || frame.allocationSize == 0 { + } else if frame.allocationSize == 0 { return nil, 0, false } g.panicTraceHead = frame.next @@ -1023,7 +1333,11 @@ func TakeDiscardedPanicTraceFrame(g *G) (raw unsafe.Pointer, total uintptr, ok b Zero(unsafe.Pointer(frame), unsafe.Sizeof(Frame{})) continue } - return frame.rawBase, frame.allocationSize, true + raw := frame.handle + if raw == nil { + return nil, 0, false + } + return raw, frame.allocationSize, true } } @@ -1045,12 +1359,12 @@ func LoadPanicTraceFrame(g *G, cursor unsafe.Pointer) (snapshot PanicTraceFrameS return PanicTraceFrameSnapshot{}, nil, false } frame := (*Frame)(cursor) - if frame.owner != g || frame.state != FrameDestroyed || frame.header != nil || frame.descriptor == nil || - frame.borrowedStorage && (frame.rawBase != nil || frame.allocationSize != 0 || frame.storage != nil) || - !frame.borrowedStorage && (frame.rawBase != cursor || frame.allocationSize == 0) { + if frame.owner != g || frame.state != FrameTraceRetained || frame.header == nil || + frame.borrowedStorage && frame.allocationSize != 0 || + !frame.borrowedStorage && frame.allocationSize == 0 { return PanicTraceFrameSnapshot{}, nil, false } - descriptor := (*FrameDescriptorV1)(frame.descriptor) + descriptor := (*FrameDescriptorV1)(unsafe.Pointer(frame.header)) if descriptor.Version != 1 || descriptor.Flags & ^frameDescriptorAllowedFlagsV1 != 0 || len(descriptor.Function) == 0 { diff --git a/runtime/internal/coro/frame_test.go b/runtime/internal/coro/frame_test.go index 3e4aa4ad00..8d1103d582 100644 --- a/runtime/internal/coro/frame_test.go +++ b/runtime/internal/coro/frame_test.go @@ -38,16 +38,14 @@ func TestHeaderV1TargetNeutralLayout(t *testing.T) { {"ResultSlot", unsafe.Offsetof(header.ResultSlot), 4 * pointerSize}, {"SuspendReason", unsafe.Offsetof(header.SuspendReason), 5 * pointerSize}, {"Lifecycle", unsafe.Offsetof(header.Lifecycle), 5*pointerSize + 2}, - {"StateID", unsafe.Offsetof(header.StateID), 5*pointerSize + 4}, - {"Line", unsafe.Offsetof(header.Line), 5*pointerSize + 8}, - {"Flags", unsafe.Offsetof(header.Flags), 5*pointerSize + 12}, + {"Line", unsafe.Offsetof(header.Line), 5*pointerSize + 4}, } for _, field := range wants { if field.got != field.want { t.Fatalf("HeaderV1.%s offset = %d, want %d", field.name, field.got, field.want) } } - rawSize := 5*pointerSize + 16 + rawSize := 5*pointerSize + 8 wantSize := (rawSize + pointerSize - 1) &^ (pointerSize - 1) if got := unsafe.Sizeof(header); got != wantSize { t.Fatalf("HeaderV1 size = %d, want %d", got, wantSize) @@ -55,8 +53,8 @@ func TestHeaderV1TargetNeutralLayout(t *testing.T) { } func TestFrameAllocationLayout(t *testing.T) { - if got, capacity := unsafe.Sizeof(Frame{}), unsafe.Sizeof(BorrowedFrameStorageV2{}); got > capacity { - t.Fatalf("Frame size = %d, borrowed ABI capacity = %d", got, capacity) + if got, capacity := unsafe.Sizeof(Frame{}), unsafe.Sizeof(BorrowedFrameStorageV2{}); got != capacity { + t.Fatalf("Frame size = %d, want exact borrowed ABI capacity %d", got, capacity) } for _, align := range []uintptr{1, 2, 4, 8, 16, 64} { total, ok := FrameAllocationSize(37, align) @@ -78,14 +76,39 @@ func TestFrameAllocationLayout(t *testing.T) { } runtime.KeepAlive(memory) } + for _, align := range []uintptr{1, 2, 4, 8, 16, 64} { + total, ok := CompilerFrameAllocationSize(37, align) + if !ok { + t.Fatalf("CompilerFrameAllocationSize(37, %d) rejected", align) + } + memory := make([]byte, total) + raw := unsafe.Pointer(&memory[0]) + storage, ok := compilerAlignedStorage(raw, align) + if !ok || uintptr(storage)%align != 0 || + !allocationContains(raw, total, storage, 37) { + t.Fatalf("compact compiler storage align %d = (%p, %t)", align, storage, ok) + } + receipt, ok := compilerFrameReceipt(storage) + if !ok || uintptr(unsafe.Pointer(receipt)) < uintptr(raw) || + !allocationContains(raw, total, unsafe.Pointer(receipt), unsafe.Sizeof(*receipt)) { + t.Fatalf("compact compiler receipt align %d = (%p, %t)", align, receipt, ok) + } + runtime.KeepAlive(memory) + } for _, align := range []uintptr{0, 3, 6} { if _, ok := FrameAllocationSize(1, align); ok { t.Fatalf("invalid alignment %d accepted", align) } + if _, ok := CompilerFrameAllocationSize(1, align); ok { + t.Fatalf("invalid compact compiler alignment %d accepted", align) + } } if _, ok := FrameAllocationSize(^uintptr(0), 8); ok { t.Fatal("overflowing frame allocation accepted") } + if _, ok := CompilerFrameAllocationSize(^uintptr(0), 8); ok { + t.Fatal("overflowing compact compiler frame allocation accepted") + } offset := unsafe.Sizeof(Frame{}) + unsafe.Sizeof(uintptr(0)) if _, ok := alignedStorageOffset(^uintptr(0)-offset-1, 8); ok { t.Fatal("overflowing aligned storage address accepted") @@ -114,7 +137,7 @@ func TestBorrowedFrameV2PublishAndDestroy(t *testing.T) { } frame := (*Frame)(unsafe.Pointer(metadata)) if g.frames != frame || frame.owner != g || frame.handle != handle || - frame.header != header || frame.storage != nil || frame.rawBase != nil || + frame.header != header || frame.allocationSize != 0 || !frame.borrowedStorage || frame.state != FrameInitialSuspended || header.AllocationBase != unsafe.Pointer(frame) { t.Fatalf("borrowed publication = %+v, header base=%p", frame, header.AllocationBase) @@ -152,15 +175,13 @@ func TestBorrowedFrameV3InitializesHeader(t *testing.T) { ResultSlot: unsafe.Pointer(new(byte)), SuspendReason: ^uint16(0), Lifecycle: ^uint16(0), - StateID: ^uint32(0), Line: ^uint32(0), - Flags: ^uint32(0), } metadata := new(BorrowedFrameStorageV2) for index := range metadata { metadata[index] = ^uintptr(0) } - if !PublishFrameV3( + if !PublishFrameV3Compiler( g, handle, header, nil, unsafe.Pointer(metadata), unsafe.Pointer(descriptor), resultSlot, ) { @@ -168,12 +189,11 @@ func TestBorrowedFrameV3InitializesHeader(t *testing.T) { } frame := (*Frame)(unsafe.Pointer(metadata)) if g.frames != frame || frame.owner != g || frame.handle != handle || frame.header != header || - frame.descriptor != unsafe.Pointer(descriptor) || !frame.borrowedStorage || + frame.header.Descriptor != unsafe.Pointer(descriptor) || !frame.borrowedStorage || header.G != unsafe.Pointer(g) || header.Parent != nil || header.Descriptor != unsafe.Pointer(descriptor) || header.AllocationBase != unsafe.Pointer(frame) || header.ResultSlot != resultSlot || header.SuspendReason != uint16(SuspendNone) || - header.Lifecycle != uint16(FrameInitialSuspended) || header.StateID != 0 || - header.Line != 0 || header.Flags != 0 { + header.Lifecycle != uint16(FrameInitialSuspended) || header.Line != 0 { t.Fatalf("V3 borrowed publication frame=%+v header=%+v", frame, header) } frame.state = FrameDestroyPending @@ -184,6 +204,90 @@ func TestBorrowedFrameV3InitializesHeader(t *testing.T) { } } +func TestCompilerDynamicFrameRegisterPublishAndRelease(t *testing.T) { + g := new(G) + if !InitG(g) { + t.Fatal("InitG failed") + } + const ( + size = uintptr(256) + align = uintptr(16) + ) + total, ok := CompilerFrameAllocationSize(size, align) + if !ok { + t.Fatal("compute compiler frame allocation") + } + memory := make([]byte, total) + descriptor := &FrameDescriptorV1{Version: 1, ResultAlign: 1, Function: "test.compiler.dynamic"} + storage, ok := RegisterFrameCompiler( + g, unsafe.Pointer(&memory[0]), total, size, align, unsafe.Pointer(descriptor), + ) + if !ok { + t.Fatal("register compiler frame") + } + handle := unsafe.Pointer(new(byte)) + header := (*HeaderV1)(unsafe.Add(storage, 16)) + metadata := (*BorrowedFrameStorageV2)(unsafe.Add(storage, 80)) + if !PublishFrameV3Compiler( + g, handle, header, storage, unsafe.Pointer(metadata), + unsafe.Pointer(descriptor), nil, + ) { + t.Fatal("publish compiler frame") + } + frame := FrameFromStorage(storage) + if frame == nil || g.frames != frame || frame.owner != g || frame.handle != handle || + frame.header != header || frame.header.Descriptor != unsafe.Pointer(descriptor) || + frame.allocationSize != total || frame.state != FrameInitialSuspended || + frame.borrowedStorage || header.AllocationBase != unsafe.Pointer(&memory[0]) || + frame != (*Frame)(unsafe.Pointer(metadata)) { + t.Fatalf("compiler frame publication = %+v, header=%+v", frame, header) + } + frame.state = FrameDestroyPending + header.Lifecycle = uint16(FrameDestroyPending) + g.destroyTarget = frame + releasedMetadata, raw, released, ok := ReleaseFrameCompiler( + g, storage, size, align, unsafe.Pointer(descriptor), + ) + if !ok || releasedMetadata != unsafe.Pointer(metadata) || + raw != unsafe.Pointer(&memory[0]) || released != total { + t.Fatalf( + "compiler frame release = (%p, %p, %d, %t), want (%p, %p, %d, true)", + releasedMetadata, raw, released, ok, metadata, &memory[0], total, + ) + } + if g.frames != nil || g.destroyTarget != nil || frame.state != FrameDestroyed || + header.Lifecycle != uint16(FrameDestroyed) { + t.Fatalf("compiler frame release retained state: frames=%p target=%p frame=%d header=%d", g.frames, g.destroyTarget, frame.state, header.Lifecycle) + } + // A panic keeps the embedded metadata while the physical raw allocation is + // distinct. The dead handle word must carry that base through trace discard + // so the allocator receives the exact original range. + typeWord, dataWord := unsafe.Pointer(new(byte)), unsafe.Pointer(new(byte)) + parent := &Frame{completion: CompletionRecord{ + status: CompletionPanic, child: handle, typeWord: typeWord, dataWord: dataWord, + }} + frame.parent = parent + frame.retainPanicTrace = true + if !RetainPendingPanicTraceFrameCompiler( + g, releasedMetadata, raw, released, + ) || g.panicTraceHead != frame || g.panicTraceTail != frame || + frame.state != FrameTraceRetained || + unsafe.Pointer(frame.header) != unsafe.Pointer(descriptor) || frame.handle != raw { + t.Fatalf("retain compact compiler panic frame = %+v", frame) + } + if !stagePanicTraceDiscard(g) { + t.Fatal("stage compact compiler panic trace discard") + } + discardedRaw, discardedTotal, ok := TakeDiscardedPanicTraceFrame(g) + if !ok || discardedRaw != raw || discardedTotal != released { + t.Fatalf( + "compact compiler panic discard = (%p, %d, %t), want (%p, %d, true)", + discardedRaw, discardedTotal, ok, raw, released, + ) + } + runtime.KeepAlive(memory) +} + type testFrame struct { handle unsafe.Pointer header *HeaderV1 @@ -263,11 +367,11 @@ func retainDetachedTestPanicTrace( } memory = make([]byte, unsafe.Sizeof(Frame{})) raw := unsafe.Pointer(&memory[0]) + header := &HeaderV1{Descriptor: unsafe.Pointer(descriptor)} *(*Frame)(raw) = Frame{ owner: g, - rawBase: raw, allocationSize: uintptr(len(memory)), - descriptor: unsafe.Pointer(descriptor), + header: header, state: FrameDestroyed, parent: carrier, } @@ -361,6 +465,62 @@ func TestReleaseFrameDoesNotPartiallyCommitFailedUnlink(t *testing.T) { runtime.KeepAlive(test.memory) } +func TestReleaseFrameRequiresExactDerivedLayout(t *testing.T) { + tests := []struct { + name string + mutate func(*testFrame) (uintptr, uintptr, unsafe.Pointer) + }{ + { + name: "size", + mutate: func(frame *testFrame) (uintptr, uintptr, unsafe.Pointer) { + return frame.size + 1, frame.align, frame.descriptor + }, + }, + { + name: "alignment", + mutate: func(frame *testFrame) (uintptr, uintptr, unsafe.Pointer) { + return frame.size, frame.align * 2, frame.descriptor + }, + }, + { + name: "invalid alignment", + mutate: func(frame *testFrame) (uintptr, uintptr, unsafe.Pointer) { + return frame.size, 3, frame.descriptor + }, + }, + { + name: "descriptor", + mutate: func(frame *testFrame) (uintptr, uintptr, unsafe.Pointer) { + return frame.size, frame.align, unsafe.Pointer(new(FrameDescriptorV1)) + }, + }, + } + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + g := new(G) + if !InitG(g) { + t.Fatal("InitG failed") + } + test := newTestFrame(t, g, unsafe.Pointer(new(byte)), nil) + frame := FrameFromStorage(test.storage) + frame.state = FrameDestroyPending + test.header.Lifecycle = uint16(FrameDestroyPending) + g.destroyTarget = frame + size, align, descriptor := testCase.mutate(test) + + if _, _, ok := ReleaseFrame(g, test.storage, size, align, descriptor); ok { + t.Fatal("release accepted mismatched compiler layout metadata") + } + if g.frames != frame || g.destroyTarget != frame || frame.state != FrameDestroyPending || + test.header.Lifecycle != uint16(FrameDestroyPending) { + t.Fatalf("failed release mutated ownership: frames=%p target=%p state=%d lifecycle=%d", g.frames, g.destroyTarget, frame.state, test.header.Lifecycle) + } + releaseTestFrame(t, g, test) + runtime.KeepAlive(test.memory) + }) + } +} + func TestSinglePSchedulerChildDestroyedBeforeParentResume(t *testing.T) { runSchedulerScenario(t) } diff --git a/runtime/internal/coro/inline_await.go b/runtime/internal/coro/inline_await.go index 23ca51ebcb..0ab9489e4e 100644 --- a/runtime/internal/coro/inline_await.go +++ b/runtime/internal/coro/inline_await.go @@ -528,8 +528,7 @@ func CommitInlineAwaitPhysicalDestroyCompiler( parent.header.SuspendReason == uint16(SuspendCall) && parent.header.Lifecycle == uint16(FrameSuspended) && child.header.Lifecycle == uint16(FrameDestroyPending) && - child.borrowedStorage && child.storage == nil && child.rawBase == nil && - child.allocationSize == 0 && child.parkWait == nil && + child.borrowedStorage && child.allocationSize == 0 && child.parkWait == nil && child.header.AllocationBase == unsafe.Pointer(child) && !child.retainPanicTrace && g.frames == child && parent.completion.child == childHandle && diff --git a/runtime/internal/coro/inline_await_test.go b/runtime/internal/coro/inline_await_test.go index 88cb39b7d8..0003a545dd 100644 --- a/runtime/internal/coro/inline_await_test.go +++ b/runtime/internal/coro/inline_await_test.go @@ -132,6 +132,32 @@ func TestCompilerInlineAwaitCompletesThroughTrustedSuffix(t *testing.T) { runtime.KeepAlive(fixture.parent.memory) } +func TestCompilerRootCompletionUsesTrustedSuffix(t *testing.T) { + p := new(P) + task := newYieldingTestG(t, "compiler-root-complete") + action := beginWaitTestResume(t, p, task) + frame := FrameFromStorage(task.frame.storage) + if frame == nil || task.g.root != frame || task.g.frames != frame || frame.next != nil { + t.Fatal("compiler root fixture is not an exact singleton frame chain") + } + task.frame.header.SuspendReason = uint16(SuspendFrameComplete) + task.frame.header.Lifecycle = uint16(FrameFinalSuspended) + if !PrepareCompleteStatusCompiler( + task.g, task.handle, task.frame.header, CompletionReturn, + ) { + t.Fatal("publish compiler root return") + } + if task.g.pending.kind != pendingComplete || task.g.pending.from != frame || + task.g.pending.target != nil { + t.Fatalf("compiler root pending transition = %+v", task.g.pending) + } + next, ok := Resumed(p, task.g, action) + if !ok || next.Kind != ActionCheckDestroy || next.Handle != task.handle { + t.Fatalf("commit compiler root return = (%+v, %t)", next, ok) + } + runtime.KeepAlive(task.frame.memory) +} + func TestCompilerInlineAwaitFusesBorrowedDestroyAndConsume(t *testing.T) { fixture := newInlineAwaitFixtureForCompiler(t, true) child := FrameFromStorage(fixture.child.storage) @@ -139,8 +165,6 @@ func TestCompilerInlineAwaitFusesBorrowedDestroyAndConsume(t *testing.T) { t.Fatal("resolve borrowed compiler child") } child.borrowedStorage = true - child.storage = nil - child.rawBase = nil child.allocationSize = 0 fixture.child.header.AllocationBase = unsafe.Pointer(child) fixture.child.header.SuspendReason = uint16(SuspendFrameComplete) diff --git a/runtime/internal/coro/run_decision_abi.go b/runtime/internal/coro/run_decision_abi.go index 15d7ad1889..856f7fdf25 100644 --- a/runtime/internal/coro/run_decision_abi.go +++ b/runtime/internal/coro/run_decision_abi.go @@ -52,8 +52,9 @@ func TakeRunDecisionWords( // TakeRunDecisionWordsCompiler is the compiler-owned scalar gate. A nested // static child's initial zero-ticket resume directly consumes the adjacent -// pendingInlineStart certificate; ordinary resumes and every non-zero ticket -// retain TakeRunDecisionWords' complete decision validation. +// pendingInlineStart certificate. A scheduler-issued all-zero decision uses +// the adjacent P/G/action receipt; non-zero decisions and every non-zero +// ticket retain TakeRunDecisionWords' complete decision validation. func TakeRunDecisionWordsCompiler( g *G, expectedEpoch, expectedGeneration uint32, @@ -65,5 +66,31 @@ func TakeRunDecisionWordsCompiler( takeInlineAwaitInitialDecisionCompiler(g) { return 0, 0, 0, 0, 0, true } + if expectedEpoch == 0 && expectedGeneration == 0 && + takeOrdinaryZeroRunDecisionCompiler(g) { + return 0, 0, 0, 0, 0, true + } return TakeRunDecisionWords(g, expectedEpoch, expectedGeneration) } + +// takeOrdinaryZeroRunDecisionCompiler consumes the overwhelmingly common +// scheduler-issued decision whose complete value is zero. checkedExecutorRun +// created this private P/G/action episode immediately before llvm.coro.resume; +// a zero RunDecision is already a valid union value, so replaying the generic +// union validator and expectedAction adapter does not prove anything new. +// Park, cancellation, nested-inline, stale, and malformed shapes fall through +// to the complete TakeRunDecision path. +func takeOrdinaryZeroRunDecisionCompiler(g *G) bool { + if !ValidG(g) || g.runP == nil { + return false + } + p := g.runP + if p.current != g || !p.inResume || g.state != GRunning || + p.runDecisionTaken || p.runDecision != (RunDecision{}) || + p.action.Kind != ActionResume || p.action.Flags != 0 || p.action.Handle == nil || + !gPreemptEnabledAtDepthZero(g) { + return false + } + p.runDecisionTaken = true + return true +} diff --git a/runtime/internal/coro/run_decision_abi_test.go b/runtime/internal/coro/run_decision_abi_test.go index b905a2e057..f2c421f9ed 100644 --- a/runtime/internal/coro/run_decision_abi_test.go +++ b/runtime/internal/coro/run_decision_abi_test.go @@ -74,6 +74,37 @@ func TestTakeRunDecisionWordsAcceptsZeroTicketNormalResume(t *testing.T) { finishWaitTestTask(t, p, task, action) } +func TestTakeRunDecisionWordsCompilerConsumesAdjacentZeroDecision(t *testing.T) { + p := new(P) + task := newYieldingTestG(t, "compiler-run-decision-words-normal") + action := beginWaitTestResumeWithoutGate(t, p, task) + outcome, caseID, taskKind, sourceSlot, generation, ok := TakeRunDecisionWordsCompiler(task.g, 0, 0) + if !ok || outcome != 0 || caseID != 0 || taskKind != 0 || sourceSlot != 0 || generation != 0 || + !p.runDecisionTaken || p.runDecision != (RunDecision{}) { + t.Fatalf("compiler zero decision = (%d,%d,%d,%d,%d,%t), retained=%+v taken=%t", + outcome, caseID, taskKind, sourceSlot, generation, ok, p.runDecision, p.runDecisionTaken) + } + if outcome, caseID, taskKind, sourceSlot, generation, ok = TakeRunDecisionWordsCompiler(task.g, 0, 0); ok || + outcome != 0 || caseID != 0 || taskKind != 0 || sourceSlot != 0 || generation != 0 { + t.Fatalf("compiler zero decision replay = (%d,%d,%d,%d,%d,%t)", + outcome, caseID, taskKind, sourceSlot, generation, ok) + } + finishWaitTestTask(t, p, task, action) +} + +func TestTakeRunDecisionWordsCompilerRejectsMalformedActionWithoutMutation(t *testing.T) { + fixture := newUncheckedResumeGateFixture(t, "compiler-run-decision-malformed") + fixture.p.action.Handle = nil + if outcome, caseID, taskKind, sourceSlot, generation, ok := TakeRunDecisionWordsCompiler( + fixture.task.g, 0, 0, + ); ok || outcome != 0 || caseID != 0 || taskKind != 0 || sourceSlot != 0 || generation != 0 || + fixture.p.runDecisionTaken || fixture.p.runDecision != (RunDecision{}) { + t.Fatalf("malformed compiler decision = (%d,%d,%d,%d,%d,%t), retained=%+v taken=%t", + outcome, caseID, taskKind, sourceSlot, generation, ok, + fixture.p.runDecision, fixture.p.runDecisionTaken) + } +} + func TestTakeRunDecisionWordsPreservesExactTicketAndScalarizesLease(t *testing.T) { p := new(P) task := newYieldingTestG(t, "run-decision-words") diff --git a/runtime/internal/coro/run_slice.go b/runtime/internal/coro/run_slice.go index e39cfbb85b..5122db10e2 100644 --- a/runtime/internal/coro/run_slice.go +++ b/runtime/internal/coro/run_slice.go @@ -922,6 +922,57 @@ func ResumedExecutorRun( return next, true, true } +// CanBeginIssuedExecutorDestroyAfterResume reports whether one still-private +// issued resume may enter its adjacent normal-completion destroy interval. +// The optimization cannot cross an already-runnable peer, panic/cancellation, +// foreign reentry, producer callback, or host boundary. Keeping the issued +// capability live avoids a ready-tail round trip for an otherwise isolated +// short-lived coroutine, while llvm.coro.done remains an independent guard. +// +// Panic unwinding and foreign reentry deliberately retain their existing +// separately scheduled destroy paths because those transitions carry an +// externally visible control boundary in addition to frame completion. +func CanBeginIssuedExecutorDestroyAfterResume( + driver *ExecutorDriver, + g *G, + action Action, +) bool { + if !validIssuedExecutorRunAction(driver) || + driver.run.issued != ActionCheckResume || g == nil { + return false + } + p := driver.p + if p.current != g || g.runP != p || p.inResume || p.inlineAwaitDepth != 0 || + p.action != action || action.Kind != ActionCheckDestroy || action.Flags != 0 || + action.Handle == nil || p.runDecision != (RunDecision{}) || p.runDecisionTaken || + g.state != GDispatching || g.destroyTarget == nil || + g.destroyTarget.handle != action.Handle || + g.destroyTarget.state != FrameDestroyPending || + g.park.taskCancelPhase == taskCancelRequested || g.panicUnwind || + p.foreignReentry != nil || driver.run.readyDebt || runnableForOSThreadOwner(p) { + return false + } + return true +} + +// BeginIssuedExecutorDestroyAfterResume consumes the checked private interval +// after the compiler-owned llvm.coro.done observation succeeds. +func BeginIssuedExecutorDestroyAfterResume( + driver *ExecutorDriver, + g *G, + action Action, + done bool, +) (Action, bool) { + if !done || !CanBeginIssuedExecutorDestroyAfterResume(driver, g, action) { + return Action{}, false + } + p := driver.p + next := Action{Kind: ActionDestroy, Handle: action.Handle} + driver.run.issued = ActionCheckDestroy + p.action = next + return next, true +} + // CommitExecutorRunAction closes the no-return physical interval opened by an // Action step. A live continuation is moved to the ready tail; terminal and // yield/park control actions are already stable. The function retains neither diff --git a/runtime/internal/coro/run_slice_test.go b/runtime/internal/coro/run_slice_test.go index ba3f41fde0..3ff9f3c44c 100644 --- a/runtime/internal/coro/run_slice_test.go +++ b/runtime/internal/coro/run_slice_test.go @@ -64,6 +64,106 @@ func runnerNextPhysicalAction(t *testing.T, driver *ExecutorDriver, task *yieldi return step } +func TestExecutorRunFusesPrivateNormalCompletionDestroy(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "fused-completion-destroy") + if !Enqueue(p, task.g) { + t.Fatal("enqueue fused completion task") + } + _ = runnerNextPhysicalAction(t, driver, task, ActionCheckResume) + resume, _, ok := BeginIssuedExecutorResumeRuntimeContext(driver, task.g) + if !ok || resume.Kind != ActionResume || resume.Handle != task.handle { + t.Fatalf("begin fused completion resume = (%+v, %t)", resume, ok) + } + takeNormalRunnerDecision(t, task.g) + task.frame.header.SuspendReason = uint16(SuspendFrameComplete) + task.frame.header.Lifecycle = uint16(FrameFinalSuspended) + if !PrepareComplete(task.g, task.handle, task.frame.header) { + t.Fatal("prepare fused completion") + } + checkDestroy, committed, ok := ResumedExecutorRun(driver, p, task.g, resume) + if !ok || committed || checkDestroy.Kind != ActionCheckDestroy || + checkDestroy.Handle != task.handle || driver.run.issued != ActionCheckResume { + t.Fatalf( + "private completion check = (%+v, committed=%t, ok=%t, issued=%d)", + checkDestroy, committed, ok, driver.run.issued, + ) + } + beforeAction, beforeIssued := p.action, driver.run.issued + if destroy, fused := BeginIssuedExecutorDestroyAfterResume( + driver, task.g, checkDestroy, false, + ); fused || destroy != (Action{}) || p.action != beforeAction || driver.run.issued != beforeIssued { + t.Fatalf( + "unfinished frame changed private interval = (%+v, fused=%t, action=%+v, issued=%d)", + destroy, fused, p.action, driver.run.issued, + ) + } + destroy, fused := BeginIssuedExecutorDestroyAfterResume( + driver, task.g, checkDestroy, true, + ) + if !fused || destroy.Kind != ActionDestroy || destroy.Handle != task.handle || + p.action != destroy || driver.run.issued != ActionCheckDestroy { + t.Fatalf( + "fused completion destroy = (%+v, fused=%t, action=%+v, issued=%d)", + destroy, fused, p.action, driver.run.issued, + ) + } + releaseTestFrame(t, task.g, task.frame) + receipt, ok := DestroyedBounded(p, task.g, destroy) + if !ok || receipt.Kind != ActionCommitDestroy || receipt.Handle != nil || + !CommitExecutorRunAction(driver, task.g, receipt) { + t.Fatalf("commit fused completion destroy = (%+v, %t)", receipt, ok) + } + commit, ok := NextExecutorRunStep(driver) + if !ok || commit.Kind != ExecutorRunStepDestroyCommit || commit.Action != receipt { + t.Fatalf("select fused completion commit = (%+v, %t)", commit, ok) + } + completed, ok := CommitExecutorRunDomainDestroy(driver, task.g, receipt) + if !ok || completed.Kind != ActionComplete { + t.Fatalf("finish fused completion domain = (%+v, %t)", completed, ok) + } + if task.g.root != nil || task.g.destroyTarget != nil || task.g.state != GDead { + t.Fatalf( + "fused completion retained task state: root=%p target=%p state=%d", + task.g.root, task.g.destroyTarget, task.g.state, + ) + } +} + +func TestExecutorRunDoesNotFuseCompletionPastReadyPeer(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "completion-before-ready-peer") + peer := newYieldingTestG(t, "ready-peer-before-destroy") + if !Enqueue(p, task.g) || !Enqueue(p, peer.g) { + t.Fatal("enqueue completion task and ready peer") + } + _ = runnerNextPhysicalAction(t, driver, task, ActionCheckResume) + resume, _, ok := BeginIssuedExecutorResumeRuntimeContext(driver, task.g) + if !ok { + t.Fatal("begin completion resume") + } + takeNormalRunnerDecision(t, task.g) + task.frame.header.SuspendReason = uint16(SuspendFrameComplete) + task.frame.header.Lifecycle = uint16(FrameFinalSuspended) + if !PrepareComplete(task.g, task.handle, task.frame.header) { + t.Fatal("prepare completion before ready peer") + } + checkDestroy, committed, ok := ResumedExecutorRun(driver, p, task.g, resume) + if !ok || committed || checkDestroy.Kind != ActionCheckDestroy { + t.Fatalf("private completion check = (%+v, committed=%t, ok=%t)", checkDestroy, committed, ok) + } + beforeAction, beforeIssued := p.action, driver.run.issued + if CanBeginIssuedExecutorDestroyAfterResume(driver, task.g, checkDestroy) { + t.Fatal("completion destroy fusion bypassed an already-ready peer") + } + if destroy, fused := BeginIssuedExecutorDestroyAfterResume(driver, task.g, checkDestroy, true); fused || + destroy != (Action{}) || p.action != beforeAction || driver.run.issued != beforeIssued { + t.Fatalf("rejected fusion changed private interval = (%+v, fused=%t)", destroy, fused) + } +} + func TestExecutorRunResumeRuntimeContextDescriptorCapability(t *testing.T) { for _, test := range []struct { name string diff --git a/runtime/internal/coro/runnable_transfer.go b/runtime/internal/coro/runnable_transfer.go index 1db9322784..f8be31fca8 100644 --- a/runtime/internal/coro/runnable_transfer.go +++ b/runtime/internal/coro/runnable_transfer.go @@ -227,7 +227,7 @@ func pNeutralRunnableHeader(g *G, queued bool) bool { if g.root == nil || active == nil || g.frames != active || g.root.owner != g || g.root.parent != nil || g.root.header == nil || g.root.header.Parent != nil || active.owner != g || active.handle == nil || active.header == nil || - active.header.G != unsafe.Pointer(g) || active.header.Flags != 0 || + active.header.G != unsafe.Pointer(g) || active.parkWait != nil || !emptyCompletionRecord(&active.completion) || active.state != FrameState(active.header.Lifecycle) { return false @@ -258,7 +258,7 @@ func pNeutralFrameChain(g *G) bool { for frame := g.active; frame != nil; frame = frame.parent { if frame.owner != g || frame.handle == nil || frame.header == nil || frame.next != frame.parent || - frame.header.G != unsafe.Pointer(g) || frame.header.Flags != 0 || + frame.header.G != unsafe.Pointer(g) || frame.parkWait != nil || !emptyCompletionRecord(&frame.completion) || frame.state != FrameState(frame.header.Lifecycle) { return false diff --git a/runtime/internal/coro/scheduler.go b/runtime/internal/coro/scheduler.go index e5eac8814b..99e981ce31 100644 --- a/runtime/internal/coro/scheduler.go +++ b/runtime/internal/coro/scheduler.go @@ -1361,8 +1361,12 @@ func checkedExecutorRun( if withRuntimeContextMode { mode := g.active.runtimeContext if mode == frameRuntimeContextUnknown { + var descriptor unsafe.Pointer + if g.active.header != nil { + descriptor = g.active.header.Descriptor + } var valid bool - mode, valid = descriptorRuntimeContextMode(g.active.descriptor) + mode, valid = descriptorRuntimeContextMode(descriptor) if !valid { return Action{}, false, false } @@ -1444,8 +1448,12 @@ func BeginIssuedExecutorResumeRuntimeContext( } mode := g.active.runtimeContext if mode == frameRuntimeContextUnknown { + var descriptor unsafe.Pointer + if g.active.header != nil { + descriptor = g.active.header.Descriptor + } var valid bool - mode, valid = descriptorRuntimeContextMode(g.active.descriptor) + mode, valid = descriptorRuntimeContextMode(descriptor) if !valid { return Action{}, false, false } diff --git a/runtime/internal/coro/scheduler_spawn_test.go b/runtime/internal/coro/scheduler_spawn_test.go index 517ddbccaf..e70e227bc0 100644 --- a/runtime/internal/coro/scheduler_spawn_test.go +++ b/runtime/internal/coro/scheduler_spawn_test.go @@ -95,6 +95,55 @@ func beginSpawnTestChildResume(t *testing.T, p *P, g *G, frame *testFrame) Actio return action } +func TestCompilerSpawnUsesAdjacentTransactionCertificates(t *testing.T) { + p := new(P) + parent := newYieldingTestG(t, "compiler-spawn-parent") + _ = beginSpawnTestResume(t, p, parent) + child := new(G) + if !BeginSpawnCompiler(parent.g, child, unsafe.Pointer(child), TaskStorageSize()) { + t.Fatal("begin compiler spawn") + } + if parent.g.spawnChild != child || child.spawnParent != parent.g || child.spawnP != p || + child.taskStorage != unsafe.Pointer(child) || child.taskSize != TaskStorageSize() || + child.taskState != taskStorageOwned || !gPreemptStateAtDepthZero(child, preemptIdle) { + t.Fatalf("compiler spawn begin state: parent-child=%p child-parent=%p child-p=%p storage=%p size=%d state=%d", + parent.g.spawnChild, child.spawnParent, child.spawnP, child.taskStorage, child.taskSize, child.taskState) + } + local := unsafe.Pointer(new(byte)) + if !BindTaskLocalCompiler(child, local) || TaskLocal(child) != local || + BindTaskLocalCompiler(child, unsafe.Pointer(new(byte))) { + t.Fatal("compiler spawn task-local binding did not publish exactly once") + } + handle := unsafe.Pointer(new(byte)) + root, _ := newSpawnTestFrame(t, child, handle, 0, 1) + if !CommitSpawnCompiler(parent.g, child, handle) { + t.Fatal("commit compiler spawn") + } + metadata := FrameFromStorage(root.storage) + if child.root != metadata || child.active != metadata || child.frames != metadata || + child.state != GRunnable || !child.queued || p.readyHead != child || p.readyTail != child || + p.readyCount != 1 || parent.g.spawnChild != nil || child.spawnParent != nil || child.spawnP != nil { + t.Fatalf("compiler spawn commit state: child=%+v p=(%p,%p,%d) parent-child=%p", + child, p.readyHead, p.readyTail, p.readyCount, parent.g.spawnChild) + } + runtime.KeepAlive(parent.frame.memory) + runtime.KeepAlive(root.memory) +} + +func TestCompilerSpawnRejectsUntakenResumeGateWithoutMutation(t *testing.T) { + fixture := newUncheckedResumeGateFixture(t, "compiler-spawn-gate") + child := new(G) + if BeginSpawnCompiler(fixture.task.g, child, unsafe.Pointer(child), TaskStorageSize()) { + t.Fatal("compiler spawn accepted an untaken resume gate") + } + if fixture.task.g.spawnChild != nil || child.magic != 0 || child.taskStorage != nil || + child.spawnParent != nil || child.spawnP != nil || + !gPreemptStateAtDepthZero(child, preemptDisabled) { + t.Fatal("rejected compiler spawn mutated transaction state") + } + assertResumeGateStillUnchecked(t, fixture) +} + func yieldSpawnTestG(t *testing.T, p *P, g *G, frame *testFrame, action Action) { t.Helper() // A sole newly spawned child now stays local without forcing the parent to @@ -354,14 +403,19 @@ func TestCompletedTaskTransfersContextAndStorageAfterOneTerminalAudit(t *testing t.Fatal("disable completed task transfer preemption") } g.state = GDead + g.taskControlLeases = 1 + if _, _, _, _, ok := ReleaseCompletedTaskCompiler(g); ok { + t.Fatal("completed task with a live control lease transferred") + } + g.taskControlLeases = 0 - releasedLocal, raw, size, owned, ok := ReleaseCompletedTask(g) + releasedLocal, raw, size, owned, ok := ReleaseCompletedTaskCompiler(g) if !ok || !owned || releasedLocal != local || raw != unsafe.Pointer(g) || size != TaskStorageSize() || g.taskLocal != nil || g.taskStorage != nil || g.taskSize != 0 || g.taskState != taskStorageReleased { t.Fatalf("completed task transfer = local:%p raw:%p size:%d owned:%t ok:%t state:%d", releasedLocal, raw, size, owned, ok, g.taskState) } - if _, _, _, _, ok := ReleaseCompletedTask(g); ok { + if _, _, _, _, ok := ReleaseCompletedTaskCompiler(g); ok { t.Fatal("completed task transferred twice") } } diff --git a/runtime/internal/coro/scheduler_test_helpers_test.go b/runtime/internal/coro/scheduler_test_helpers_test.go index 777ba81692..da56ad19c6 100644 --- a/runtime/internal/coro/scheduler_test_helpers_test.go +++ b/runtime/internal/coro/scheduler_test_helpers_test.go @@ -47,16 +47,16 @@ func pollCompilerSafepointForTest(t *testing.T, g *G) bool { if !ValidG(g) || g.active == nil || g.active.header == nil { t.Fatal("compiler preemption checkpoint has no active frame") } - remaining := g.active.header.StateID + remaining := g.active.panicLine if remaining == 0 || remaining > preemptCheckpointStride { remaining = preemptCheckpointStride } remaining-- - g.active.header.StateID = remaining + g.active.panicLine = remaining if remaining != 0 { return false } - g.active.header.StateID = preemptCheckpointStride + g.active.panicLine = preemptCheckpointStride return PollPreemptCompiler(g) } diff --git a/runtime/internal/coro/shutdown.go b/runtime/internal/coro/shutdown.go index 8d86830448..0c74bd062d 100644 --- a/runtime/internal/coro/shutdown.go +++ b/runtime/internal/coro/shutdown.go @@ -28,16 +28,11 @@ func CommandMainReturnPoint(p *P, main *G) bool { func validCancelFrame(frame *Frame, g *G) bool { if frame == nil || frame.owner != g || frame.handle == nil || frame.header == nil || - frame.descriptor == nil || frame.header.G != unsafe.Pointer(g) || - frame.header.Descriptor != frame.descriptor { + frame.header.Descriptor == nil || frame.header.G != unsafe.Pointer(g) || + !validFrameAllocationIdentity(frame) { return false } - if frame.borrowedStorage { - return frame.storage == nil && frame.rawBase == nil && frame.allocationSize == 0 && - frame.header.AllocationBase == unsafe.Pointer(frame) - } - return frame.storage != nil && frame.rawBase != nil && - frame.header.AllocationBase == frame.rawBase + return true } // validCancelableReadyG proves that a ready G contains exactly one structured diff --git a/runtime/internal/coro/spawn.go b/runtime/internal/coro/spawn.go index 9452099b8a..c4d0fe03be 100644 --- a/runtime/internal/coro/spawn.go +++ b/runtime/internal/coro/spawn.go @@ -51,6 +51,21 @@ func BindTaskLocal(g *G, local unsafe.Pointer) bool { return true } +// BindTaskLocalCompiler attaches the runtime sidecar inside an active compiler +// spawn transaction. BeginSpawnCompiler has already proved the zero-filled +// owned allocation and the child is not scheduler-visible until CommitSpawn; +// the reciprocal parent/P links are therefore the exact publication guard. +func BindTaskLocalCompiler(g *G, local unsafe.Pointer) bool { + if !ValidG(g) || local == nil || g.taskLocal != nil || g.state != GNew || + g.spawnParent == nil || g.spawnP == nil || g.spawnParent.spawnChild != g || + g.taskState != taskStorageOwned || g.taskStorage != unsafe.Pointer(g) || + g.taskSize != TaskStorageSize() || g.root != nil || g.active != nil || g.frames != nil { + return false + } + g.taskLocal = local + return true +} + // TaskLocal returns the runtime-adapter context attached to g. func TaskLocal(g *G) unsafe.Pointer { if !ValidG(g) { @@ -167,6 +182,45 @@ func BeginSpawn(parent, child *G, storage unsafe.Pointer, size uintptr) bool { return true } +// BeginSpawnCompiler consumes the private generated go-statement boundary. +// The child range comes directly from the runtime's zero-filled task allocator +// and the parent is inside the exact compiler resume episode. Those two +// capabilities make InitG's arbitrary-memory audit and runningSpawnContext's +// queue/park compatibility audit redundant on this adjacent path. The public +// BeginSpawn contract remains the full validator for every other caller. +func BeginSpawnCompiler(parent, child *G, storage unsafe.Pointer, size uintptr) bool { + if parent == nil || child == nil || child == parent || storage != unsafe.Pointer(child) || + size != TaskStorageSize() || uintptr(storage)%unsafe.Alignof(G{}) != 0 || + parent.spawnChild != nil || !resumeGateTaken(parent) { + return false + } + p := parent.runP + active := parent.active + if p == nil || active == nil || active.owner != parent || + active.header == nil || active.state != FrameActive || + active.header.G != unsafe.Pointer(parent) || + active.header.SuspendReason != uint16(SuspendNone) || + active.header.Lifecycle != uint16(FrameActive) || + child.magic != 0 || !gPreemptStateAtDepthZero(child, preemptDisabled) || + child.state != GNew || child.taskState != taskStorageStatic || + child.taskStorage != nil || child.taskLocal != nil { + return false + } + child.magic = gMagic + child.taskStorage = storage + child.taskSize = size + child.taskState = taskStorageOwned + child.spawnParent = parent + child.spawnP = p + // Publish the asynchronous preemption gate only after the owned allocation + // and reciprocal transaction fields are complete. + if !compareAndSwapGPreemptStateAtDepthZero(child, preemptDisabled, preemptIdle) { + return false + } + parent.spawnChild = child + return true +} + // activeSpawnTransaction consumes the reciprocal parent/child/P links // published by BeginSpawn as an unforgeable scheduler-owned certificate. The // compiler's root factory cannot suspend or run another scheduler reduction, @@ -189,14 +243,14 @@ func validDiscardResultSpawnRoot(child *G, handle unsafe.Pointer) (*Frame, bool) if root == nil || child.frames != root || root.next != nil || root.owner != child || root.parent != nil || root.handle != handle || root.header == nil || root.header.G != unsafe.Pointer(child) || root.header.Parent != nil || - root.header.Descriptor != root.descriptor || root.header.ResultSlot != nil || + root.header.Descriptor == nil || root.header.ResultSlot != nil || root.header.SuspendReason != uint16(SuspendNone) || root.header.Lifecycle != uint16(FrameInitialSuspended) || - root.state != FrameInitialSuspended || root.descriptor == nil || - !checkedProgramObjectV1(root.descriptor, unsafe.Sizeof(FrameDescriptorV1{}), unsafe.Alignof(FrameDescriptorV1{})) { + root.state != FrameInitialSuspended || + !checkedProgramObjectV1(root.header.Descriptor, unsafe.Sizeof(FrameDescriptorV1{}), unsafe.Alignof(FrameDescriptorV1{})) { return nil, false } - descriptor := (*FrameDescriptorV1)(root.descriptor) + descriptor := (*FrameDescriptorV1)(root.header.Descriptor) if descriptor.Version != 1 || descriptor.Flags&^frameDescriptorAllowedFlagsV1 != 0 || !validProgramPayloadLayoutV1(descriptor.ResultSize, descriptor.ResultAlign) { return nil, false @@ -249,6 +303,56 @@ func CommitSpawn(parent, child *G, handle unsafe.Pointer) bool { return true } +// CommitSpawnCompiler consumes the reciprocal begin receipt immediately after +// the generated root ramp published its initial-suspended frame. The new root +// is necessarily the frame-list head and PublishFrameV3Compiler has already +// validated its descriptor capability, so an O(n) handle lookup and another +// program-object range proof are unnecessary here. +func CommitSpawnCompiler(parent, child *G, handle unsafe.Pointer) bool { + if parent == nil || child == nil || child == parent || handle == nil || + parent.spawnChild != child || child.spawnParent != parent || + child.spawnP == nil || parent.runP != child.spawnP { + return false + } + p := child.spawnP + // BeginSpawnCompiler authenticated the complete resume gate before + // publishing these reciprocal links. The generated root ramp reaches only + // initial suspend and cannot run a scheduler reduction, so the compact live + // episode check below consumes that receipt without replaying the full gate. + if p.current != parent || !p.inResume || parent.state != GRunning || + !p.runDecisionTaken || p.runDecision != (RunDecision{}) || + p.action.Kind != ActionResume || p.action.Flags != 0 || p.action.Handle == nil { + return false + } + root := child.frames + if child.magic != gMagic || child.state != GNew || child.root != nil || child.active != nil || + child.spawnChild != nil || child.nextReady != nil || child.queued || child.waiting || + child.runP != nil || child.transferState != runnableTransferGIdle || + child.taskState != taskStorageOwned || child.taskStorage != unsafe.Pointer(child) || + child.taskSize != TaskStorageSize() || !gPreemptStateAtDepthZero(child, preemptIdle) || + root == nil || root.next != nil || root.owner != child || root.parent != nil || + root.handle != handle || root.header == nil || root.header.G != unsafe.Pointer(child) || + root.header.Parent != nil || root.header.Descriptor == nil || + root.header.ResultSlot != nil || root.header.SuspendReason != uint16(SuspendNone) || + root.header.Lifecycle != uint16(FrameInitialSuspended) || + root.state != FrameInitialSuspended || + !validReadyQueueHeader(p) || p.readyCount == ^uint32(0) { + return false + } + if p.readyCount != 0 && !RequestPreempt(parent) { + return false + } + + child.root = root + child.active = root + child.state = GRunnable + child.spawnParent = nil + child.spawnP = nil + appendReadyUnchecked(p, child) + parent.spawnChild = nil + return true +} + // RollbackSpawn releases a begin transaction only before any coroutine frame // has been allocated. Once a factory has published a handle, rejection is // fail-stop: only the scheduler may destroy that handle, so the exported ABI @@ -367,6 +471,34 @@ func ReleaseCompletedTask(g *G) (local, raw unsafe.Pointer, size uintptr, owned, return local, raw, size, owned, true } +// ReleaseCompletedTaskCompiler consumes the ActionComplete receipt returned +// immediately after a generated root completed and its physical frame was +// destroyed. The normal spawned task has never parked or been canceled, so a +// zero ParkState plus the terminal ownership headers certifies that no source +// can retain it. Tasks with any exceptional residue fall back to the complete +// reclaimability audit. +func ReleaseCompletedTaskCompiler(g *G) (local, raw unsafe.Pointer, size uintptr, owned, ok bool) { + if g != nil && g.magic == gMagic && gPreemptStateAtDepthZero(g, preemptDisabled) && + g.state == GDead && g.taskControlLeases == 0 && g.runAction == ActionInvalid && + g.transferState == runnableTransferGIdle && g.osThreadLockDepth == 0 && + g.root == nil && g.active == nil && g.frames == nil && g.pending == (pendingTransition{}) && + g.destroyTarget == nil && !g.destroyRoot && g.nextReady == nil && !g.queued && !g.waiting && + g.runP == nil && g.park == (ParkState{}) && + g.spawnChild == nil && g.spawnParent == nil && g.spawnP == nil && + emptyPanicRecord(&g.panicRecord) && !g.panicUnwind && + g.panicTraceHead == nil && g.panicTraceTail == nil && g.panicTraceCount == 0 && + g.taskLocal != nil && g.taskState == taskStorageOwned && + g.taskStorage == unsafe.Pointer(g) && g.taskSize == TaskStorageSize() { + local, raw, size = g.taskLocal, g.taskStorage, g.taskSize + g.taskLocal = nil + g.taskStorage = nil + g.taskSize = 0 + g.taskState = taskStorageReleased + return local, raw, size, true, true + } + return ReleaseCompletedTask(g) +} + // DeadG is a narrow program-driver query. It does not imply that a command // main may safely return: TerminalG must still prove that no ready or parked G // survives. diff --git a/runtime/internal/coroalloc/_cache/cache.c b/runtime/internal/coroalloc/_cache/cache.c new file mode 100644 index 0000000000..45ef00c1a8 --- /dev/null +++ b/runtime/internal/coroalloc/_cache/cache.c @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +enum { + llgo_coro_alloc_cache_small_min = 256, + llgo_coro_alloc_cache_small_max = 1024, + llgo_coro_alloc_cache_small_quantum = 32, + llgo_coro_alloc_cache_small_count = + (llgo_coro_alloc_cache_small_max - llgo_coro_alloc_cache_small_min) / + llgo_coro_alloc_cache_small_quantum + + 1, + llgo_coro_alloc_cache_large_min = 1152, + llgo_coro_alloc_cache_large_max = 4096, + llgo_coro_alloc_cache_large_quantum = 128, + llgo_coro_alloc_cache_large_count = + (llgo_coro_alloc_cache_large_max - llgo_coro_alloc_cache_large_min) / + llgo_coro_alloc_cache_large_quantum + + 1, + llgo_coro_alloc_cache_power_min_shift = 13, + llgo_coro_alloc_cache_power_max_shift = 16, + llgo_coro_alloc_cache_power_count = + llgo_coro_alloc_cache_power_max_shift - + llgo_coro_alloc_cache_power_min_shift + 1, + llgo_coro_alloc_cache_bin_count = + llgo_coro_alloc_cache_small_count + + llgo_coro_alloc_cache_large_count + + llgo_coro_alloc_cache_power_count, + /* Small frames dominate high-rate spawn/retire loops, so preserve their + prior reuse depth. The larger compact classes get a small budget: their + main benefit is low live slack, not retaining a deep free list. Across + all 53 bins the theoretical retention bound is below 3.6 MiB. */ + llgo_coro_alloc_cache_small_bytes_per_bin = 128 * 1024, + llgo_coro_alloc_cache_large_bytes_per_bin = 16 * 1024, +}; + +struct llgo_coro_alloc_cache_bin_v1 { + _Atomic(uint32_t) lock; + void *head; + uint32_t count; + /* Zero is uninitialized; UINT32_MAX marks a class whose budget cannot + retain even one object. On pointer-64 this occupies existing padding. */ + uint32_t capacity; +}; + +static struct llgo_coro_alloc_cache_bin_v1 + llgo_coro_alloc_cache_bins_v1[llgo_coro_alloc_cache_bin_count]; + +static int llgo_coro_alloc_cache_index_v1(uintptr_t size) { + if (size >= llgo_coro_alloc_cache_small_min && + size <= llgo_coro_alloc_cache_small_max && + (size & (llgo_coro_alloc_cache_small_quantum - 1)) == 0) { + return (int)((size - llgo_coro_alloc_cache_small_min) / + llgo_coro_alloc_cache_small_quantum); + } + if (size >= llgo_coro_alloc_cache_large_min && + size <= llgo_coro_alloc_cache_large_max && + (size & (llgo_coro_alloc_cache_large_quantum - 1)) == 0) { + return llgo_coro_alloc_cache_small_count + + (int)((size - llgo_coro_alloc_cache_large_min) / + llgo_coro_alloc_cache_large_quantum); + } + uintptr_t value = + (uintptr_t)1 << llgo_coro_alloc_cache_power_min_shift; + for (int index = llgo_coro_alloc_cache_small_count + + llgo_coro_alloc_cache_large_count; + index < llgo_coro_alloc_cache_bin_count; + ++index, value <<= 1) { + if (size == value) { + return index; + } + } + return -1; +} + +static void llgo_coro_alloc_cache_lock_v1( + struct llgo_coro_alloc_cache_bin_v1 *bin) { + while (atomic_exchange_explicit(&bin->lock, 1, memory_order_acquire) != 0) { + atomic_signal_fence(memory_order_seq_cst); + } +} + +static void llgo_coro_alloc_cache_unlock_v1( + struct llgo_coro_alloc_cache_bin_v1 *bin) { + atomic_store_explicit(&bin->lock, 0, memory_order_release); +} + +void *__llgo_coro_alloc_cache_take_v1(uintptr_t size) { + int index = llgo_coro_alloc_cache_index_v1(size); + if (index < 0) { + return NULL; + } + struct llgo_coro_alloc_cache_bin_v1 *bin = + &llgo_coro_alloc_cache_bins_v1[index]; + llgo_coro_alloc_cache_lock_v1(bin); + void *result = bin->head; + if (result != NULL) { + bin->head = *(void **)result; + --bin->count; + } + llgo_coro_alloc_cache_unlock_v1(bin); + if (result != NULL) { + *(void **)result = NULL; + } + return result; +} + +bool __llgo_coro_alloc_cache_put_v1(void *pointer, uintptr_t size) { + int index = llgo_coro_alloc_cache_index_v1(size); + if (index < 0 || pointer == NULL) { + return false; + } + + /* Break every conservative GC edge before the process cache roots this + allocation. The first word is then reused only as the private free link. */ + memset(pointer, 0, size); + struct llgo_coro_alloc_cache_bin_v1 *bin = + &llgo_coro_alloc_cache_bins_v1[index]; + llgo_coro_alloc_cache_lock_v1(bin); + uint32_t capacity = bin->capacity; + if (capacity == 0) { + uintptr_t budget = size <= llgo_coro_alloc_cache_small_max + ? llgo_coro_alloc_cache_small_bytes_per_bin + : llgo_coro_alloc_cache_large_bytes_per_bin; + uintptr_t computed = budget / size; + capacity = computed == 0 ? UINT32_MAX : (uint32_t)computed; + bin->capacity = capacity; + } + if (capacity == UINT32_MAX || bin->count >= capacity) { + llgo_coro_alloc_cache_unlock_v1(bin); + return false; + } + *(void **)pointer = bin->head; + bin->head = pointer; + ++bin->count; + llgo_coro_alloc_cache_unlock_v1(bin); + return true; +} diff --git a/runtime/internal/coroalloc/allocator.go b/runtime/internal/coroalloc/allocator.go index 91ff6e4142..c7b7f7fec8 100644 --- a/runtime/internal/coroalloc/allocator.go +++ b/runtime/internal/coroalloc/allocator.go @@ -92,7 +92,7 @@ func AllocFrame(size uintptr) unsafe.Pointer { if !backendAllocationsAreZeroed || !Ready() || size == 0 { return nil } - return backendAllocFrame(size) + return alloc(size) } // FreeFrame releases a range previously returned by AllocFrame. Backends that @@ -102,7 +102,7 @@ func FreeFrame(ptr unsafe.Pointer, size uintptr) bool { if !Ready() || ptr == nil || size == 0 { return false } - return backendFreeFrame(ptr, size) + return free(ptr, size) } // AllocTask allocates zero-filled, pointer-containing scheduler task storage. @@ -115,7 +115,7 @@ func AllocTask(size uintptr) unsafe.Pointer { if !backendAllocationsAreZeroed || !Ready() || size == 0 { return nil } - return backendAllocFrame(size) + return alloc(size) } // FreeTask performs the physical half of the scheduler's exactly-once task @@ -125,5 +125,21 @@ func FreeTask(ptr unsafe.Pointer, size uintptr) bool { if !Ready() || ptr == nil || size == 0 { return false } - return backendFreeFrame(ptr, size) + return free(ptr, size) +} + +func alloc(size uintptr) unsafe.Pointer { + allocationSize := cacheAllocationSize(size) + if ptr := cacheTake(allocationSize); ptr != nil { + return ptr + } + return backendAllocFrame(allocationSize) +} + +func free(ptr unsafe.Pointer, size uintptr) bool { + allocationSize := cacheAllocationSize(size) + if cachePut(ptr, allocationSize) { + return true + } + return backendFreeFrame(ptr, allocationSize) } diff --git a/runtime/internal/coroalloc/allocator_test.go b/runtime/internal/coroalloc/allocator_test.go index 92cf827362..05487ca225 100644 --- a/runtime/internal/coroalloc/allocator_test.go +++ b/runtime/internal/coroalloc/allocator_test.go @@ -83,3 +83,20 @@ func TestSelectedBackendKindIsKnown(t *testing.T) { t.Fatalf("unknown statically selected backend %q", backendKind) } } + +func TestNativeCacheAllocationSizeClasses(t *testing.T) { + tests := []struct { + size uintptr + want uintptr + }{ + {0, 256}, {256, 256}, {257, 288}, + {1024, 1024}, {1025, 1152}, {1152, 1152}, + {4096, 4096}, {4097, 8192}, {8192, 8192}, + {8193, 16384}, {65536, 65536}, {65537, 65537}, + } + for _, test := range tests { + if got := nativeCacheAllocationSize(test.size); got != test.want { + t.Fatalf("native class for %d = %d, want %d", test.size, got, test.want) + } + } +} diff --git a/runtime/internal/coroalloc/backend_target_selection_test.go b/runtime/internal/coroalloc/backend_target_selection_test.go index 8b1e74d95f..a771056755 100644 --- a/runtime/internal/coroalloc/backend_target_selection_test.go +++ b/runtime/internal/coroalloc/backend_target_selection_test.go @@ -65,6 +65,11 @@ func TestWebAssemblyTargetsSelectExplicitCollectorBackend(t *testing.T) { if slices.Contains(pkg.GoFiles, "backend_gc.go") { t.Fatalf("GoFiles = %v, unexpectedly selected BDWGC backend", pkg.GoFiles) } + if !slices.Contains(pkg.GoFiles, "cache_disabled.go") || + slices.Contains(pkg.GoFiles, "cache_native_llgo.go") || + slices.Contains(pkg.GoFiles, "build_cache_native_llgo.go") { + t.Fatalf("GoFiles = %v, want the cache-free WebAssembly allocator", pkg.GoFiles) + } if !slices.Contains(pkg.TestGoFiles, "backend_webassembly_test.go") { t.Fatalf("TestGoFiles = %v, want backend_webassembly_test.go", pkg.TestGoFiles) } diff --git a/runtime/internal/coroalloc/build_cache_native_llgo.go b/runtime/internal/coroalloc/build_cache_native_llgo.go new file mode 100644 index 0000000000..5c7c14de7a --- /dev/null +++ b/runtime/internal/coroalloc/build_cache_native_llgo.go @@ -0,0 +1,7 @@ +//go:build llgo && (darwin || linux) && !baremetal && !wasm && !tinygo.wasm + +package coroalloc + +// LLGoFiles adds the bounded native size-class cache. Its C bin metadata keeps +// the lazily computed class capacity in otherwise unused alignment padding. +const LLGoFiles = "_cache/cache.c" diff --git a/runtime/internal/coroalloc/cache_disabled.go b/runtime/internal/coroalloc/cache_disabled.go new file mode 100644 index 0000000000..4c9c0db371 --- /dev/null +++ b/runtime/internal/coroalloc/cache_disabled.go @@ -0,0 +1,33 @@ +//go:build !llgo || (!darwin && !linux) || baremetal || wasm || tinygo.wasm + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import "unsafe" + +func cacheAllocationSize(size uintptr) uintptr { + return size +} + +func cacheTake(uintptr) unsafe.Pointer { + return nil +} + +func cachePut(unsafe.Pointer, uintptr) bool { + return false +} diff --git a/runtime/internal/coroalloc/cache_native_c_test.go b/runtime/internal/coroalloc/cache_native_c_test.go new file mode 100644 index 0000000000..e81ac05195 --- /dev/null +++ b/runtime/internal/coroalloc/cache_native_c_test.go @@ -0,0 +1,208 @@ +//go:build darwin || linux + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestNativeCacheBoundsZeroesAndSerializes(t *testing.T) { + cc, err := exec.LookPath("cc") + if err != nil { + t.Skip("cc is unavailable") + } + dir := t.TempDir() + harness := filepath.Join(dir, "cache_test.c") + program := ` +#include +#include +#include +#include +#include + +void *__llgo_coro_alloc_cache_take_v1(uintptr_t size); +bool __llgo_coro_alloc_cache_put_v1(void *pointer, uintptr_t size); + +enum { + bounded_size = 256, + bounded_capacity = 512, + compact_size = 576, + large_compact_size = 1152, + thread_count = 8, + thread_iterations = 10000, + concurrent_size = 512, +}; + +static void *exercise_cache(void *unused) { + (void)unused; + for (int iteration = 0; iteration < thread_iterations; ++iteration) { + unsigned char *pointer = + __llgo_coro_alloc_cache_take_v1(concurrent_size); + if (pointer == NULL) { + pointer = calloc(1, concurrent_size); + } + if (pointer == NULL) { + return (void *)(uintptr_t)1; + } + for (int index = 0; index < concurrent_size; ++index) { + if (pointer[index] != 0) { + return (void *)(uintptr_t)2; + } + } + memset(pointer, 0xa5, concurrent_size); + if (!__llgo_coro_alloc_cache_put_v1(pointer, concurrent_size)) { + free(pointer); + } + } + return NULL; +} + +int main(void) { + if (__llgo_coro_alloc_cache_take_v1(255) != NULL || + __llgo_coro_alloc_cache_put_v1(NULL, bounded_size)) { + return 10; + } + unsigned char *unsupported = malloc(255); + if (unsupported == NULL || + __llgo_coro_alloc_cache_put_v1(unsupported, 255)) { + return 11; + } + free(unsupported); + + unsigned char *stored[bounded_capacity]; + for (int index = 0; index < bounded_capacity; ++index) { + stored[index] = malloc(bounded_size); + if (stored[index] == NULL) { + return 12; + } + memset(stored[index], 0xa5, bounded_size); + if (!__llgo_coro_alloc_cache_put_v1(stored[index], bounded_size)) { + return 13; + } + } + unsigned char *overflow = malloc(bounded_size); + if (overflow == NULL) { + return 14; + } + memset(overflow, 0xa5, bounded_size); + if (__llgo_coro_alloc_cache_put_v1(overflow, bounded_size)) { + return 15; + } + free(overflow); + + for (int index = 0; index < bounded_capacity; ++index) { + unsigned char *pointer = + __llgo_coro_alloc_cache_take_v1(bounded_size); + if (pointer == NULL) { + return 16; + } + for (int byte = 0; byte < bounded_size; ++byte) { + if (pointer[byte] != 0) { + return 17; + } + } + free(pointer); + } + if (__llgo_coro_alloc_cache_take_v1(bounded_size) != NULL) { + return 18; + } + + unsigned char *compact = malloc(compact_size); + if (compact == NULL) { + return 21; + } + memset(compact, 0xa5, compact_size); + if (!__llgo_coro_alloc_cache_put_v1(compact, compact_size)) { + return 22; + } + compact = __llgo_coro_alloc_cache_take_v1(compact_size); + if (compact == NULL) { + return 23; + } + for (int byte = 0; byte < compact_size; ++byte) { + if (compact[byte] != 0) { + return 24; + } + } + free(compact); + + unsigned char *large_compact = malloc(large_compact_size); + if (large_compact == NULL) { + return 25; + } + memset(large_compact, 0xa5, large_compact_size); + if (!__llgo_coro_alloc_cache_put_v1(large_compact, large_compact_size)) { + return 26; + } + large_compact = __llgo_coro_alloc_cache_take_v1(large_compact_size); + if (large_compact == NULL) { + return 27; + } + for (int byte = 0; byte < large_compact_size; ++byte) { + if (large_compact[byte] != 0) { + return 28; + } + } + free(large_compact); + + pthread_t threads[thread_count]; + for (int index = 0; index < thread_count; ++index) { + if (pthread_create(&threads[index], NULL, exercise_cache, NULL) != 0) { + return 19; + } + } + for (int index = 0; index < thread_count; ++index) { + void *result = (void *)(uintptr_t)1; + if (pthread_join(threads[index], &result) != 0 || result != NULL) { + return 20; + } + } + for (;;) { + void *pointer = + __llgo_coro_alloc_cache_take_v1(concurrent_size); + if (pointer == NULL) { + break; + } + free(pointer); + } + return 0; +} +` + if err := os.WriteFile(harness, []byte(program), 0o600); err != nil { + t.Fatal(err) + } + cache, err := filepath.Abs(filepath.Join("_cache", "cache.c")) + if err != nil { + t.Fatal(err) + } + executable := filepath.Join(dir, "cache_test") + compile := exec.Command( + cc, "-std=c11", "-Wall", "-Wextra", "-Werror", "-pthread", + cache, harness, "-o", executable, + ) + if output, err := compile.CombinedOutput(); err != nil { + t.Fatalf("compile native coroutine cache test: %v\n%s", err, output) + } + if output, err := exec.Command(executable).CombinedOutput(); err != nil { + t.Fatalf("native coroutine cache test failed: %v\n%s", err, output) + } +} diff --git a/runtime/internal/coroalloc/cache_native_llgo.go b/runtime/internal/coroalloc/cache_native_llgo.go new file mode 100644 index 0000000000..86181c8d51 --- /dev/null +++ b/runtime/internal/coroalloc/cache_native_llgo.go @@ -0,0 +1,46 @@ +//go:build llgo && (darwin || linux) && !baremetal && !wasm && !tinygo.wasm + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import "unsafe" + +// cacheAllocationSize maps the repeatedly allocated coroutine/task range onto +// compact classes. Compiler frames are commonly 250 bytes to a few KiB; using +// powers of two there can waste almost half of every live goroutine. The hot +// 256..1024 byte range therefore uses 32-byte classes; 1152..4096 uses 128-byte +// classes. Both mappings remain one range check plus an add/mask, and the C +// cache derives the same bin index with arithmetic rather than a search. +// Larger, uncommon ranges retain powers of two and outliers retain the +// backend's exact-size path. +func cacheAllocationSize(size uintptr) uintptr { + return nativeCacheAllocationSize(size) +} + +// The cache leaf owns no Go pointer, callback, scheduler object, or coroutine +// handle. It only transfers a zeroed raw allocation under a short C11 atomic +// critical section, so these are the two exact native boundary facts which +// cannot be inferred from a C signature. + +//llgo:coro noblock +//go:linkname cacheTake C.__llgo_coro_alloc_cache_take_v1 +func cacheTake(size uintptr) unsafe.Pointer + +//llgo:coro noblock +//go:linkname cachePut C.__llgo_coro_alloc_cache_put_v1 +func cachePut(ptr unsafe.Pointer, size uintptr) bool diff --git a/runtime/internal/coroalloc/size_class.go b/runtime/internal/coroalloc/size_class.go new file mode 100644 index 0000000000..9773d86f5c --- /dev/null +++ b/runtime/internal/coroalloc/size_class.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +// nativeCacheAllocationSize is target-independent so the host test suite can +// gate the exact classes used by the llgo-only native cache. +func nativeCacheAllocationSize(size uintptr) uintptr { + if size <= 1024 { + if size <= 256 { + return 256 + } + return (size + 31) &^ 31 + } + if size <= 4096 { + return (size + 127) &^ 127 + } + switch { + case size <= 8192: + return 8192 + case size <= 16384: + return 16384 + case size <= 32768: + return 32768 + case size <= 65536: + return 65536 + default: + return size + } +} diff --git a/runtime/internal/runtime/coro_channel_adapter_test.go b/runtime/internal/runtime/coro_channel_adapter_test.go index 7b2f53639d..1c4c3809b2 100644 --- a/runtime/internal/runtime/coro_channel_adapter_test.go +++ b/runtime/internal/runtime/coro_channel_adapter_test.go @@ -1177,7 +1177,7 @@ func TestCoroChannelAdapterPairCommitAndResume(t *testing.T) { } if migrateRecvFrame.header.SuspendReason != uint16(coro.SuspendPark) || migrateRecvFrame.header.Lifecycle != uint16(coro.FrameSuspended) || - migrateRecvFrame.header.StateID != 41 || migrateRecvFrame.header.Line != 73 { + migrateRecvFrame.header.Line != 73 { t.Fatalf("compact try-or-park receive header = %+v", *migrateRecvFrame.header) } if parked, ok := coro.Resumed(p, migrateRecvFrame.g, migrateRecvAction); !ok || parked.Kind != coro.ActionPark { diff --git a/runtime/internal/runtime/coro_frame.go b/runtime/internal/runtime/coro_frame.go index f09aa9eef6..4ff6c66617 100644 --- a/runtime/internal/runtime/coro_frame.go +++ b/runtime/internal/runtime/coro_frame.go @@ -25,7 +25,7 @@ import ( //export __llgo_coro_frame_alloc_v1 func __llgo_coro_frame_alloc_v1(g unsafe.Pointer, size, align uintptr, descriptor unsafe.Pointer) unsafe.Pointer { - total, ok := coro.FrameAllocationSize(size, align) + total, ok := coro.CompilerFrameAllocationSize(size, align) if !ok { coroRuntimeAbort("invalid coroutine frame allocation size") } @@ -33,7 +33,7 @@ func __llgo_coro_frame_alloc_v1(g unsafe.Pointer, size, align uintptr, descripto if raw == nil { coroRuntimeAbort("coroutine frame allocation failed") } - storage, ok := coro.RegisterFrame((*coro.G)(g), raw, total, size, align, descriptor) + storage, ok := coro.RegisterFrameCompiler((*coro.G)(g), raw, total, size, align, descriptor) if !ok { if !coroalloc.FreeFrame(raw, total) { coroRuntimeAbort("coroutine frame allocation rollback failed") @@ -44,10 +44,12 @@ func __llgo_coro_frame_alloc_v1(g unsafe.Pointer, size, align uintptr, descripto } //export __llgo_coro_frame_publish_v1 -func __llgo_coro_frame_publish_v1(g, handle, header, storage unsafe.Pointer) { - if !coro.PublishFrame((*coro.G)(g), handle, (*coro.HeaderV1)(header), storage) { - coroRuntimeAbort("invalid coroutine frame publication") - } +func __llgo_coro_frame_publish_v1(_, _, _, _ unsafe.Pointer) { + // The compact allocator stores Frame inside compiler-provided metadata; + // publication without that address cannot reconstruct ownership safely. + // Every current compiler path, including the synthesized program bootstrap, + // uses V3. Fail closed if an incompatible archive reaches this runtime. + coroRuntimeAbort("legacy coroutine frame publication is unsupported") } //go:noinline @@ -55,7 +57,7 @@ func __llgo_coro_frame_publish_v1(g, handle, header, storage unsafe.Pointer) { func __llgo_coro_frame_publish_v3( g, handle, header, storage, metadata, descriptor, resultSlot unsafe.Pointer, ) { - if !coro.PublishFrameV3( + if !coro.PublishFrameV3Compiler( (*coro.G)(g), handle, (*coro.HeaderV1)(header), storage, metadata, descriptor, resultSlot, ) { @@ -134,15 +136,15 @@ func __llgo_coro_complete_prepare_v2(g, handle, header unsafe.Pointer, status ui //export __llgo_coro_frame_free_v1 func __llgo_coro_frame_free_v1(g, storage unsafe.Pointer, size, align uintptr, descriptor unsafe.Pointer) { task := (*coro.G)(g) - raw, total, ok := coro.ReleaseFrame(task, storage, size, align, descriptor) + metadata, raw, total, ok := coro.ReleaseFrameCompiler(task, storage, size, align, descriptor) if !ok { coroRuntimeAbort("invalid coroutine frame destruction") } // A managed child panic remains recoverable by its parent, so every logical // G may retain that pending frame. Only the static command G may retain a // terminal frame chain because its native entry owns the no-return report. - if coro.RetainPendingPanicTraceFrame(task, raw, total) || - task == &coroProgramGV1State && coro.RetainPanicTraceFrame(task, raw, total) { + if coro.RetainPendingPanicTraceFrameCompiler(task, metadata, raw, total) || + task == &coroProgramGV1State && coro.RetainPanicTraceFrameCompiler(task, metadata, raw, total) { return } if !coroalloc.FreeFrame(raw, total) { diff --git a/runtime/internal/runtime/coro_run_slice.go b/runtime/internal/runtime/coro_run_slice.go index 545d6e9ac4..79dc5d1357 100644 --- a/runtime/internal/runtime/coro_run_slice.go +++ b/runtime/internal/runtime/coro_run_slice.go @@ -162,45 +162,64 @@ func coroRunPhysicalActionV1( g *coro.G, action coro.Action, runtimeContext unsafe.Pointer, -) (next coro.Action, advanced, committed bool) { + fuseCompletionDestroy bool, +) (next coro.Action, advanced, committed, fusedDestroy bool) { switch action.Kind { case coro.ActionCheckResume: next, needsRuntimeContext, ok := coro.BeginIssuedExecutorResumeRuntimeContext(driver, g) if !ok { - return coro.Action{}, false, false + return coro.Action{}, false, false, false } if needsRuntimeContext { activation, entered := coroEnterRuntimeContextFrom(g, runtimeContext) if !entered { - return coro.Action{}, false, false + return coro.Action{}, false, false, false } coroHandleResume(next.Handle) if !coroLeaveRuntimeContext(g, activation) { - return coro.Action{}, false, false + return coro.Action{}, false, false, false } } else { coroHandleResume(next.Handle) } next, committed, advanced = coro.ResumedExecutorRun(driver, p, g, next) - return next, advanced, committed + if !advanced || committed || !fuseCompletionDestroy || + next.Kind != coro.ActionCheckDestroy { + return next, advanced, committed, false + } + checkDestroy := next + if !coro.CanBeginIssuedExecutorDestroyAfterResume(driver, g, checkDestroy) { + return checkDestroy, advanced, committed, false + } + next, ok = coro.BeginIssuedExecutorDestroyAfterResume( + driver, g, checkDestroy, coroHandleDone(checkDestroy.Handle), + ) + if !ok { + return checkDestroy, advanced, committed, false + } + if !coroHandleDestroyCommitted(g, next.Handle) { + return coro.Action{}, false, false, false + } + next, advanced = coro.DestroyedBounded(p, g, next) + return next, advanced, false, advanced case coro.ActionCheckDestroy: next, ok := coro.CheckedExecutorRun(driver, g, action, coroHandleDone(action.Handle)) if !ok || next.Kind != coro.ActionDestroy || next.Handle != action.Handle { - return coro.Action{}, false, false + return coro.Action{}, false, false, false } if !coroHandleDestroyCommitted(g, next.Handle) { - return coro.Action{}, false, false + return coro.Action{}, false, false, false } next, advanced = coro.DestroyedBounded(p, g, next) - return next, advanced, false + return next, advanced, false, false case coro.ActionPanicDestroy: if !coroHandleDestroyCommitted(g, action.Handle) { - return coro.Action{}, false, false + return coro.Action{}, false, false, false } next, advanced = coro.PanicDestroyedBounded(p, g, action) - return next, advanced, false + return next, advanced, false, false default: - return coro.Action{}, false, false + return coro.Action{}, false, false, false } } @@ -276,9 +295,14 @@ func coroReduceExecutorRunActionPreparedV1( dispatched bool, returnRequested bool, runtimeContext unsafe.Pointer, + remaining uint32, result *coroRunResultV1, ) (terminal, ok bool) { - if g == nil || action.Handle == nil || runtimeContext == nil { + baseCost := uint32(1) + if dispatched { + baseCost++ + } + if g == nil || action.Handle == nil || runtimeContext == nil || remaining < baseCost { return false, false } if dispatched { @@ -288,7 +312,9 @@ func coroReduceExecutorRunActionPreparedV1( } result.dispatches++ } - next, advanced, committed := coroRunPhysicalActionV1(p, driver, g, action, runtimeContext) + next, advanced, committed, fusedDestroy := coroRunPhysicalActionV1( + p, driver, g, action, runtimeContext, remaining > baseCost, + ) // The physical resume may have changed program lifecycle. Re-read the live // policy before selecting the scheduler commit placement. running, returnRequested := false, false @@ -362,6 +388,10 @@ func coroReduceExecutorRunActionPreparedV1( if dispatched { result.used++ } + if fusedDestroy { + result.used++ + result.destroys++ + } switch action.Kind { case coro.ActionCheckResume: result.resumes++ @@ -428,6 +458,7 @@ func coroReduceExecutorRunStepV1( target coroRunTargetCapabilityV1, step coro.ExecutorRunStep, runtimeContext unsafe.Pointer, + remaining uint32, result *coroRunResultV1, ) (terminal, ok bool) { if p == nil || driver == nil || runtimeContext == nil || result == nil || !policy.valid() { @@ -490,7 +521,7 @@ func coroReduceExecutorRunStepV1( case coro.ExecutorRunStepAction: return coroReduceExecutorRunActionPreparedV1( p, driver, policy, target, step.G, step.Action, step.Dispatched, - returnRequested, runtimeContext, result, + returnRequested, runtimeContext, remaining, result, ) case coro.ExecutorRunStepDestroyCommit: if step.G == nil || step.Action.Kind != coro.ActionCommitDestroy || step.Action.Handle != nil { @@ -562,7 +593,7 @@ func coroRunSliceAtV1(p *coro.P, driver *coro.ExecutorDriver, now int64, budget terminal, reduced = coroReduceExecutorRunActionPreparedV1( p, driver, coroRunPolicyV1{}, target, actionStep.G, actionStep.Action, actionStep.Dispatched, - false, runtimeContext, &result, + false, runtimeContext, budget-result.used, &result, ) } else { var step coro.ExecutorRunStep @@ -576,7 +607,8 @@ func coroRunSliceAtV1(p *coro.P, driver *coro.ExecutorDriver, now int64, budget return coroRunResultV1{} } terminal, reduced = coroReduceExecutorRunStepV1( - p, driver, coroRunPolicyV1{}, target, step, runtimeContext, &result, + p, driver, coroRunPolicyV1{}, target, step, runtimeContext, + budget-result.used, &result, ) } if !reduced { diff --git a/runtime/internal/runtime/coro_sched.go b/runtime/internal/runtime/coro_sched.go index 188860594f..387ada91c3 100644 --- a/runtime/internal/runtime/coro_sched.go +++ b/runtime/internal/runtime/coro_sched.go @@ -104,7 +104,7 @@ func coroRunSlice(p *coroP, main *coroG, driver *coro.ExecutorDriver, budget uin terminal, reduced = coroReduceExecutorRunActionPreparedV1( p, driver, policy, target, actionStep.G, actionStep.Action, actionStep.Dispatched, - returnRequested, runtimeContext, &result, + returnRequested, runtimeContext, budget-result.used, &result, ) } else { step, nextOK := coroProgramNextRunStepV1(driver, &run, combineDispatch) @@ -113,7 +113,8 @@ func coroRunSlice(p *coroP, main *coroG, driver *coro.ExecutorDriver, budget uin return coroRunResultV1{} } terminal, reduced = coroReduceExecutorRunStepV1( - p, driver, policy, target, step, runtimeContext, &result, + p, driver, policy, target, step, runtimeContext, + budget-result.used, &result, ) } if !reduced { diff --git a/runtime/internal/runtime/coro_spawn.go b/runtime/internal/runtime/coro_spawn.go index 3cd1d98aa8..92e05ae4da 100644 --- a/runtime/internal/runtime/coro_spawn.go +++ b/runtime/internal/runtime/coro_spawn.go @@ -45,14 +45,14 @@ func coroSpawnBeginV1(parentPointer unsafe.Pointer) (unsafe.Pointer, bool) { return nil, false } child, _, actualSize, allocationOK := coroTaskAllocationAt(raw) - if !allocationOK || actualSize != allocationSize || !coro.BeginSpawn(parent, child, raw, taskSize) { + if !allocationOK || actualSize != allocationSize || !coro.BeginSpawnCompiler(parent, child, raw, taskSize) { coro.Zero(raw, allocationSize) if !coroalloc.FreeTask(raw, allocationSize) { return nil, false } return nil, false } - if !coroBindTaskAllocationRuntimeContext(child, parent) { + if !coroBindTaskAllocationRuntimeContextCompiler(child, parent) { rolled, rolledSize, ok := coro.RollbackSpawn(parent, child) if !ok || rolled != raw || rolledSize != taskSize { return nil, false @@ -68,7 +68,7 @@ func coroSpawnBeginV1(parentPointer unsafe.Pointer) (unsafe.Pointer, bool) { func coroSpawnCommitV1(parentPointer, childPointer, handle unsafe.Pointer) bool { parent, child := (*coroG)(parentPointer), (*coroG)(childPointer) - return coro.CommitSpawn(parent, child, handle) + return coro.CommitSpawnCompiler(parent, child, handle) } // coroReleaseCompletedTask performs the physical half of spawned-G @@ -80,13 +80,13 @@ func coroReleaseCompletedTask(g *coroG) bool { // frame completion after source-specific park cleanup. The cancellation // record remains sticky until the G is physically dead; acknowledge it here // before applying the normal reclaimability/storage transfer contract. - local, raw, taskSize, owned, ok := coro.ReleaseCompletedTask(g) + local, raw, taskSize, owned, ok := coro.ReleaseCompletedTaskCompiler(g) if !ok { if !coro.AcknowledgeTaskCancellation(g, coro.TaskCancelAbort) && !coro.AcknowledgeTaskCancellation(g, coro.TaskCancelShutdown) { return false } - local, raw, taskSize, owned, ok = coro.ReleaseCompletedTask(g) + local, raw, taskSize, owned, ok = coro.ReleaseCompletedTaskCompiler(g) if !ok { return false } diff --git a/runtime/internal/runtime/coro_task_context.go b/runtime/internal/runtime/coro_task_context.go index 9012478749..fb72211f2a 100644 --- a/runtime/internal/runtime/coro_task_context.go +++ b/runtime/internal/runtime/coro_task_context.go @@ -116,6 +116,62 @@ func coroBindTaskAllocationRuntimeContext(task, parent *coro.G) bool { return coroBindRuntimeContextAt(task, parentG, ctx, true) } +// coroBindTaskAllocationRuntimeContextCompiler is the adjacent spawn lane. +// BeginSpawnCompiler has already established that task is the aligned base of +// a fresh zero-filled coroTaskAllocation, while the currently resumed parent +// owns the only reference to it. Validate the parent's live logical context +// directly, initialize the known tail address, and publish it through the +// reciprocal compiler transaction without repeating generic task audits. +func coroBindTaskAllocationRuntimeContextCompiler(task, parent *coro.G) bool { + if task == nil || parent == nil || coro.TaskLocal(task) != nil { + return false + } + parentContext := (*coroRuntimeContext)(coro.TaskLocal(parent)) + if parentContext == nil { + return false + } + parentG := &parentContext.g + if parentG.context != parentContext || parentG.localContext != &parentContext.local || + parentG.startfn != nil || parentG.startarg != unsafe.Pointer(parent) { + return false + } + if parentG.coroEmbedded { + if parentContext != &(*coroTaskAllocation)(unsafe.Pointer(parent)).context { + return false + } + } else if !parentG.isMain { + return false + } + switch readgstatus(parentG) { + case _Grunnable: + if parentG.m != nil { + return false + } + case _Grunning: + if parentG.m == nil || parentG.m.curg != parentG || parentG.m.p == nil || + parentG.m.p.m != parentG.m || readpstatus(parentG.m.p) != _Prunning { + return false + } + default: + return false + } + + ctx := &(*coroTaskAllocation)(unsafe.Pointer(task)).context + if ctx.g.context != nil || ctx.g.localContext != nil { + return false + } + gp := initCoroRuntimeContext(ctx, parentG, _Grunnable) + gp.localContext = &ctx.local + gp.isMain = false + gp.coroEmbedded = true + if coro.BindTaskLocalCompiler(task, unsafe.Pointer(ctx)) { + gp.startarg = unsafe.Pointer(task) + return true + } + discardCoroRuntimeContext(ctx, false) + return false +} + // validCoroRuntimeContext checks only state which follows the logical G. Its // temporary physical M/P attachment is validated exactly once by enter/leave; // parent spawn admission performs its own running-state check. diff --git a/runtime/internal/runtime/coro_task_context_test_adapter.go b/runtime/internal/runtime/coro_task_context_test_adapter.go index 30eeb08645..d598aaa570 100644 --- a/runtime/internal/runtime/coro_task_context_test_adapter.go +++ b/runtime/internal/runtime/coro_task_context_test_adapter.go @@ -43,6 +43,10 @@ func coroBindTaskAllocationRuntimeContext(task, parent *coro.G) bool { return task != nil } +func coroBindTaskAllocationRuntimeContextCompiler(task, parent *coro.G) bool { + return task != nil && parent != nil +} + var coroTestRuntimeContextV1 byte func coroCaptureRuntimeContextV1() unsafe.Pointer { diff --git a/runtime/internal/runtime/z_chan_coro.go b/runtime/internal/runtime/z_chan_coro.go index 35281a2b89..a1c6b62cf6 100644 --- a/runtime/internal/runtime/z_chan_coro.go +++ b/runtime/internal/runtime/z_chan_coro.go @@ -1703,7 +1703,7 @@ func prepareCoroChanParkStateV2( elem unsafe.Pointer, size int, send bool, - stateID, line uint32, + _ uint32, line uint32, ) (*coro.ExecutorDriver, coro.RouteID) { // magic is the compiler-spill lifecycle capability. Fresh coroutine frames // are zero-filled, and the direct resume prologue clears magic only after @@ -1715,7 +1715,6 @@ func prepareCoroChanParkStateV2( } frameHeader.SuspendReason = uint16(coro.SuspendPark) frameHeader.Lifecycle = uint16(coro.FrameSuspended) - frameHeader.StateID = stateID frameHeader.Line = line // magic == 0 certifies that the prior resume retired every embedded record. // Initialize only the five words consumed by the hchan queue instead of