diff --git a/.github/workflows/coroutine.yml b/.github/workflows/coroutine.yml index 4c34865ab2..a5f4acabea 100644 --- a/.github/workflows/coroutine.yml +++ b/.github/workflows/coroutine.yml @@ -45,7 +45,7 @@ jobs: - name: Test coroutine build integration if: matrix.llvm == 19 - run: go test ./internal/build -run 'Test(CoroPlanBuilderRunsBeforeCodegenWithoutChangingIR|CoroPlanInputCanonicalizesPatchedRoot|BuildCoroPlanErrors|CoroEntryResolutionUsesPlanMatchedPackageCache|CoroEntryResolutionBuildsPreparedRuntimePackages|CoroEmissionCoverageStopsBeforeAnyPackageCodegen|CoroUnsupportedEntryResolutionReturnsErrorBeforeCodegen|CoroEmissionUniverseAcceptsModeTestVariants)$' -count=1 + run: go test ./internal/build -run 'Test(CoroPlanBuilderRunsBeforeCodegenWithoutChangingIR|CoroPlanInputCanonicalizesPatchedRoot|ActiveCoroABIVersions|BuildCoroPlanErrors|CoroEntryResolutionUsesPlanMatchedPackageCache|CoroEntryResolutionBuildsPreparedRuntimePackages|CoroEmissionCoverageStopsBeforeAnyPackageCodegen|CoroUnsupportedEntryResolutionReturnsErrorBeforeCodegen|CoroEmissionUniverseAcceptsModeTestVariants)$' -count=1 - name: Test coroutine compiler integration if: matrix.llvm == 19 @@ -54,7 +54,7 @@ jobs: go test -race ./cl -run '^Test(CompilationCoroPlanObservationAndCacheRegistration|CoroEntryResolutionPlainPrimaryPreservesIR|ResolveFunctionSymbolUsesPrimaryAndExactPlan|CoroEntryRejectsUnsupportedBeforeCreatingSymbol|CoroEntryResolutionPreflightRejectsWholePlanBeforeCodegen|CoroEntryResolutionPreflightRejectsMissingPlanAndCache|Emission.*)$' -count=1 - name: Test structured LLVM coroutine builder - run: go test -tags='${{ matrix.tags }}' -v ./ssa -run '^TestCoroBuilder' -count=1 + run: go test -tags='${{ matrix.tags }}' -v ./ssa -run '^TestCoro(Builder|Handle|Promise|RootFactory)' -count=1 - name: Test canonical coroutine plan digest and cache identity run: | @@ -63,7 +63,7 @@ jobs: go test -tags='${{ matrix.tags }}' ./cl -run '^Test(CompilationCoroABIIdentityValidation|CoroEntryResolutionCacheRegistrationWithDigest|CoroPhysicalABICacheRegistrationPreservesPhysicalMetadata)$' -count=1 - name: Test coroutine physical ABI lowering - run: go test -tags='${{ matrix.tags }}' -v ./cl -run '^TestCoro(LeafPhysicalABI|PhysicalABI)' -count=1 + run: go test -tags='${{ matrix.tags }}' -v ./cl -run '^TestCoro(LeafPhysicalABI|PhysicalABI|ChildAwaitPhysicalABIV1|ExplicitAsyncRootFactoryV1|ExplicitRootFactoryV1)' -count=1 - name: Test LLVM 22 tool configuration if: matrix.llvm == 22 diff --git a/cl/compilation.go b/cl/compilation.go index 7335a0eb1a..cd477f63ee 100644 --- a/cl/compilation.go +++ b/cl/compilation.go @@ -53,8 +53,15 @@ type Compilation struct { FuncRepABI string // EnableCoroPhysicalABI permits the conservative leaf-only coroutine ABI // lowering implemented by the current experimental slice. It requires entry - // resolution and does not enable await, dispatch, roots, or a scheduler. + // resolution and does not by itself enable await, dispatch, roots, or a + // scheduler. EnableCoroPhysicalABI bool + // EnableCoroChildAwait permits the narrowly-scoped static child handoff ABI. + // It requires the physical ABI and emits typed factories for explicit async + // roots. A generated parent only publishes an initial-suspended child and + // suspends itself; a matching scheduler owns every resume and destroy + // operation. + EnableCoroChildAwait bool // EmissionUniverse is the immutable, compilation-scoped set of exact SSA // functions that cl may resolve while emitting this compilation. Active @@ -85,13 +92,20 @@ func (c *Compilation) validateCoroABIIdentity(required bool) error { if c.EnableCoroPhysicalABI { wantCoroABI = coro.PhysicalABIV0 } + if c.EnableCoroChildAwait { + wantCoroABI = coro.PhysicalABIV1 + } + wantSchedulerABI := coro.SchedulerNoneABIV0 + if c.EnableCoroChildAwait { + wantSchedulerABI = coro.SchedulerChildAwaitABIV0 + } checks := []struct { name string got string want string }{ {"coroutine", c.CoroABI, wantCoroABI}, - {"scheduler", c.SchedulerABI, coro.SchedulerNoneABIV0}, + {"scheduler", c.SchedulerABI, wantSchedulerABI}, {"panic", c.PanicABI, coro.PanicLegacyABIV0}, {"function representation", c.FuncRepABI, coro.FuncRepABIV0}, } diff --git a/cl/compilation_test.go b/cl/compilation_test.go index 09a83d95bc..2d06bac3a3 100644 --- a/cl/compilation_test.go +++ b/cl/compilation_test.go @@ -94,6 +94,31 @@ func TestCompilationCoroABIIdentityValidation(t *testing.T) { if err := (&Compilation{EnableCoroEntryResolution: true, EnableCoroPhysicalABI: true}).validateCoroABIIdentity(false); err != nil { t.Fatalf("omitted source ABI identity should use current defaults: %v", err) } + newChildAwait := func() *Compilation { + return &Compilation{ + EnableCoroEntryResolution: true, + EnableCoroPhysicalABI: true, + EnableCoroChildAwait: true, + CoroABI: coro.PhysicalABIV1, + SchedulerABI: coro.SchedulerChildAwaitABIV0, + PanicABI: coro.PanicLegacyABIV0, + FuncRepABI: coro.FuncRepABIV0, + } + } + childAwait := newChildAwait() + if err := childAwait.validateCoroABIIdentity(false); err != nil { + t.Fatalf("complete child-await ABI identity: %v", err) + } + wrongChildAwait := newChildAwait() + wrongChildAwait.CoroABI = coro.PhysicalABIV0 + if err := wrongChildAwait.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "coroutine ABI") { + t.Fatalf("child-await physical ABI mismatch = %v", err) + } + wrongChildAwait = newChildAwait() + wrongChildAwait.SchedulerABI = coro.SchedulerNoneABIV0 + if err := wrongChildAwait.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "scheduler ABI") { + t.Fatalf("child-await scheduler ABI mismatch = %v", err) + } partial := newPhysical() partial.SchedulerABI = "" if err := partial.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "scheduler ABI") { diff --git a/cl/compile.go b/cl/compile.go index 3c6cb61bd0..1e3b34d415 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -184,6 +184,7 @@ type context struct { cacheRegistration bool // cached archive: skip observers; emitted IR is transient pcLineSeq uint64 sourceParamBase int // hidden physical parameters before source params + currentCoro *coroBodyContext patches Patches blkInfos []blocks.Info @@ -536,6 +537,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun }() return p.patchType(f.Signature).(*types.Signature) }() + sourceSig := sig state := p.state isInit := (f.Name() == "init" && sig.Recv() == nil) if isInit && state == pkgHasPatch { @@ -578,6 +580,9 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun fn.DisableTailCalls() } p.funcs[f] = fn + if physicalABI != nil && entry.childAwait { + p.emitCoroRootFactory(pkg, entry, *physicalABI, sourceSig, fn) + } isCgo := isCgoExternSymbol(f) if nblk := len(f.Blocks); nblk > 0 { if p.prog.FuncInfoMetadataEnabled() { @@ -639,7 +644,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun p.bvals = make(map[ssa.Value]llssa.Expr) p.methodNilDerefChecks = collectMethodNilDerefChecks(f) if physicalABI != nil { - p.compileCoroLeafBody(b, f, *physicalABI) + p.compileCoroPhysicalBody(b, f, *physicalABI) b.EndBuild() return } @@ -1205,6 +1210,10 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue } switch v := iv.(type) { case *ssa.Call: + if value, handled := p.tryCompileCoroStaticAwait(b, v); handled { + ret = value + break + } ret = p.call(b, llssa.Call, &v.Call) if p.rangeFuncCallNeedsDeferDrain(&v.Call) { b.DeferStackDrain() diff --git a/cl/coro_abi.go b/cl/coro_abi.go index 9d45d913c9..94adbc4d33 100644 --- a/cl/coro_abi.go +++ b/cl/coro_abi.go @@ -37,17 +37,88 @@ const ( coroFrameAllocHook = "__llgo_coro_frame_alloc_v0" coroFrameFreeHook = "__llgo_coro_frame_free_v0" coroDescriptorPrefix = "__llgo_coro_frame_descriptor_v0." + + coroPhysicalABIVersionV1 uint32 = 1 + coroFrameAllocHookV1 = "__llgo_coro_frame_alloc_v1" + coroFramePublishHookV1 = "__llgo_coro_frame_publish_v1" + coroAwaitPrepareHookV1 = "__llgo_coro_await_prepare_v1" + coroCompletePrepareHookV1 = "__llgo_coro_complete_prepare_v1" + coroFrameFreeHookV1 = "__llgo_coro_frame_free_v1" + coroDescriptorPrefixV1 = "__llgo_coro_frame_descriptor_v1." +) + +const ( + coroHeaderTask = iota + coroHeaderParent + coroHeaderDescriptor + coroHeaderAllocationBase + coroHeaderResultSlot + coroHeaderSuspendReason + coroHeaderLifecycle + coroHeaderStateID + coroHeaderFlags +) + +const ( + coroSuspendNone uint64 = iota + coroSuspendCall + coroSuspendFrameComplete +) + +const ( + coroLifecycleAllocated uint64 = iota + coroLifecycleInitialSuspended + coroLifecycleActive + coroLifecycleSuspended + coroLifecycleFinalSuspended + coroLifecycleDestroyPending + coroLifecycleDestroyed ) type coroPhysicalABI struct { - hash [16]byte - descriptorName string - physicalSig *types.Signature - resultSlotType types.Type - resultCount int + version uint32 + hash [16]byte + descriptorName string + frameAllocHook string + frameFreeHook string + framePublishHook string + awaitPrepareHook string + completePrepareHook string + physicalSig *types.Signature + resultSlotType types.Type + resultCount int +} + +// coroBodyContext exists only while emitting one physical coroutine body. It +// carries the current handle/header explicitly so call lowering never guesses a +// frame layout from a raw handle. +type coroBodyContext struct { + coro *llssa.CoroBuilder + abi coroPhysicalABI + header llssa.Expr + task llssa.Expr + resultSlot llssa.Expr + completePrepare llssa.Expr + nextState uint32 } func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *types.Signature) coroPhysicalABI { + version := coroPhysicalABIVersion + frameAllocHook := coroFrameAllocHook + frameFreeHook := coroFrameFreeHook + descriptorPrefix := coroDescriptorPrefix + framePublishHook := "" + awaitPrepareHook := "" + completePrepareHook := "" + if p.compilation != nil && p.compilation.EnableCoroChildAwait { + version = coroPhysicalABIVersionV1 + frameAllocHook = coroFrameAllocHookV1 + frameFreeHook = coroFrameFreeHookV1 + descriptorPrefix = coroDescriptorPrefixV1 + framePublishHook = coroFramePublishHookV1 + awaitPrepareHook = coroAwaitPrepareHookV1 + completePrepareHook = coroCompletePrepareHookV1 + } resultFields := make([]*types.Var, sourceSig.Results().Len()) for i := range resultFields { resultFields[i] = types.NewField(token.NoPos, nil, fmt.Sprintf("r%d", i), sourceSig.Results().At(i).Type(), false) @@ -73,6 +144,10 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type target := p.prog.TargetSpec() coroABI := coro.PhysicalABIV0 schedulerABI := coro.SchedulerNoneABIV0 + if p.compilation != nil && p.compilation.EnableCoroChildAwait { + coroABI = coro.PhysicalABIV1 + schedulerABI = coro.SchedulerChildAwaitABIV0 + } panicABI := coro.PanicLegacyABIV0 funcRepABI := coro.FuncRepABIV0 if p.compilation != nil { @@ -91,7 +166,7 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type } key := fmt.Sprintf( "llgo-coro-physical-v%d\x00%s\x00coro=%s\x00scheduler=%s\x00panic=%s\x00func-rep=%s\x00triple=%s\x00cpu=%s\x00features=%s\x00target-abi=%s\x00data-layout=%s\x00ptr=%d\x00sig=%s\x00result=%s", - coroPhysicalABIVersion, + version, entry.plan.ID, coroABI, schedulerABI, @@ -110,27 +185,22 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type var hash [16]byte copy(hash[:], sum[:len(hash)]) return coroPhysicalABI{ - hash: hash, - descriptorName: coroDescriptorPrefix + hex.EncodeToString(hash[:]), - physicalSig: physicalSig, - resultSlotType: resultSlotType, - resultCount: sourceSig.Results().Len(), + version: version, + hash: hash, + descriptorName: descriptorPrefix + hex.EncodeToString(hash[:]), + frameAllocHook: frameAllocHook, + frameFreeHook: frameFreeHook, + framePublishHook: framePublishHook, + awaitPrepareHook: awaitPrepareHook, + completePrepareHook: completePrepareHook, + physicalSig: physicalSig, + resultSlotType: resultSlotType, + resultCount: sourceSig.Results().Len(), } } -func (p *context) beginCoroLeaf(b llssa.Builder, abi coroPhysicalABI) (*llssa.CoroBuilder, llssa.Expr) { - prog := p.prog - resultType := prog.Type(abi.resultSlotType, llssa.InGo) - descriptor := p.pkg.NewCoroFrameDescriptor(abi.descriptorName, llssa.CoroFrameDescriptorOptions{ - Version: coroPhysicalABIVersion, - ABIHash: abi.hash, - Result: resultType, - }) - descriptorPtr := b.Convert(prog.VoidPtr(), descriptor) - task := p.fn.PhysicalParam(0) - resultSlot := p.fn.PhysicalParam(1) - null := prog.Nil(prog.VoidPtr()) - headerType := prog.Struct( +func coroHeaderType(prog llssa.Program) llssa.Type { + return prog.Struct( prog.VoidPtr(), // g prog.VoidPtr(), // parent prog.VoidPtr(), // descriptor @@ -141,61 +211,176 @@ func (p *context) beginCoroLeaf(b llssa.Builder, abi coroPhysicalABI) (*llssa.Co prog.Uint32(), // state ID prog.Uint32(), // flags ) +} + +func (p *context) beginCoroBody(b llssa.Builder, abi coroPhysicalABI) *coroBodyContext { + prog := p.prog + resultType := prog.Type(abi.resultSlotType, llssa.InGo) + descriptor := p.pkg.NewCoroFrameDescriptor(abi.descriptorName, llssa.CoroFrameDescriptorOptions{ + Version: abi.version, + ABIHash: abi.hash, + Result: resultType, + }) + descriptorPtr := b.Convert(prog.VoidPtr(), descriptor) + task := p.fn.PhysicalParam(0) + resultSlot := p.fn.PhysicalParam(1) + null := prog.Nil(prog.VoidPtr()) + headerType := coroHeaderType(prog) header := b.AllocaT(headerType) + initialLifecycle := uint64(coroLifecycleAllocated) + if abi.version >= coroPhysicalABIVersionV1 { + initialLifecycle = coroLifecycleInitialSuspended + } headerValues := []llssa.Expr{ task, null, descriptorPtr, null, resultSlot, - prog.IntVal(0, prog.Uint16()), - prog.IntVal(0, prog.Uint16()), + prog.IntVal(coroSuspendNone, prog.Uint16()), + prog.IntVal(initialLifecycle, prog.Uint16()), prog.IntVal(0, prog.Uint32()), prog.IntVal(0, prog.Uint32()), } - allocSig := coroFrameAllocSignature() - freeSig := coroFrameFreeSignature() - alloc := p.pkg.NewFunc(coroFrameAllocHook, allocSig, llssa.InC) - free := p.pkg.NewFunc(coroFrameFreeHook, freeSig, llssa.InC) + allocSig := coroFrameAllocSignature(abi.version) + freeSig := coroFrameFreeSignature(abi.version) + alloc := p.pkg.NewFunc(abi.frameAllocHook, allocSig, llssa.InC) + free := p.pkg.NewFunc(abi.frameFreeHook, freeSig, llssa.InC) frame := llssa.CoroFrameOps{ Alloc: func(b llssa.Builder, size, align llssa.Expr) llssa.Expr { + if abi.version >= coroPhysicalABIVersionV1 { + return b.Call(alloc.Expr, task, size, align, descriptorPtr) + } return b.Call(alloc.Expr, size, align, descriptorPtr) }, Free: func(b llssa.Builder, storage, size, align llssa.Expr) { + if abi.version >= coroPhysicalABIVersionV1 { + b.Call(free.Expr, task, storage, size, align, descriptorPtr) + return + } b.Call(free.Expr, storage, size, align, descriptorPtr) }, } - return b.BeginCoro(llssa.CoroOptions{ + body := &coroBodyContext{ + abi: abi, + header: header, + task: task, + resultSlot: resultSlot, + nextState: 1, + } + if abi.completePrepareHook != "" { + body.completePrepare = p.pkg.NewFunc(abi.completePrepareHook, coroCompletePrepareSignature(), llssa.InC).Expr + } + body.coro = b.BeginCoro(llssa.CoroOptions{ Promise: header, Frame: frame, - BeforeInitialSuspend: func(b llssa.Builder, _ llssa.Expr) { + BeforeInitialSuspend: func(b llssa.Builder, handle, storage llssa.Expr) { for i, value := range headerValues { b.Store(b.FieldAddr(header, i), value) } + if abi.framePublishHook != "" { + publish := p.pkg.NewFunc(abi.framePublishHook, coroFramePublishSignature(), llssa.InC) + b.Call(publish.Expr, task, handle, b.Convert(prog.VoidPtr(), header), storage) + } }, - }), resultSlot + }) + return body } -func coroFrameAllocSignature() *types.Signature { - params := types.NewTuple( +func coroFrameAllocSignature(version uint32) *types.Signature { + params := []*types.Var{ types.NewParam(token.NoPos, nil, "size", types.Typ[types.Uintptr]), types.NewParam(token.NoPos, nil, "align", types.Typ[types.Uintptr]), types.NewParam(token.NoPos, nil, "descriptor", types.Typ[types.UnsafePointer]), - ) + } + if version >= coroPhysicalABIVersionV1 { + params = append([]*types.Var{types.NewParam(token.NoPos, nil, "g", types.Typ[types.UnsafePointer])}, params...) + } results := types.NewTuple(types.NewParam(token.NoPos, nil, "frame", types.Typ[types.UnsafePointer])) - return types.NewSignatureType(nil, nil, nil, params, results, false) + return types.NewSignatureType(nil, nil, nil, types.NewTuple(params...), results, false) } -func coroFrameFreeSignature() *types.Signature { - params := types.NewTuple( +func coroFrameFreeSignature(version uint32) *types.Signature { + params := []*types.Var{ types.NewParam(token.NoPos, nil, "frame", types.Typ[types.UnsafePointer]), types.NewParam(token.NoPos, nil, "size", types.Typ[types.Uintptr]), types.NewParam(token.NoPos, nil, "align", types.Typ[types.Uintptr]), types.NewParam(token.NoPos, nil, "descriptor", types.Typ[types.UnsafePointer]), + } + if version >= coroPhysicalABIVersionV1 { + params = append([]*types.Var{types.NewParam(token.NoPos, nil, "g", types.Typ[types.UnsafePointer])}, params...) + } + return types.NewSignatureType(nil, nil, nil, types.NewTuple(params...), nil, false) +} + +func coroFramePublishSignature() *types.Signature { + params := types.NewTuple( + types.NewParam(token.NoPos, nil, "g", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "handle", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "header", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "storage", types.Typ[types.UnsafePointer]), + ) + return types.NewSignatureType(nil, nil, nil, params, nil, false) +} + +func coroAwaitPrepareSignature() *types.Signature { + params := types.NewTuple( + types.NewParam(token.NoPos, nil, "g", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "parent", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "child", types.Typ[types.UnsafePointer]), ) return types.NewSignatureType(nil, nil, nil, params, nil, false) } +func coroCompletePrepareSignature() *types.Signature { + params := types.NewTuple( + types.NewParam(token.NoPos, nil, "g", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "handle", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "header", types.Typ[types.UnsafePointer]), + ) + return types.NewSignatureType(nil, nil, nil, params, nil, false) +} + +func (c *coroBodyContext) publishState(b llssa.Builder, reason, lifecycle uint64, stateID uint32) { + prog := b.Prog + b.Store(b.FieldAddr(c.header, coroHeaderSuspendReason), prog.IntVal(reason, prog.Uint16())) + b.Store(b.FieldAddr(c.header, coroHeaderLifecycle), prog.IntVal(lifecycle, prog.Uint16())) + b.Store(b.FieldAddr(c.header, coroHeaderStateID), prog.IntVal(uint64(stateID), prog.Uint32())) +} + +func (c *coroBodyContext) activate(b llssa.Builder) { + if c.abi.version < coroPhysicalABIVersionV1 { + return + } + prog := b.Prog + b.Store(b.FieldAddr(c.header, coroHeaderSuspendReason), prog.IntVal(coroSuspendNone, prog.Uint16())) + b.Store(b.FieldAddr(c.header, coroHeaderLifecycle), prog.IntVal(coroLifecycleActive, prog.Uint16())) +} + +func (c *coroBodyContext) suspendForChild(b llssa.Builder) uint32 { + if c.abi.version < coroPhysicalABIVersionV1 { + panic("coroutine child suspension requires PhysicalABIV1") + } + stateID := c.nextState + c.nextState++ + c.publishState(b, coroSuspendCall, coroLifecycleSuspended, stateID) + return stateID +} + +func (c *coroBodyContext) finish(b llssa.Builder) { + if c.abi.version < coroPhysicalABIVersionV1 { + c.coro.Finish() + return + } + stateID := c.nextState + c.nextState++ + c.publishState(b, coroSuspendFrameComplete, coroLifecycleFinalSuspended, stateID) + if !c.completePrepare.IsNil() { + b.Call(c.completePrepare, c.task, c.coro.Handle(), b.Convert(b.Prog.VoidPtr(), c.header)) + } + c.coro.Finish() +} + func (p *context) storeCoroLeafResult(b llssa.Builder, abi coroPhysicalABI, resultSlot llssa.Expr, results []llssa.Expr) { if len(results) != abi.resultCount { panic(fmt.Sprintf("coroutine result count %d does not match ABI count %d", len(results), abi.resultCount)) @@ -208,22 +393,28 @@ func (p *context) storeCoroLeafResult(b llssa.Builder, abi coroPhysicalABI, resu b.Store(b.FieldAddr(typedSlot, 0), results[0]) } -func (p *context) compileCoroLeafBody(b llssa.Builder, fn *ssa.Function, abi coroPhysicalABI) { +func (p *context) compileCoroPhysicalBody(b llssa.Builder, fn *ssa.Function, abi coroPhysicalABI) { if len(fn.Blocks) != 1 { - panic("coroutine leaf body reached codegen without one-block preflight") + panic("coroutine physical body reached codegen without one-block preflight") } oldBase := p.sourceParamBase + oldCoro := p.currentCoro p.sourceParamBase = 2 - defer func() { p.sourceParamBase = oldBase }() + defer func() { + p.sourceParamBase = oldBase + p.currentCoro = oldCoro + }() b.SetBlock(p.fn.Block(0)) if enableDbgSyms && fn.Origin() == nil { p.debugParams(b, fn) } - leaf, resultSlot := p.beginCoroLeaf(b, abi) - body := leaf.InitialResumeBlock() + physical := p.beginCoroBody(b, abi) + p.currentCoro = physical + body := physical.coro.InitialResumeBlock() completion := p.fn.MakeBlock() b.SetBlock(body) + physical.activate(b) for _, instr := range fn.Blocks[0].Instrs { if _, debug := instr.(*ssa.DebugRef); debug { @@ -237,16 +428,138 @@ func (p *context) compileCoroLeafBody(b llssa.Builder, fn *ssa.Function, abi cor for i, result := range ret.Results { results[i] = p.compileValue(b, result) } - p.storeCoroLeafResult(b, abi, resultSlot, results) + p.storeCoroLeafResult(b, abi, physical.resultSlot, results) b.Jump(completion) continue } p.compileInstr(b, instr) } b.SetBlock(completion) - leaf.Finish() + physical.finish(b) +} + +func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, childAwait bool) error { + if !childAwait { + return validateCoroLeafPhysicalABI(fn, plan) + } + + fail := func(format string, args ...any) error { + return fmt.Errorf("coroutine physical ABI: function %q: %s", plan.ID, fmt.Sprintf(format, args...)) + } + if fn == nil || plan.External != coro.Defined || len(fn.Blocks) == 0 { + return fail("requires one defined SSA body") + } + if plan.Primary != coro.PrimaryCoroutine || plan.FuncRep != coro.DirectCoro { + return fail("requires a direct coroutine primary, got primary=%s representation=%s", plan.Primary, plan.FuncRep) + } + if plan.Demand != coro.AsyncDemand { + return fail("requires async-only demand until root and hard-sync adapters exist, got %s", plan.Demand) + } + if plan.Recursive { + return fail("recursive coroutine lowering requires child frames and preemption polls") + } + if unsupported := plan.Exec &^ coro.MayUnwind; unsupported != 0 { + return fail("execution flags %s require lowering outside the linear physical ABI", unsupported) + } + if fn.Parent() != nil || len(fn.FreeVars) != 0 { + return fail("closures require the coroutine context ABI") + } + if len(fn.AnonFuncs) != 0 { + return fail("nested function literals require closure body lowering") + } + if fn.Signature.Recv() != nil { + return fail("methods require descriptor and receiver ABI lowering") + } + if fn.Signature.Variadic() { + return fail("variadic coroutine ABI is not implemented") + } + if directive := coroLeafABIDirective(fn); directive != "" { + return fail("ABI directive %q requires a root or foreign adapter", directive) + } + if isCgoExternSymbol(fn) { + return fail("cgo entry requires a foreign adapter") + } + if fn.Synthetic != "" { + return fail("synthetic function %q is outside the leaf ABI", fn.Synthetic) + } + if list := fn.TypeParams(); list != nil && list.Len() != 0 { + return fail("generic declarations are not materialized coroutine bodies") + } + if list := fn.TypeArgs(); len(list) != 0 { + return fail("generic instances require a frozen instantiated ABI") + } + if fn.Name() == "main" || strings.HasPrefix(fn.Name(), "init") { + return fail("program roots require scheduler bootstrap lowering") + } + if len(fn.Blocks) != 1 { + return fail("requires exactly one basic block, got %d", len(fn.Blocks)) + } + if err := validateCoroLeafPhysicalSignature(plan, fn.Signature); err != nil { + return err + } + + returns := 0 + awaits := 0 + for _, instr := range fn.Blocks[0].Instrs { + switch instr := instr.(type) { + case *ssa.DebugRef: + case *ssa.Return: + returns++ + case *ssa.BinOp: + if instr.Op == token.QUO || instr.Op == token.REM || instr.Op == token.SHL || instr.Op == token.SHR || + !coroLeafScalar(instr.Type()) || + !coroLeafScalar(instr.X.Type()) || !coroLeafScalar(instr.Y.Type()) { + return coroLeafInstructionError(fn, plan, instr, "potentially panicking or non-scalar binary operation") + } + case *ssa.UnOp: + if (instr.Op != token.SUB && instr.Op != token.XOR && instr.Op != token.NOT) || !coroLeafScalar(instr.Type()) { + return coroLeafInstructionError(fn, plan, instr, "unsupported unary operation") + } + case *ssa.Convert, *ssa.ChangeType: + value, ok := instr.(ssa.Value) + if !ok || !coroLeafScalar(value.Type()) { + return coroLeafInstructionError(fn, plan, instr, "non-scalar conversion") + } + case *ssa.Call: + callee, calleePlan, err := resolveCoroStaticAwait(whole, plan, instr) + if err != nil { + return coroLeafInstructionError(fn, plan, instr, "unsupported child await: "+err.Error()) + } + if err := validateCoroLeafPhysicalSignature(calleePlan, callee.Signature); err != nil { + return coroLeafInstructionError(fn, plan, instr, "child await signature: "+err.Error()) + } + awaits++ + default: + return coroLeafInstructionError(fn, plan, instr, "instruction is outside the linear physical ABI allowlist") + } + } + if returns != 1 { + return fail("requires exactly one return instruction, got %d", returns) + } + if awaits == 0 { + if plan.DeclaredEffect != coro.YieldOnly || plan.LocalEffect != coro.YieldOnly || plan.Effect != coro.YieldOnly { + return fail("requires an explicit, isolated yield-only effect, got declared=%s local=%s final=%s", plan.DeclaredEffect, plan.LocalEffect, plan.Effect) + } + return nil + } + if !plan.Effect.Contains(coro.AwaitStructured) { + return fail("child-await body lacks await-structured final effect: %s", plan.Effect) + } + if unsupported := plan.Effect &^ (coro.YieldOnly | coro.AwaitStructured); unsupported != 0 { + return fail("child-await body has unsupported final effect %s", unsupported) + } + if unsupported := plan.DeclaredEffect &^ coro.YieldOnly; unsupported != 0 { + return fail("child-await body has unsupported declared effect %s", unsupported) + } + if unsupported := plan.LocalEffect &^ coro.YieldOnly; unsupported != 0 { + return fail("child-await body has unsupported local effect %s", unsupported) + } + return nil } +// validateCoroLeafPhysicalABI preserves the v0 leaf-only acceptance boundary +// and diagnostics. Enabling later physical ABI capabilities must not silently +// change an archive still identified as PhysicalABIV0/SchedulerNoneABIV0. func validateCoroLeafPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan) error { fail := func(format string, args ...any) error { return fmt.Errorf("coroutine physical ABI: function %q: %s", plan.ID, fmt.Sprintf(format, args...)) @@ -398,7 +711,7 @@ func coroLeafABIDirective(fn *ssa.Function) string { return "" } -func validateCoroPhysicalConsumers(plan *coro.SSAPlan) error { +func validateCoroPhysicalConsumers(plan *coro.SSAPlan, childAwait bool) error { coroutineIDs := make(map[coro.FunctionID]struct{}) for _, function := range plan.Functions() { if function.Plan.Primary == coro.PrimaryCoroutine { @@ -417,10 +730,23 @@ func validateCoroPhysicalConsumers(plan *coro.SSAPlan) error { if !found { return coroLeafInstructionError(fn, function.Plan, instr, "call has no compilation CallPlan") } + hasCoroutineTarget := false for _, target := range callPlan.Targets { if _, isCoroutine := coroutineIDs[target]; isCoroutine { - return coroLeafInstructionError(fn, function.Plan, instr, "coroutine target requires direct await or root lowering") + hasCoroutineTarget = true + break + } + } + if hasCoroutineTarget { + direct, ordinary := call.(*ssa.Call) + if childAwait && ordinary && function.Plan.Primary == coro.PrimaryCoroutine { + if _, _, err := resolveCoroStaticAwait(plan, function.Plan, direct); err == nil { + // The static callee operand is represented by this exact + // CallPlan and is not an escaped function value. + continue + } } + return coroLeafInstructionError(fn, function.Plan, instr, "coroutine target requires a supported static child await or root lowering") } } for _, operand := range instr.Operands(nil) { diff --git a/cl/coro_abi_test.go b/cl/coro_abi_test.go index 8145f03c93..91c8688ee6 100644 --- a/cl/coro_abi_test.go +++ b/cl/coro_abi_test.go @@ -19,8 +19,10 @@ package cl import ( + "bytes" "go/ast" "regexp" + "strconv" "strings" "testing" @@ -49,6 +51,7 @@ func TestCoroLeafPhysicalABIPresplit(t *testing.T) { t.Fatalf("physical coroutine symbol is absent:\n%s", ir) } leafIR := leaf.String() + assertCoroV0HeaderStateZero(t, leafIR) if !regexp.MustCompile(`define ptr @"?foo\.Leaf\$coro"?\(ptr [^,]+, ptr [^,]+, i32 `).MatchString(leafIR) { t.Fatalf("coroutine leaf does not use (g, out, args...) -> handle ABI:\n%s", leafIR) } @@ -198,6 +201,443 @@ func TestCoroLeafPhysicalABIUsesTargetPointerWidth(t *testing.T) { } } +func TestCoroChildAwaitPhysicalABIV1Presplit(t *testing.T) { + prog, pkg := compileCoroChildAwaitPhysicalABI(t, nil) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify child-await coroutines: %v\n%s", err, module.String()) + } + ir := module.String() + parent := requireCoroPhysicalFunction(t, module, "foo.Parent") + child := requireCoroPhysicalFunction(t, module, "foo.Child") + parentIR, childIR := parent.String(), child.String() + + if got := strings.Count(parentIR, "call i8 @llvm.coro.suspend"); got != 3 { + t.Fatalf("Parent coro.suspend calls = %d, want initial + await + final:\n%s", got, parentIR) + } + if got := strings.Count(childIR, "call i8 @llvm.coro.suspend"); got != 2 { + t.Fatalf("Child coro.suspend calls = %d, want initial + final:\n%s", got, childIR) + } + for _, forbidden := range []string{"llvm.coro.resume", "llvm.coro.done", "llvm.coro.destroy"} { + if hasLLVMCall(parentIR, forbidden) { + t.Fatalf("Parent directly owns forbidden %s operation:\n%s", forbidden, parentIR) + } + } + + for _, hook := range []string{ + coroFrameAllocHookV1, + coroFramePublishHookV1, + coroAwaitPrepareHookV1, + coroCompletePrepareHookV1, + coroFrameFreeHookV1, + } { + if !strings.Contains(ir, hook) { + t.Fatalf("child-await module is missing PhysicalABIV1 hook %q:\n%s", hook, ir) + } + } + for _, forbidden := range []string{coroFrameAllocHook, coroFrameFreeHook, coroDescriptorPrefix} { + if strings.Contains(ir, forbidden) { + t.Fatalf("PhysicalABIV1 module leaked v0 ABI symbol %q:\n%s", forbidden, ir) + } + } + if got := strings.Count(ir, "call ptr @"+coroFrameAllocHookV1); got != 2 { + t.Fatalf("task-aware v1 frame allocations = %d, want Parent + Child:\n%s", got, ir) + } + if got := strings.Count(ir, "call void @"+coroFramePublishHookV1); got != 2 { + t.Fatalf("v1 frame publications = %d, want Parent + Child:\n%s", got, ir) + } + if got := strings.Count(ir, "call void @"+coroAwaitPrepareHookV1); got != 1 { + t.Fatalf("v1 await preparations = %d, want one Parent->Child handoff:\n%s", got, ir) + } + if got := strings.Count(ir, "call void @"+coroCompletePrepareHookV1); got != 2 { + t.Fatalf("v1 completion preparations = %d, want Parent + Child:\n%s", got, ir) + } + if got := strings.Count(ir, "call void @"+coroFrameFreeHookV1); got != 2 { + t.Fatalf("task-aware v1 frame frees = %d, want Parent + Child:\n%s", got, ir) + } + for name, body := range map[string]string{"Parent": parentIR, "Child": childIR} { + assertCoroV1TaskAwareFrameCalls(t, name, body, prog.PointerSize()*8) + assertCoroV1InitialPublish(t, name, body) + assertCoroV1Completion(t, name, body) + } + assertCoroStaticChildAwait(t, parentIR) + + descriptor := regexp.MustCompile( + `@__llgo_coro_frame_descriptor_v1\.[0-9a-f]+ = linkonce_odr unnamed_addr constant \{ [^}]+ \} \{ i32 1,`, + ) + if got := len(descriptor.FindAllString(ir, -1)); got != 2 { + t.Fatalf("PhysicalABIV1 descriptors = %d, want Parent + Child:\n%s", got, ir) + } + for _, forbidden := range []string{"@malloc", "@free(", "stacksave", "stackrestore", "pthread"} { + if strings.Contains(ir, forbidden) { + t.Fatalf("child-await lowering introduced forbidden stack/runtime coupling %q:\n%s", forbidden, ir) + } + } +} + +func TestCoroChildAwaitPhysicalABIV1CoroSplit(t *testing.T) { + prog, pkg := compileCoroChildAwaitPhysicalABI(t, nil) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + runCoroABITestPipeline(t, prog, module) + ir := module.String() + for _, function := range []string{"foo.Parent$coro", "foo.Child$coro"} { + if module.NamedFunction(function).IsNil() { + t.Fatalf("post-split module lost ramp %q:\n%s", function, ir) + } + for _, suffix := range []string{".resume", ".destroy"} { + if module.NamedFunction(function + suffix).IsNil() { + t.Fatalf("CoroSplit did not create %s%s:\n%s", function, suffix, ir) + } + } + } + for _, intrinsic := range []string{ + "llvm.coro.id", "llvm.coro.begin", "llvm.coro.suspend", "llvm.coro.end", + "llvm.coro.resume", "llvm.coro.done", "llvm.coro.destroy", + } { + if hasLLVMCall(ir, intrinsic) { + t.Fatalf("post-split module still calls %s:\n%s", intrinsic, ir) + } + } + parentResume := module.NamedFunction("foo.Parent$coro.resume").String() + if !regexp.MustCompile(`call ptr @"?foo\.Child\$coro"?\(`).MatchString(parentResume) { + t.Fatalf("Parent resume entry lost the static child ramp call:\n%s", parentResume) + } + for _, hook := range []string{coroAwaitPrepareHookV1, coroCompletePrepareHookV1} { + if !strings.Contains(parentResume, "call void @"+hook) { + t.Fatalf("Parent resume entry lost %s:\n%s", hook, parentResume) + } + } + for _, forbidden := range []string{"llvm.coro.resume", "llvm.coro.done", "llvm.coro.destroy"} { + if hasLLVMCall(parentResume, forbidden) { + t.Fatalf("post-split Parent directly calls forbidden %s:\n%s", forbidden, parentResume) + } + } + for _, function := range []string{"foo.Parent$coro", "foo.Child$coro"} { + ramp := module.NamedFunction(function).String() + if !strings.Contains(ramp, "call void @"+coroFramePublishHookV1) { + t.Fatalf("%s ramp lost frame publication:\n%s", function, ramp) + } + destroy := module.NamedFunction(function + ".destroy").String() + if !strings.Contains(destroy, "call void @"+coroFrameFreeHookV1) { + t.Fatalf("%s destroy entry lost task-aware frame free:\n%s", function, destroy) + } + } +} + +func TestCoroChildAwaitPhysicalABIV1Wasm32(t *testing.T) { + llssa.Initialize(llssa.InitAll) + prog, pkg := compileCoroChildAwaitPhysicalABI(t, &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + if got := prog.PointerSize(); got != 4 { + t.Fatalf("wasm pointer size = %d, want 4", got) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify wasm child-await coroutines: %v\n%s", err, module.String()) + } + ir := module.String() + for _, intrinsic := range []string{"size", "align"} { + if !strings.Contains(ir, "@llvm.coro."+intrinsic+".i32") { + t.Fatalf("wasm child-await coroutine uses non-i32 %s intrinsic:\n%s", intrinsic, ir) + } + } + if !regexp.MustCompile( + `@__llgo_coro_frame_descriptor_v1\.[0-9a-f]+ = linkonce_odr unnamed_addr constant \{ i32, i32, i64, i64, i32, i32 \} \{ i32 1, i32 [^,]+, i64 [^,]+, i64 [^,]+, i32 [^,]+, i32 [^}]+ \}`, + ).MatchString(ir) { + t.Fatalf("wasm PhysicalABIV1 descriptor does not use i32 size/alignment fields:\n%s", ir) + } + parentIR := requireCoroPhysicalFunction(t, module, "foo.Parent").String() + assertCoroV1TaskAwareFrameCalls(t, "wasm Parent", parentIR, 32) + if !regexp.MustCompile(`call ptr @llvm\.coro\.promise\(ptr [^,]+, i32 4, i1 false\)`).MatchString(parentIR) { + t.Fatalf("wasm child header lookup does not use wasm32 ABI alignment and from=false:\n%s", parentIR) + } + assertCoroStaticChildAwait(t, parentIR) + runCoroABITestPipeline(t, prog, module) + post := module.String() + for _, function := range []string{"foo.Parent$coro", "foo.Child$coro"} { + for _, suffix := range []string{".resume", ".destroy"} { + if module.NamedFunction(function + suffix).IsNil() { + t.Fatalf("wasm CoroSplit did not create %s%s:\n%s", function, suffix, post) + } + } + } +} + +func TestCoroChildAwaitPhysicalABIV1FailsClosed(t *testing.T) { + prog, ssaPkg, files, universe, plan := prepareCoroChildAwaitPhysicalABI(t, nil) + defer prog.Dispose() + + base := func() *Compilation { + return &Compilation{ + CoroPlan: plan, + EmissionUniverse: universe, + } + } + for _, test := range []struct { + name string + edit func(*Compilation) + want string + }{ + { + name: "child await without physical ABI", + edit: func(c *Compilation) { + c.EnableCoroEntryResolution = true + c.EnableCoroChildAwait = true + }, + want: "requires coroutine physical ABI", + }, + { + name: "physical ABI without entry resolution", + edit: func(c *Compilation) { + c.EnableCoroPhysicalABI = true + c.EnableCoroChildAwait = true + }, + want: "requires coroutine entry resolution", + }, + { + name: "static coroutine call without child await capability", + edit: func(c *Compilation) { + c.EnableCoroEntryResolution = true + c.EnableCoroPhysicalABI = true + }, + // PhysicalABIV0 retains its original leaf-only validation order and + // diagnostic rather than adopting any v1 acceptance behavior. + want: "requires an explicit, isolated yield-only effect", + }, + { + name: "v0 physical identity", + edit: func(c *Compilation) { + enableCoroChildAwaitCompilation(c) + c.CoroABI = coro.PhysicalABIV0 + }, + want: `coroutine compilation coroutine ABI "llgo.coro.physical.v0" does not match "llgo.coro.physical.v1"`, + }, + { + name: "scheduler-none identity", + edit: func(c *Compilation) { + enableCoroChildAwaitCompilation(c) + c.SchedulerABI = coro.SchedulerNoneABIV0 + }, + want: `coroutine compilation scheduler ABI "llgo.coro.scheduler.none.v0" does not match "llgo.coro.scheduler.child-await.v0"`, + }, + } { + t.Run(test.name, func(t *testing.T) { + compilation := base() + test.edit(compilation) + observerCalls := 0 + compilation.CoroPlanObserver = func(*ssa.Package, *coro.SSAPlan) { observerCalls++ } + got, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("preflight result = %v, %v; want error containing %q", got, err, test.want) + } + if got != nil { + t.Fatal("child-await preflight failure returned a partial package") + } + if observerCalls != 0 { + t.Fatalf("observer calls = %d, want pre-codegen rejection", observerCalls) + } + }) + } +} + +func TestCoroExplicitAsyncRootFactoryV1Presplit(t *testing.T) { + prog, pkg := compileCoroChildAwaitPhysicalABI(t, nil) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify explicit async root factory: %v\n%s", err, module.String()) + } + ir := module.String() + parent := requireCoroPhysicalFunction(t, module, "foo.Parent") + child := requireCoroPhysicalFunction(t, module, "foo.Child") + hash, factory := requireSingleCoroRootFactoryV1(t, module) + parentHash := requireCoroFrameDescriptorHash(t, "Parent", parent.String()) + childHash := requireCoroFrameDescriptorHash(t, "Child", child.String()) + if hash != parentHash { + t.Fatalf("root factory hash = %s, want explicit Parent frame hash %s", hash, parentHash) + } + if childHash == parentHash { + t.Fatalf("propagated Child and explicit Parent unexpectedly share ABI hash %s", childHash) + } + if !module.NamedFunction(coroRootFactoryPrefix+childHash).IsNil() || + strings.Contains(ir, coroRootFactoryDescriptorPrefix+childHash) { + t.Fatalf("propagated AsyncDemand Child incorrectly received a root factory/descriptor:\n%s", ir) + } + assertCoroRootFactoryV1Body(t, factory.String()) + assertCoroRootFactoryV1Descriptor(t, ir, hash, parentHash, prog.PointerSize()*8) + assertCoroRootDescriptorLLVMUsed(t, module, hash) + if got := len(regexp.MustCompile(`define ptr @"?`+regexp.QuoteMeta(coroRootFactoryPrefix)+`[0-9a-f]{32}"?\(`).FindAllString(ir, -1)); got != 1 { + t.Fatalf("root factory definitions = %d, want only explicit Parent:\n%s", got, ir) + } + if got := len(regexp.MustCompile(`@`+regexp.QuoteMeta(coroRootFactoryDescriptorPrefix)+`[0-9a-f]{32} =`).FindAllString(ir, -1)); got != 1 { + t.Fatalf("root factory descriptors = %d, want only explicit Parent:\n%s", got, ir) + } +} + +func TestCoroExplicitAsyncRootFactoryV1CoroSplit(t *testing.T) { + prog, pkg := compileCoroChildAwaitPhysicalABI(t, nil) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + hash, _ := requireSingleCoroRootFactoryV1(t, module) + runCoroABITestPipeline(t, prog, module) + ir := module.String() + factoryName := coroRootFactoryPrefix + hash + factory := module.NamedFunction(factoryName) + if factory.IsNil() { + t.Fatalf("CoroSplit lost explicit root factory %q:\n%s", factoryName, ir) + } + assertCoroRootFactoryV1Body(t, factory.String()) + if !module.NamedFunction(factoryName+".resume").IsNil() || + !module.NamedFunction(factoryName+".destroy").IsNil() { + t.Fatalf("non-coroutine root factory was cloned into resume/destroy entries:\n%s", ir) + } + for _, function := range []string{"foo.Parent$coro", "foo.Child$coro"} { + for _, suffix := range []string{".resume", ".destroy"} { + if module.NamedFunction(function + suffix).IsNil() { + t.Fatalf("CoroSplit did not create %s%s:\n%s", function, suffix, ir) + } + } + } + if !strings.Contains(ir, coroRootFactoryDescriptorPrefix+hash) { + t.Fatalf("CoroSplit lost explicit root descriptor %q:\n%s", coroRootFactoryDescriptorPrefix+hash, ir) + } + assertCoroRootDescriptorLLVMUsed(t, module, hash) + runCoroABIGlobalDCE(t, prog, module) + if module.NamedGlobal(coroRootFactoryDescriptorPrefix+hash).IsNil() || + module.NamedFunction(factoryName).IsNil() { + t.Fatalf("GlobalDCE lost linker-retained root descriptor/factory:\n%s", module.String()) + } + assertCoroRootDescriptorLLVMUsed(t, module, hash) + assertCoroRootObjectRetained(t, prog, module, hash) +} + +func TestCoroExplicitAsyncRootFactoryV1Wasm32(t *testing.T) { + llssa.Initialize(llssa.InitAll) + prog, pkg := compileCoroChildAwaitPhysicalABI(t, &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + if got := prog.PointerSize(); got != 4 { + t.Fatalf("wasm pointer size = %d, want 4", got) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify wasm explicit async root factory: %v\n%s", err, module.String()) + } + ir := module.String() + hash, factory := requireSingleCoroRootFactoryV1(t, module) + parentHash := requireCoroFrameDescriptorHash(t, "wasm Parent", requireCoroPhysicalFunction(t, module, "foo.Parent").String()) + childHash := requireCoroFrameDescriptorHash(t, "wasm Child", requireCoroPhysicalFunction(t, module, "foo.Child").String()) + if hash != parentHash || childHash == hash { + t.Fatalf("wasm root hashes: factory=%s Parent=%s Child=%s", hash, parentHash, childHash) + } + assertCoroRootFactoryV1Body(t, factory.String()) + assertCoroRootFactoryV1Descriptor(t, ir, hash, parentHash, 32) + assertCoroRootDescriptorLLVMUsed(t, module, hash) + if !module.NamedFunction(coroRootFactoryPrefix+childHash).IsNil() || + strings.Contains(ir, coroRootFactoryDescriptorPrefix+childHash) { + t.Fatalf("wasm propagated Child incorrectly received a root factory/descriptor:\n%s", ir) + } + + runCoroABITestPipeline(t, prog, module) + post := module.String() + factoryName := coroRootFactoryPrefix + hash + postFactory := module.NamedFunction(factoryName) + if postFactory.IsNil() { + t.Fatalf("wasm CoroSplit lost root factory %q:\n%s", factoryName, post) + } + assertCoroRootFactoryV1Body(t, postFactory.String()) + if !module.NamedFunction(factoryName+".resume").IsNil() || + !module.NamedFunction(factoryName+".destroy").IsNil() { + t.Fatalf("wasm root factory was incorrectly coroutine-split:\n%s", post) + } + assertCoroRootDescriptorLLVMUsed(t, module, hash) + for _, function := range []string{"foo.Parent$coro", "foo.Child$coro"} { + for _, suffix := range []string{".resume", ".destroy"} { + if module.NamedFunction(function + suffix).IsNil() { + t.Fatalf("wasm CoroSplit did not create %s%s:\n%s", function, suffix, post) + } + } + } +} + +func TestCoroExplicitRootFactoryV1FailsClosed(t *testing.T) { + const childAwaitSource = `package foo +func Child(first uint8, second uint32) uint32 { return uint32(first) + second } +func Parent(first uint8, second uint32) uint32 { return Child(first, second) + 1 } +` + for _, test := range []struct { + name string + source string + roots []coroRootFactoryTestRoot + yieldOnly []string + want string + }{ + { + name: "sync explicit coroutine root", + source: childAwaitSource, + roots: []coroRootFactoryTestRoot{{name: "Parent", demand: coro.SyncDemand}}, + yieldOnly: []string{"Child"}, + want: "requires explicit async-only demand, got sync", + }, + { + name: "both-demand explicit coroutine root", + source: childAwaitSource, + roots: []coroRootFactoryTestRoot{ + {name: "Parent", demand: coro.SyncDemand}, + {name: "Parent", demand: coro.AsyncDemand}, + }, + yieldOnly: []string{"Child"}, + want: "requires explicit async-only demand, got both", + }, + { + name: "plain explicit async root", + source: `package foo; func Plain(first uint8, second uint32) uint32 { return uint32(first) + second }`, + roots: []coroRootFactoryTestRoot{{name: "Plain", demand: coro.AsyncDemand}}, + want: "requires an async-only defined direct coroutine", + }, + } { + t.Run(test.name, func(t *testing.T) { + prog, ssaPkg, files, universe, plan := prepareCoroRootFactoryTestPlan( + t, test.source, test.roots, test.yieldOnly, + ) + defer prog.Dispose() + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroChildAwaitCompilation(compilation) + observerCalls := 0 + compilation.CoroPlanObserver = func(*ssa.Package, *coro.SSAPlan) { observerCalls++ } + got, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("preflight result = %v, %v; want error containing %q", got, err, test.want) + } + if got != nil { + t.Fatal("root-factory preflight failure returned a partial package") + } + if observerCalls != 0 { + t.Fatalf("observer calls = %d, want pre-codegen rejection", observerCalls) + } + }) + } +} + func TestCoroLeafPhysicalABIPreflightRejectsUnsupported(t *testing.T) { for _, test := range []struct { name string @@ -465,3 +905,463 @@ func compileCoroLeafPhysicalABIPackage(t *testing.T, target *llssa.Target, ssaPk } return prog, pkg } + +func compileCoroChildAwaitPhysicalABI(t *testing.T, target *llssa.Target) (llssa.Program, llssa.Package) { + t.Helper() + prog, ssaPkg, files, universe, plan := prepareCoroChildAwaitPhysicalABI(t, target) + compilation := &Compilation{ + CoroPlan: plan, + EmissionUniverse: universe, + } + enableCoroChildAwaitCompilation(compilation) + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + return prog, pkg +} + +func prepareCoroChildAwaitPhysicalABI(t *testing.T, target *llssa.Target) ( + llssa.Program, *ssa.Package, []*ast.File, *EmissionUniverse, *coro.SSAPlan, +) { + t.Helper() + const source = `package foo +func Child(first uint8, second uint32) uint32 { return uint32(first) + second } +func Parent(first uint8, second uint32) uint32 { return Child(first, second) + 1 } +` + ssaPkg, _, files := buildGoSSAPkg(t, source) + var prog llssa.Program + if target == nil { + prog = newLLSSAProg(t) + } else { + prog = newLLSSAProgForTarget(t, target) + } + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerChildAwaitABIV0 + functionIDs.ArchiveReady = true + parent, child := ssaPkg.Func("Parent"), ssaPkg.Func("Child") + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: parent, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == child { + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + }) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + for name, fn := range map[string]*ssa.Function{"Parent": parent, "Child": child} { + function, ok := plan.FunctionPlan(fn) + if !ok || function.Primary != coro.PrimaryCoroutine || function.FuncRep != coro.DirectCoro || function.Demand != coro.AsyncDemand { + prog.Dispose() + t.Fatalf("%s child-await plan = %+v, present=%t; want async-only direct coroutine", name, function, ok) + } + } + return prog, ssaPkg, files, universe, plan +} + +func enableCoroChildAwaitCompilation(compilation *Compilation) { + compilation.EnableCoroEntryResolution = true + compilation.EnableCoroPhysicalABI = true + compilation.EnableCoroChildAwait = true + compilation.CoroABI = coro.PhysicalABIV1 + compilation.SchedulerABI = coro.SchedulerChildAwaitABIV0 + compilation.PanicABI = coro.PanicLegacyABIV0 + compilation.FuncRepABI = coro.FuncRepABIV0 +} + +func requireCoroPhysicalFunction(t *testing.T, module llvm.Module, sourceName string) llvm.Value { + t.Helper() + if legacy := module.NamedFunction(sourceName); !legacy.IsNil() { + t.Fatalf("coroutine retained legacy source ABI symbol %q:\n%s", sourceName, module.String()) + } + physical := module.NamedFunction(sourceName + "$coro") + if physical.IsNil() { + t.Fatalf("coroutine physical symbol %q is absent:\n%s", sourceName+"$coro", module.String()) + } + return physical +} + +func assertCoroV1TaskAwareFrameCalls(t *testing.T, name, body string, pointerBits int) { + t.Helper() + integer := "i" + strconv.Itoa(pointerBits) + alloc := regexp.MustCompile( + `call ptr @` + regexp.QuoteMeta(coroFrameAllocHookV1) + + `\(ptr [^,]+, ` + integer + ` [^,]+, ` + integer + ` [^,]+, ptr @__llgo_coro_frame_descriptor_v1\.[0-9a-f]+\)`, + ) + if !alloc.MatchString(body) { + t.Fatalf("%s lacks task-aware v1 frame allocation:\n%s", name, body) + } + free := regexp.MustCompile( + `call void @` + regexp.QuoteMeta(coroFrameFreeHookV1) + + `\(ptr [^,]+, ptr [^,]+, ` + integer + ` [^,]+, ` + integer + ` [^,]+, ptr @__llgo_coro_frame_descriptor_v1\.[0-9a-f]+\)`, + ) + if !free.MatchString(body) { + t.Fatalf("%s lacks task-aware v1 frame free:\n%s", name, body) + } +} + +func assertCoroV0HeaderStateZero(t *testing.T, body string) { + t.Helper() + for _, field := range []struct { + index int + type_ string + name string + }{ + {index: coroHeaderSuspendReason, type_: "i16", name: "suspend reason"}, + {index: coroHeaderLifecycle, type_: "i16", name: "lifecycle"}, + {index: coroHeaderStateID, type_: "i32", name: "state ID"}, + } { + addresses := regexp.MustCompile( + `(?m)^\s*(%[-a-zA-Z$._0-9]+) = getelementptr[^\n{]* \{ ptr, ptr, ptr, ptr, ptr, i16, i16, i32, i32 \}, ptr [^,]+, i32 0, i32 `+strconv.Itoa(field.index)+`\s*$`, + ).FindAllStringSubmatch(body, -1) + if len(addresses) == 0 { + t.Fatalf("v0 coroutine has no header %s store:\n%s", field.name, body) + } + for _, address := range addresses { + store := regexp.MustCompile( + `(?m)^\s*store ` + field.type_ + ` ([^,]+), ptr ` + regexp.QuoteMeta(address[1]) + `(?:,|\s*$)`, + ).FindStringSubmatch(body) + if len(store) != 2 { + t.Fatalf("v0 coroutine header %s address %s has no store:\n%s", field.name, address[1], body) + } + if store[1] != "0" { + t.Fatalf("v0 coroutine header %s = %s, want reserved zero state:\n%s", field.name, store[1], body) + } + } + } +} + +func assertCoroV1InitialPublish(t *testing.T, name, body string) { + t.Helper() + begin := strings.Index(body, "call ptr @llvm.coro.begin") + publish := strings.Index(body, "call void @"+coroFramePublishHookV1) + suspend := strings.Index(body, "call i8 @llvm.coro.suspend") + if begin < 0 || publish < 0 || suspend < 0 || !(begin < publish && publish < suspend) { + t.Fatalf("%s does not publish its v1 frame after coro.begin and before initial suspend:\n%s", name, body) + } + call := regexp.MustCompile( + `call void @` + regexp.QuoteMeta(coroFramePublishHookV1) + `\(ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^)]+\)`, + ) + if !call.MatchString(body) { + t.Fatalf("%s frame publication lacks (task, handle, header, storage):\n%s", name, body) + } +} + +func assertCoroV1Completion(t *testing.T, name, body string) { + t.Helper() + complete := strings.Index(body, "call void @"+coroCompletePrepareHookV1) + finalSuspend := strings.Index(body, "@llvm.coro.suspend(token none, i1 true)") + if complete < 0 || finalSuspend < 0 || complete >= finalSuspend { + t.Fatalf("%s does not prepare completion before final suspend:\n%s", name, body) + } + segment := body[:complete] + state := regexp.MustCompile(`(?s)store i16 2,.*store i16 4,.*store i32 [1-9][0-9]*,`) + if !state.MatchString(segment) { + t.Fatalf("%s does not publish final reason/lifecycle/stateID before completion preparation:\n%s", name, body) + } +} + +func assertCoroStaticChildAwait(t *testing.T, parent string) { + t.Helper() + childCall := regexp.MustCompile(`call ptr @"?foo\.Child\$coro"?\(`).FindStringIndex(parent) + publish := strings.Index(parent, "call void @"+coroFramePublishHookV1) + initialSuspend := strings.Index(parent, "call i8 @llvm.coro.suspend") + await := strings.Index(parent, "call void @"+coroAwaitPrepareHookV1) + if childCall == nil || publish < 0 || initialSuspend < 0 || await < 0 || + !(publish < initialSuspend && initialSuspend < childCall[0] && childCall[0] < await) { + t.Fatalf("Parent hook order is not frame_publish -> initial suspend -> Child -> await_prepare:\n%s", parent) + } + prefix := parent[childCall[0]:await] + promiseResult := regexp.MustCompile(`(%[-a-zA-Z$._0-9]+) = call ptr @llvm\.coro\.promise\(ptr [^,]+, i32 [0-9]+, i1 false\)`).FindStringSubmatch(prefix) + parentHandle := regexp.MustCompile(`(%[-a-zA-Z$._0-9]+) = call ptr @llvm\.coro\.begin`).FindStringSubmatch(parent) + if len(promiseResult) != 2 || len(parentHandle) != 2 { + t.Fatalf("Parent child-await lacks named child promise or parent handle:\n%s", parent) + } + parentLink := regexp.MustCompile( + `(?s)getelementptr [^\n]+, ptr ` + regexp.QuoteMeta(promiseResult[1]) + + `, i32 0, i32 1\s+store ptr ` + regexp.QuoteMeta(parentHandle[1]) + `,`, + ) + if !parentLink.MatchString(prefix) { + t.Fatalf("Parent does not store its handle into child.parent before handoff:\n%s", prefix) + } + state := regexp.MustCompile(`(?s)store i16 1,.*store i16 3,.*store i32 1,`) + if !state.MatchString(prefix) { + t.Fatalf("Parent does not publish Call/Suspended/stateID=1 before await_prepare:\n%s", prefix) + } + awaitSuspend := strings.Index(parent[await:], "call i8 @llvm.coro.suspend") + if awaitSuspend < 0 { + t.Fatalf("Parent does not suspend after await_prepare:\n%s", parent) + } + awaitSuspend += await + complete := strings.Index(parent[awaitSuspend:], "call void @"+coroCompletePrepareHookV1) + if complete < 0 { + t.Fatalf("Parent does not complete after its await resume:\n%s", parent) + } + complete += awaitSuspend + completionState := regexp.MustCompile(`(?s)store i16 2,.*store i16 4,.*store i32 2,`) + if !completionState.MatchString(parent[awaitSuspend:complete]) { + t.Fatalf("Parent does not publish FrameComplete/FinalSuspended/stateID=2 after await:\n%s", parent[awaitSuspend:complete]) + } +} + +func runCoroABITestPipeline(t *testing.T, prog llssa.Program, module llvm.Module) { + t.Helper() + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify before CoroSplit: %v\n%s", err, module.String()) + } + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + options.SetVerifyEach(true) + const pipeline = "coro-early,cgscc(coro-split),coro-cleanup" + if err := module.RunPasses(pipeline, prog.TargetMachine(), options); err != nil { + t.Fatalf("run %s: %v\n%s", pipeline, err, module.String()) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify after CoroSplit: %v\n%s", err, module.String()) + } +} + +func runCoroABIGlobalDCE(t *testing.T, prog llssa.Program, module llvm.Module) { + t.Helper() + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + options.SetVerifyEach(true) + if err := module.RunPasses("globaldce", prog.TargetMachine(), options); err != nil { + t.Fatalf("run globaldce: %v\n%s", err, module.String()) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify after globaldce: %v\n%s", err, module.String()) + } +} + +type coroRootFactoryTestRoot struct { + name string + demand coro.Demand +} + +func prepareCoroRootFactoryTestPlan( + t *testing.T, + source string, + testRoots []coroRootFactoryTestRoot, + yieldOnly []string, +) (llssa.Program, *ssa.Package, []*ast.File, *EmissionUniverse, *coro.SSAPlan) { + t.Helper() + ssaPkg, _, files := buildGoSSAPkg(t, source) + prog := newLLSSAProg(t) + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerChildAwaitABIV0 + functionIDs.ArchiveReady = true + roots := make(coro.Roots, len(testRoots)) + for i, root := range testRoots { + fn := ssaPkg.Func(root.name) + if fn == nil { + prog.Dispose() + t.Fatalf("test root %q is absent", root.name) + } + roots[i] = coro.Root{Function: fn, Demand: root.demand} + } + yieldSet := make(map[*ssa.Function]bool, len(yieldOnly)) + for _, name := range yieldOnly { + fn := ssaPkg.Func(name) + if fn == nil { + prog.Dispose() + t.Fatalf("yield-only function %q is absent", name) + } + yieldSet[fn] = true + } + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, roots, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if yieldSet[fn] { + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + }) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + return prog, ssaPkg, files, universe, plan +} + +func requireSingleCoroRootFactoryV1(t *testing.T, module llvm.Module) (string, llvm.Value) { + t.Helper() + ir := module.String() + pattern := regexp.MustCompile( + `(?m)^define ptr @"?` + regexp.QuoteMeta(coroRootFactoryPrefix) + `([0-9a-f]{32})"?\(ptr [^,]+, ptr [^,]+, ptr [^)]+\)`, + ) + matches := pattern.FindAllStringSubmatch(ir, -1) + if len(matches) != 1 { + t.Fatalf("root factory definitions = %d, want exactly one explicit factory:\n%s", len(matches), ir) + } + hash := matches[0][1] + factory := module.NamedFunction(coroRootFactoryPrefix + hash) + if factory.IsNil() { + t.Fatalf("root factory %q is absent despite its definition:\n%s", coroRootFactoryPrefix+hash, ir) + } + return hash, factory +} + +func requireCoroFrameDescriptorHash(t *testing.T, name, body string) string { + t.Helper() + matches := regexp.MustCompile( + `@`+regexp.QuoteMeta(coroDescriptorPrefixV1)+`([0-9a-f]{32})`, + ).FindAllStringSubmatch(body, -1) + if len(matches) == 0 { + t.Fatalf("%s has no PhysicalABIV1 frame descriptor:\n%s", name, body) + } + hash := matches[0][1] + for _, match := range matches[1:] { + if match[1] != hash { + t.Fatalf("%s references multiple frame descriptor hashes %s and %s:\n%s", name, hash, match[1], body) + } + } + return hash +} + +func assertCoroRootFactoryV1Body(t *testing.T, body string) { + t.Helper() + if !regexp.MustCompile( + `define ptr @"?` + regexp.QuoteMeta(coroRootFactoryPrefix) + `[0-9a-f]{32}"?\(ptr %0, ptr %1, ptr %2\)`, + ).MatchString(body) { + t.Fatalf("root factory does not use (g, out, startup) -> handle ABI:\n%s", body) + } + loads := regexp.MustCompile( + `(?s)(%[-a-zA-Z$._0-9]+) = getelementptr inbounds[^\n{]*\{ i8, i32 \}, ptr %2, i32 0, i32 0\s+` + + `(%[-a-zA-Z$._0-9]+) = load i8, ptr [^,]+, align 1.*?` + + `(%[-a-zA-Z$._0-9]+) = getelementptr inbounds[^\n{]*\{ i8, i32 \}, ptr %2, i32 0, i32 1\s+` + + `(%[-a-zA-Z$._0-9]+) = load i32, ptr [^,]+, align 4`, + ).FindStringSubmatch(body) + if len(loads) != 5 { + t.Fatalf("root factory does not load typed {uint8,uint32} startup arguments:\n%s", body) + } + call := regexp.MustCompile( + `call ptr @"?foo\.Parent\$coro"?\(ptr %0, ptr %1, i8 ` + regexp.QuoteMeta(loads[2]) + + `, i32 ` + regexp.QuoteMeta(loads[4]) + `\)`, + ) + if !call.MatchString(body) { + t.Fatalf("root factory does not pass (g, out, typed startup args) to Parent$coro exactly:\n%s", body) + } + if got := len(regexp.MustCompile(`\bcall\b`).FindAllString(body, -1)); got != 1 { + t.Fatalf("root factory calls = %d, want only Parent$coro:\n%s", got, body) + } + for _, forbidden := range []string{ + "llvm.coro.", "coro.suspend", ".resume", ".destroy", "clone", + `@"foo.Parent"(`, "@foo.Parent(", `@"foo.Child$coro"(`, "@foo.Child$coro(", + } { + if strings.Contains(body, forbidden) { + t.Fatalf("root factory contains forbidden coroutine/clone/plain-primary marker %q:\n%s", forbidden, body) + } + } +} + +func assertCoroRootFactoryV1Descriptor(t *testing.T, ir, hash, parentHash string, pointerBits int) { + t.Helper() + if hash != parentHash { + t.Fatalf("root factory hash %s does not match Parent physical ABI hash %s", hash, parentHash) + } + uintptrType := "i" + strconv.Itoa(pointerBits) + rootPattern := regexp.MustCompile( + `@` + regexp.QuoteMeta(coroRootFactoryDescriptorPrefix+hash) + + ` = linkonce_odr unnamed_addr constant \{ i32, i32, i64, i64, ptr, ` + uintptrType + `, ` + uintptrType + `, ` + uintptrType + `, ` + uintptrType + ` \} ` + + `\{ i32 1, i32 0, i64 ([^,]+), i64 ([^,]+), ptr @"?` + regexp.QuoteMeta(coroRootFactoryPrefix+hash) + `"?, ` + + uintptrType + ` 8, ` + uintptrType + ` 4, ` + uintptrType + ` 4, ` + uintptrType + ` 4 \}`, + ) + root := rootPattern.FindStringSubmatch(ir) + if len(root) != 3 { + t.Fatalf("root descriptor lacks v1/hash/factory/startup(8,4)/result(4,4) target layout:\n%s", ir) + } + framePattern := regexp.MustCompile( + `@` + regexp.QuoteMeta(coroDescriptorPrefixV1+parentHash) + + ` = linkonce_odr unnamed_addr constant \{ [^}]+ \} \{ i32 1, i32 0, i64 ([^,]+), i64 ([^,]+),`, + ) + frame := framePattern.FindStringSubmatch(ir) + if len(frame) != 3 { + t.Fatalf("Parent frame descriptor %q is absent:\n%s", coroDescriptorPrefixV1+parentHash, ir) + } + if root[1] != frame[1] || root[2] != frame[2] { + t.Fatalf("root descriptor hash words = (%s,%s), Parent frame hash words = (%s,%s)", root[1], root[2], frame[1], frame[2]) + } +} + +func assertCoroRootDescriptorLLVMUsed(t *testing.T, module llvm.Module, hash string) { + t.Helper() + used := module.NamedGlobal("llvm.used") + if used.IsNil() { + t.Fatalf("root descriptor is not protected from final-link dead stripping by llvm.used:\n%s", module.String()) + } + if got := used.Linkage(); got != llvm.AppendingLinkage { + t.Fatalf("llvm.used linkage = %v, want appending", got) + } + if got := used.Section(); got != "llvm.metadata" { + t.Fatalf("llvm.used section = %q, want llvm.metadata", got) + } + name := coroRootFactoryDescriptorPrefix + hash + var usedLine string + for _, line := range strings.Split(module.String(), "\n") { + if strings.HasPrefix(line, "@llvm.used =") { + usedLine = line + break + } + } + if usedLine == "" || (!strings.Contains(usedLine, "ptr @"+name) && + !strings.Contains(usedLine, `ptr @"`+name+`"`)) { + t.Fatalf("llvm.used does not retain root descriptor %q: %s", name, usedLine) + } +} + +func assertCoroRootObjectRetained(t *testing.T, prog llssa.Program, module llvm.Module, hash string) { + t.Helper() + object, err := prog.TargetMachine().EmitToMemoryBuffer(module, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit root-retention object: %v\n%s", err, module.String()) + } + defer object.Dispose() + for _, name := range []string{ + coroRootFactoryDescriptorPrefix + hash, + coroRootFactoryPrefix + hash, + } { + if !bytes.Contains(object.Bytes(), []byte(name)) { + t.Fatalf("object symbol table lost linker-retained root symbol %q", name) + } + } +} + +func hasLLVMCall(ir, intrinsic string) bool { + return regexp.MustCompile(`call [^\n]*@` + regexp.QuoteMeta(intrinsic) + `\b`).MatchString(ir) +} diff --git a/cl/coro_await.go b/cl/coro_await.go new file mode 100644 index 0000000000..ad41797d7e --- /dev/null +++ b/cl/coro_await.go @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "fmt" + + "github.com/goplus/llgo/internal/coro" + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +// resolveCoroStaticAwait proves the exact subset implemented by the physical +// child-await lowering. The returned function is the canonical target recorded +// by the whole-program plan, not an identity inferred from an SSA display name. +func resolveCoroStaticAwait(plan *coro.SSAPlan, caller coro.FunctionPlan, call ssa.CallInstruction) (*ssa.Function, coro.FunctionPlan, error) { + if plan == nil || call == nil || call.Common() == nil { + return nil, coro.FunctionPlan{}, fmt.Errorf("requires a compilation CallPlan") + } + common := call.Common() + if common.IsInvoke() || common.StaticCallee() == nil { + return nil, coro.FunctionPlan{}, fmt.Errorf("requires a static non-invoke call") + } + callPlan, ok := plan.CallPlan(call) + if !ok { + return nil, coro.FunctionPlan{}, fmt.Errorf("call has no compilation CallPlan") + } + if callPlan.Kind != coro.CallDirect || callPlan.Rep != coro.DirectCoro || callPlan.Open || callPlan.MayBeNil || len(callPlan.Targets) != 1 { + return nil, coro.FunctionPlan{}, fmt.Errorf( + "requires one closed non-nil direct coroutine target, got kind=%v representation=%s open=%t may-be-nil=%t targets=%d", + callPlan.Kind, callPlan.Rep, callPlan.Open, callPlan.MayBeNil, len(callPlan.Targets), + ) + } + target, ok := plan.Function(callPlan.Targets[0]) + if !ok || target == nil { + return nil, coro.FunctionPlan{}, fmt.Errorf("direct coroutine target %q is absent from the compilation plan", callPlan.Targets[0]) + } + targetPlan, ok := plan.FunctionPlan(target) + if !ok || targetPlan.ID != callPlan.Targets[0] { + return nil, coro.FunctionPlan{}, fmt.Errorf("direct coroutine target %q has no canonical function plan", callPlan.Targets[0]) + } + if caller.Primary != coro.PrimaryCoroutine { + return nil, coro.FunctionPlan{}, fmt.Errorf("caller primary is %s, want coroutine", caller.Primary) + } + if targetPlan.External != coro.Defined || targetPlan.Primary != coro.PrimaryCoroutine || targetPlan.FuncRep != coro.DirectCoro || targetPlan.Demand != coro.AsyncDemand { + return nil, coro.FunctionPlan{}, fmt.Errorf( + "target %q is not an async-only defined direct coroutine (external=%s primary=%s representation=%s demand=%s)", + targetPlan.ID, targetPlan.External, targetPlan.Primary, targetPlan.FuncRep, targetPlan.Demand, + ) + } + return target, targetPlan, nil +} + +// tryCompileCoroStaticAwait lowers a source-style synchronous call into one +// stackless child handoff. It creates the child only to its initial suspend; +// this function never resumes or destroys a handle. Those operations belong to +// the scheduler after the parent's resume episode has returned. +func (p *context) tryCompileCoroStaticAwait(b llssa.Builder, call *ssa.Call) (llssa.Expr, bool) { + if p.currentCoro == nil || p.compilation == nil || p.compilation.CoroPlan == nil || !p.compilation.EnableCoroChildAwait || call == nil { + return llssa.Nil, false + } + callPlan, ok := p.compilation.CoroPlan.CallPlan(call) + if !ok || callPlan.Rep != coro.DirectCoro { + return llssa.Nil, false + } + callerPlan, ok := p.compilation.CoroPlan.FunctionPlan(p.goFn) + if !ok { + panic("coroutine child await: current function has no compilation plan") + } + callee, _, err := resolveCoroStaticAwait(p.compilation.CoroPlan, callerPlan, call) + if err != nil { + panic(fmt.Sprintf("coroutine child await: function %q: %v", callerPlan.ID, err)) + } + + p.recordCallerLocationForCall(b, &call.Call) + p.emitPCLineLabel(b, call.Pos()) + + // Preserve Go's left-to-right argument evaluation before publishing any + // child or parent scheduler state. + args := p.compileValues(b, call.Call.Args, p.funcKind(call.Call.Value)) + entry := p.mustFunctionSymbol(callee) + if p.emissionUniverse == nil { + panic("coroutine child await requires a prepared emission universe") + } + sourceSig, err := p.emissionUniverse.coroPhysicalSourceSignature(callee) + if err != nil { + panic(fmt.Sprintf("coroutine child await: derive target %q ABI: %v", entry.plan.ID, err)) + } + abi := newCoroPhysicalABI(p, entry, sourceSig) + childFn, _, kind := p.compileFunction(callee) + if kind != goFunc { + panic(fmt.Sprintf("coroutine child await: target %q did not resolve to a Go entry", entry.plan.ID)) + } + + resultType := p.prog.Type(abi.resultSlotType, llssa.InGo) + resultSlot := b.AllocaT(resultType) + physicalArgs := make([]llssa.Expr, 0, len(args)+2) + physicalArgs = append(physicalArgs, + p.currentCoro.task, + b.Convert(p.prog.VoidPtr(), resultSlot), + ) + physicalArgs = append(physicalArgs, args...) + child := b.Call(childFn.Expr, physicalArgs...) + childHeader := b.CoroPromise(child, coroHeaderType(p.prog)) + b.Store(b.FieldAddr(childHeader, coroHeaderParent), p.currentCoro.coro.Handle()) + p.currentCoro.suspendForChild(b) + + if p.currentCoro.abi.awaitPrepareHook == "" { + panic("coroutine child await has no scheduler handoff hook") + } + publish := p.pkg.NewFunc(p.currentCoro.abi.awaitPrepareHook, coroAwaitPrepareSignature(), llssa.InC) + b.Call(publish.Expr, p.currentCoro.task, p.currentCoro.coro.Handle(), child) + p.currentCoro.coro.Suspend() + p.currentCoro.activate(b) + + if abi.resultCount == 0 { + return llssa.Nil, true + } + return b.Load(b.FieldAddr(resultSlot, 0)), true +} diff --git a/cl/coro_entry.go b/cl/coro_entry.go index 237f12503b..5297aca159 100644 --- a/cl/coro_entry.go +++ b/cl/coro_entry.go @@ -30,13 +30,15 @@ const coroPrimarySuffix = "$coro" // Primary selects the source body; FuncRep only describes escaped function // values and never authorizes a second body. type plannedFunctionSymbol struct { - function *ssa.Function - pkgTypes *types.Package - name string - ftype int - plan coro.FunctionPlan - planned bool - physical bool + function *ssa.Function + pkgTypes *types.Package + name string + ftype int + plan coro.FunctionPlan + planned bool + physical bool + childAwait bool + coroPlan *coro.SSAPlan } // resolveFunctionSymbol is shared by function definitions and declarations so @@ -79,6 +81,8 @@ func (p *context) resolveFunctionSymbol(fn *ssa.Function) (plannedFunctionSymbol entry.plan = plan entry.planned = true entry.physical = p.compilation.EnableCoroPhysicalABI + entry.childAwait = p.compilation.EnableCoroChildAwait + entry.coroPlan = p.compilation.CoroPlan if err := validatePlannedFunction(fn, plan); err != nil { return entry, err } @@ -125,7 +129,7 @@ func (e plannedFunctionSymbol) checkSupported() error { if !e.physical { return fmt.Errorf("coroutine primary %q requires coroutine physical ABI lowering", e.plan.ID) } - return validateCoroLeafPhysicalABI(e.function, e.plan) + return validateCoroPhysicalABI(e.function, e.plan, e.coroPlan, e.childAwait) } if e.plan.Primary == coro.PrimaryExternal && e.plan.FuncRep == coro.DirectCoro { return fmt.Errorf("external coroutine primary %q requires coroutine physical ABI lowering", e.plan.ID) @@ -144,6 +148,9 @@ func (c *Compilation) preflightCoroPlan() error { if c.EnableCoroPhysicalABI && !c.EnableCoroEntryResolution { return fmt.Errorf("coroutine physical ABI requires coroutine entry resolution") } + if c.EnableCoroChildAwait && !c.EnableCoroPhysicalABI { + return fmt.Errorf("coroutine child await requires coroutine physical ABI") + } if !c.EnableCoroEntryResolution { return nil } @@ -164,16 +171,24 @@ func (c *Compilation) preflightCoroPlan() error { c.coroPreflightErr = err return } + if c.EnableCoroChildAwait { + if err := validateCoroRootFactories(c.CoroPlan); err != nil { + c.coroPreflightErr = err + return + } + } for _, function := range c.CoroPlan.Functions() { if err := validatePlannedFunction(function.Function, function.Plan); err != nil { c.coroPreflightErr = err return } entry := plannedFunctionSymbol{ - function: function.Function, - plan: function.Plan, - planned: true, - physical: c.EnableCoroPhysicalABI, + function: function.Function, + plan: function.Plan, + planned: true, + physical: c.EnableCoroPhysicalABI, + childAwait: c.EnableCoroChildAwait, + coroPlan: c.CoroPlan, } if err := entry.checkSupported(); err != nil { c.coroPreflightErr = err @@ -191,7 +206,7 @@ func (c *Compilation) preflightCoroPlan() error { } } if c.EnableCoroPhysicalABI { - c.coroPreflightErr = validateCoroPhysicalConsumers(c.CoroPlan) + c.coroPreflightErr = validateCoroPhysicalConsumers(c.CoroPlan, c.EnableCoroChildAwait) } }) return c.coroPreflightErr diff --git a/cl/coro_root.go b/cl/coro_root.go new file mode 100644 index 0000000000..a61750090a --- /dev/null +++ b/cl/coro_root.go @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "encoding/hex" + "fmt" + "go/token" + "go/types" + + "github.com/goplus/llgo/internal/coro" + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +const ( + coroRootFactoryPrefix = "__llgo_coro_root_factory_v1." + coroRootFactoryDescriptorPrefix = "__llgo_coro_root_factory_descriptor_v1." +) + +func coroRootFactorySignature() *types.Signature { + params := types.NewTuple( + types.NewParam(token.NoPos, nil, "g", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "out", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "startup", types.Typ[types.UnsafePointer]), + ) + results := types.NewTuple(types.NewParam(token.NoPos, nil, "handle", types.Typ[types.UnsafePointer])) + return types.NewSignatureType(nil, nil, nil, params, results, false) +} + +func explicitCoroRoot(plan *coro.SSAPlan, fn *ssa.Function) (coro.SSARootPlan, bool) { + if plan == nil || fn == nil { + return coro.SSARootPlan{}, false + } + for _, root := range plan.Roots() { + if root.Function == fn { + return root, true + } + } + return coro.SSARootPlan{}, false +} + +func validateCoroRootFactories(plan *coro.SSAPlan) error { + if plan == nil { + return fmt.Errorf("coroutine root factory requires a compilation CoroPlan") + } + for _, root := range plan.Roots() { + if root.Function == nil { + return fmt.Errorf("coroutine root factory %q has no SSA function", root.ID) + } + if root.Demand != coro.AsyncDemand { + return fmt.Errorf("coroutine root factory %q requires explicit async-only demand, got %s", root.ID, root.Demand) + } + function, ok := plan.FunctionPlan(root.Function) + if !ok || function.ID != root.ID { + return fmt.Errorf("coroutine root factory %q has no canonical function plan", root.ID) + } + if function.External != coro.Defined || function.Primary != coro.PrimaryCoroutine || function.FuncRep != coro.DirectCoro || function.Demand != coro.AsyncDemand { + return fmt.Errorf( + "coroutine root factory %q requires an async-only defined direct coroutine (external=%s primary=%s representation=%s demand=%s)", + root.ID, function.External, function.Primary, function.FuncRep, function.Demand, + ) + } + } + return nil +} + +// emitCoroRootFactory emits a typed, non-coroutine factory only for an +// explicitly declared Async root. The startup/result objects are owned by the +// runtime and outlive this native wrapper invocation; the factory merely loads +// scalar arguments and calls the root's unique coroutine ramp. +func (p *context) emitCoroRootFactory(pkg llssa.Package, entry plannedFunctionSymbol, abi coroPhysicalABI, sourceSig *types.Signature, ramp llssa.Function) { + if p.compilation == nil || p.compilation.CoroPlan == nil { + panic("coroutine root factory requires a compilation CoroPlan") + } + root, ok := explicitCoroRoot(p.compilation.CoroPlan, entry.function) + if !ok { + return + } + if root.Demand != coro.AsyncDemand || entry.plan.ID != root.ID { + panic(fmt.Sprintf("coroutine root factory: unsupported root %q demand %s", root.ID, root.Demand)) + } + + fields := make([]*types.Var, sourceSig.Params().Len()) + for i := range fields { + fields[i] = types.NewField(token.NoPos, nil, fmt.Sprintf("a%d", i), sourceSig.Params().At(i).Type(), false) + } + startupGoType := types.NewStruct(fields, nil) + startupType := p.prog.Type(startupGoType, llssa.InGo) + resultType := p.prog.Type(abi.resultSlotType, llssa.InGo) + hash := hex.EncodeToString(abi.hash[:]) + factoryName := coroRootFactoryPrefix + hash + factory := pkg.FuncOf(factoryName) + if factory == nil { + factory = pkg.NewFunc(factoryName, coroRootFactorySignature(), llssa.InC) + } + if !factory.HasBody() { + b := factory.MakeBody(1) + physicalArgs := make([]llssa.Expr, 0, len(fields)+2) + physicalArgs = append(physicalArgs, factory.PhysicalParam(0), factory.PhysicalParam(1)) + if len(fields) != 0 { + startup := b.Convert(p.prog.Pointer(startupType), factory.PhysicalParam(2)) + for i := range fields { + physicalArgs = append(physicalArgs, b.Load(b.FieldAddr(startup, i))) + } + } + handle := b.Call(ramp.Expr, physicalArgs...) + b.Return(handle) + b.EndBuild() + b.Dispose() + } + pkg.NewCoroRootFactoryDescriptor(coroRootFactoryDescriptorPrefix+hash, llssa.CoroRootFactoryDescriptorOptions{ + Version: coroPhysicalABIVersionV1, + ABIHash: abi.hash, + Factory: factory.Expr, + Startup: startupType, + Result: resultType, + }) +} diff --git a/doc/llvm-coro-runtime-design.md b/doc/llvm-coro-runtime-design.md index c77eb0c91d..e38abcd0e5 100644 --- a/doc/llvm-coro-runtime-design.md +++ b/doc/llvm-coro-runtime-design.md @@ -1778,14 +1778,16 @@ Pure sync library/archive不需要链接scheduler。Executable一旦选择 `-sch 验收：纯 sync chain 只有 `F`；纯 async chain 只有 `F$coro`；动态 escape 才出现 descriptor/adapter；所有 `go` root和可挂起call都以LLVM-coro frame表示。 -当前落地状态（2026-07，实验 ABI v0）： +当前落地状态（2026-07，实验 ABI v0/v1）： - 已完成全程序 SSA 的 Effect、Demand、FuncRep、稳定 FunctionID、精确 emission universe 和单 primary symbol 选择。激活 lowering 使用 archive-ready FunctionID，并以独立 canonical schema 对全部 function/call/value plan、Coro/Scheduler/Panic/FuncRep ABI 及 effective LLVM target/data layout 生成 `CoroPlanDigest`；相同完整计划可安全复用 package build cache，缺失或不匹配的 manifest 继续 fail closed。 -- `cpunion/llvm` 已覆盖 LLVM 19、21、22 的 switched-resume builder/CoroSplit；LLGo 已能为严格受限的 top-level `YieldOnly` 单块 leaf 只生成 `F$coro(Task, ResultSlot, args...) -> CoroHandle`，并生成目标相关 result descriptor 与版本化 frame alloc/free hook。 -- Promise/header 在 `coro.begin` 后、initial suspend 前发布；结果写入 frame 外的 caller-owned slot。pre-/post-CoroSplit 与 wasm32 pointer-width 测试覆盖该时序，且禁止 malloc、pthread、stack-copy fallback。 -- 该 v0 切片故意拒绝 call/await、spawn consumer、循环与抢占、channel/select、defer/panic、closure/method/generic、aggregate/pointer result、Dispatch 和 root/bootstrap；这些路径在 module 创建前 fail closed。因此它只计入 Phase 0 的 ABI/codegen 骨架，尚不表示 scheduler 或标准库兼容已经完成。 +- `cpunion/llvm` 已覆盖 LLVM 19、21、22 的 switched-resume builder/CoroSplit；LLGo 的 v0 路径能为严格受限的 top-level `YieldOnly` 单块 leaf 只生成 `F$coro(Task, ResultSlot, args...) -> CoroHandle`，并生成目标相关 result descriptor 与版本化 frame alloc/free hook。未启用 v1 时，v0 symbol、hook 与 `scheduler.none` 行为保持不变。 +- v1 已加入 closed static `CallDirect + DirectCoro` 的 ordinary child await。父 frame 先按 Go 的从左到右顺序求值参数，在自己的 frame 中保留 result slot，创建只运行到 initial suspend 的 child，写入 parent link，发布 `Call/Suspended/stateID`，调用 `__llgo_coro_await_prepare_v1` 后切断栈。父代码不调用 child 的 `resume`、`done` 或 `destroy`；调度器是后续所有 resume/done/destroy 以及 active-frame 转换的唯一 owner。 +- v1 只为显式 `AsyncDemand` root 生成 `(g, out, startup) -> handle` typed factory 和 linker-discoverable descriptor；仅因调用传播成为 async 的函数不生成第二入口。startup/result 的 size/alignment 使用目标 data layout，native64 与 wasm32 都有 pre-/post-CoroSplit 覆盖，descriptor 由 linker-retained `llvm.used` 保活，不能被 `-dead_strip`/`--gc-sections` 删除。`llvm.used` 不会主动抽取完全无人引用的静态 archive member；当前 descriptor 与会被普通 init/import/main 引用拉入的 package object 同处一员，未来若拆成独立 registry archive，必须增加 anchor 或 whole-archive/force-load 协议。 +- Promise/header 在 `coro.begin` 后、initial suspend 前发布；结果写入 frame 外、由 parent/root runtime 持有的 slot。v1 runtime contract 通过 `__llgo_coro_frame_alloc_v1`、`__llgo_coro_frame_publish_v1`、`__llgo_coro_await_prepare_v1`、`__llgo_coro_complete_prepare_v1`、`__llgo_coro_frame_free_v1` 传递 task/handle/header/storage；这些 hook 必须 NoSuspend、NoCallback，且不得进入用户 Go。`frame_publish_v1` 负责登记 handle/storage 并使 header 的 allocation-base 记录与实际分配一致。 +- 当前 v1 仍只允许线性单块 scalar body，故意拒绝 spawn consumer、循环与抢占、channel/select、defer/panic、closure/method/generic、aggregate/pointer result、Dispatch、普通 main/init bootstrap 及动态 call。所有未实现路径在 module 创建前 fail closed；该切片只完成 child 生命周期与 root ABI，不表示 runtime scheduler 或标准库兼容已经完成。 - 当前 cache digest 只解决同一完整程序计划下的内部 package cache；未知未来 caller 可复用的预编译 archive/标准库仍需 producer summary、canonical boundary Dispatch 和 linker ABI 校验，不能把 cache digest 当作 producer ABI summary。 -- 下一依赖顺序为：加入 ordinary child await 与 root factory，落地单 P scheduler 和 frame registry，再插入并验证 loop/recursion/long-block 抢占 poll。不得用扩大 leaf allowlist 绕过这些生命周期协议。 +- 下一依赖顺序为：实现遵循上述 v1 owner 规则的单 P scheduler、frame registry，以及由 build driver 生成显式 descriptor 数组/anchor 的 root bootstrap（`llvm.used` 负责保留，不承担运行时枚举）；随后扩展 CFG/递归 lowering，并插入和验证 loop/recursion/long-block 抢占 poll。不得用扩大线性 allowlist 绕过这些生命周期协议。 ### Phase 1：单 P deterministic scheduler diff --git a/internal/build/build.go b/internal/build/build.go index 08cae14a9e..985fb0f0f3 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -252,12 +252,17 @@ type Config struct { // leaving it false preserves report-only behavior. Package archives are // reused only when their complete plan/ABI/target fingerprint matches. EnableCoroEntryResolution bool - // EnableCoroPhysicalABI enables the experimental, leaf-only LLVM coroutine - // physical ABI. It requires EnableCoroEntryResolution and remains fail-closed - // for await, dispatch, spawn, defer, and scheduler paths. + // EnableCoroPhysicalABI enables the experimental LLVM coroutine physical ABI. + // It requires EnableCoroEntryResolution and remains leaf-only unless a more + // specific lowering capability is enabled. EnableCoroPhysicalABI bool - CoroPlanBuilder CoroPlanBuilder - CoroPlanObserver CoroPlanObserver + // EnableCoroChildAwait enables the first scheduler handoff slice: a physical + // coroutine may await a statically resolved coroutine child, and an explicit + // async root receives a typed factory descriptor. It requires the physical + // ABI and does not enable a runtime scheduler, spawn, park, or preemption. + EnableCoroChildAwait bool + CoroPlanBuilder CoroPlanBuilder + CoroPlanObserver CoroPlanObserver } type Rewrites map[string]string @@ -680,6 +685,9 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { if ctx.buildConf.EnableCoroPhysicalABI && !ctx.buildConf.EnableCoroEntryResolution { return fmt.Errorf("enable coroutine physical ABI: coroutine entry resolution is required") } + if ctx.buildConf.EnableCoroChildAwait && !ctx.buildConf.EnableCoroPhysicalABI { + return fmt.Errorf("enable coroutine child await: coroutine physical ABI is required") + } builder := ctx.buildConf.CoroPlanBuilder if builder == nil { if ctx.buildConf.EnableCoroEntryResolution { @@ -716,7 +724,7 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { config.CoroABI = activeCoroABIVersion(ctx.buildConf) } if config.SchedulerABI == "" { - config.SchedulerABI = coro.SchedulerNoneABIV0 + config.SchedulerABI = activeCoroSchedulerABIVersion(ctx.buildConf) } config.ArchiveReady = true } @@ -761,6 +769,7 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { CoroPlanObserver: ctx.buildConf.CoroPlanObserver, EnableCoroEntryResolution: ctx.buildConf.EnableCoroEntryResolution, EnableCoroPhysicalABI: ctx.buildConf.EnableCoroPhysicalABI, + EnableCoroChildAwait: ctx.buildConf.EnableCoroChildAwait, CoroPlanDigest: digest, CoroABI: metadata.CoroABI, SchedulerABI: metadata.SchedulerABI, @@ -772,12 +781,22 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { } func activeCoroABIVersion(conf *Config) string { + if conf != nil && conf.EnableCoroChildAwait { + return coro.PhysicalABIV1 + } if conf != nil && conf.EnableCoroPhysicalABI { return coro.PhysicalABIV0 } return coro.EntryResolutionABIV0 } +func activeCoroSchedulerABIVersion(conf *Config) string { + if conf != nil && conf.EnableCoroChildAwait { + return coro.SchedulerChildAwaitABIV0 + } + return coro.SchedulerNoneABIV0 +} + func buildCoroPlanDigestMetadata(ctx *context) (coro.PlanDigestMetadata, error) { if ctx == nil || ctx.buildConf == nil { return coro.PlanDigestMetadata{}, fmt.Errorf("missing build context") @@ -794,7 +813,7 @@ func buildCoroPlanDigestMetadata(ctx *context) (coro.PlanDigestMetadata, error) } return coro.PlanDigestMetadata{ CoroABI: activeCoroABIVersion(ctx.buildConf), - SchedulerABI: coro.SchedulerNoneABIV0, + SchedulerABI: activeCoroSchedulerABIVersion(ctx.buildConf), PanicABI: coro.PanicLegacyABIV0, FuncRepABI: coro.FuncRepABIV0, TargetTriple: target.Triple, diff --git a/internal/build/collect.go b/internal/build/collect.go index 40e9586cc9..3810e15edb 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -375,12 +375,13 @@ func (c *context) canUsePackageCache() bool { metadata := c.coroPlanMetadata return c.clCompilation.EnableCoroEntryResolution && c.clCompilation.EnableCoroPhysicalABI == c.buildConf.EnableCoroPhysicalABI && + c.clCompilation.EnableCoroChildAwait == c.buildConf.EnableCoroChildAwait && c.clCompilation.CoroABI == metadata.CoroABI && c.clCompilation.SchedulerABI == metadata.SchedulerABI && c.clCompilation.PanicABI == metadata.PanicABI && c.clCompilation.FuncRepABI == metadata.FuncRepABI && metadata.CoroABI == activeCoroABIVersion(c.buildConf) && - metadata.SchedulerABI == coro.SchedulerNoneABIV0 && + metadata.SchedulerABI == activeCoroSchedulerABIVersion(c.buildConf) && metadata.PanicABI == coro.PanicLegacyABIV0 && metadata.FuncRepABI == coro.FuncRepABIV0 && metadata.TargetTriple != "" && metadata.PointerBits > 0 && diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index fca04b3ff6..accc430943 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -356,6 +356,29 @@ func g() {} } } +func TestActiveCoroABIVersions(t *testing.T) { + tests := []struct { + name string + config *Config + coroABI string + scheduler string + }{ + {"entry resolution", &Config{}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0}, + {"physical leaf", &Config{EnableCoroPhysicalABI: true}, coro.PhysicalABIV0, coro.SchedulerNoneABIV0}, + {"child await", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true}, coro.PhysicalABIV1, coro.SchedulerChildAwaitABIV0}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := activeCoroABIVersion(test.config); got != test.coroABI { + t.Fatalf("coroutine ABI = %q, want %q", got, test.coroABI) + } + if got := activeCoroSchedulerABIVersion(test.config); got != test.scheduler { + t.Fatalf("scheduler ABI = %q, want %q", got, test.scheduler) + } + }) + } +} + func TestBuildCoroPlanErrors(t *testing.T) { t.Run("builder error", func(t *testing.T) { sentinel := errors.New("sentinel") @@ -426,6 +449,20 @@ func TestBuildCoroPlanErrors(t *testing.T) { } }) + t.Run("child await requires physical ABI", func(t *testing.T) { + ctx := &context{buildConf: &Config{ + EnableCoroEntryResolution: true, + EnableCoroChildAwait: true, + }} + err := buildCoroPlan(ctx) + if err == nil || !strings.Contains(err.Error(), "physical ABI is required") { + t.Fatalf("buildCoroPlan error = %v, want physical-ABI requirement", err) + } + if ctx.coroPlan != nil || ctx.clCompilation != nil { + t.Fatal("invalid child-await configuration installed coroutine compilation state") + } + }) + t.Run("entry resolution requires prepared emission universe", func(t *testing.T) { builderCalls := 0 ctx := &context{buildConf: &Config{ diff --git a/internal/coro/plan_digest.go b/internal/coro/plan_digest.go index 02848bcff3..ce034c8280 100644 --- a/internal/coro/plan_digest.go +++ b/internal/coro/plan_digest.go @@ -31,16 +31,22 @@ import ( // PlanDigestSchema is the independent canonical schema used for archive cache // identity. It is deliberately separate from SummarySchema: summaries remain // diagnostic snapshots, while this document covers every lowering plan site. -const PlanDigestSchema = "llgo.coro.plan-digest.v0" +const PlanDigestSchema = "llgo.coro.plan-digest.v1" // Current experimental ABI identities. Keeping these in the analysis package // gives build, cache, and lowering code one version source of truth. const ( EntryResolutionABIV0 = "llgo.coro.entry-resolution.v0" PhysicalABIV0 = "llgo.coro.physical.v0" + PhysicalABIV1 = "llgo.coro.physical.v1" SchedulerNoneABIV0 = "llgo.coro.scheduler.none.v0" - PanicLegacyABIV0 = "llgo.coro.panic.legacy.v0" - FuncRepABIV0 = "llgo.coro.func-rep.v0" + // SchedulerChildAwaitABIV0 identifies the first scheduler handoff contract: + // a coroutine parent may publish one initial-suspended static child and cut + // its stack, but only the scheduler may subsequently resume or destroy either + // frame. It deliberately does not claim spawn, park, preemption, or roots. + SchedulerChildAwaitABIV0 = "llgo.coro.scheduler.child-await.v0" + PanicLegacyABIV0 = "llgo.coro.panic.legacy.v0" + FuncRepABIV0 = "llgo.coro.func-rep.v0" ) // PlanDigestMetadata contains every effective ABI and target input that may @@ -64,11 +70,17 @@ type planDigestDocument struct { Schema string `json:"schema"` FunctionIDSchema string `json:"function_id_schema"` Metadata PlanDigestMetadata `json:"metadata"` + Roots []planDigestRoot `json:"roots"` Functions []planDigestFunction `json:"functions"` Calls []planDigestCall `json:"calls"` Values []planDigestValue `json:"values"` } +type planDigestRoot struct { + Function FunctionID `json:"function"` + Demand uint8 `json:"demand"` +} + type planDigestFunction struct { ID FunctionID `json:"id"` DeclaredEffect uint16 `json:"declared_effect"` @@ -162,6 +174,10 @@ func (p *SSAPlan) canonicalPlanDigest(metadata PlanDigestMetadata) (planDigestDo return planDigestDocument{}, fmt.Errorf("coro: plan digest scheduler ABI %q does not match FunctionID ABI %q", metadata.SchedulerABI, identity.SchedulerABI) } + roots, err := p.canonicalDigestRoots() + if err != nil { + return planDigestDocument{}, err + } functions, err := p.canonicalDigestFunctions() if err != nil { return planDigestDocument{}, err @@ -175,6 +191,7 @@ func (p *SSAPlan) canonicalPlanDigest(metadata PlanDigestMetadata) (planDigestDo Schema: PlanDigestSchema, FunctionIDSchema: FunctionIDSchema, Metadata: metadata, + Roots: roots, Functions: functions, Calls: make([]planDigestCall, 0, len(p.callPlans)), Values: make([]planDigestValue, 0, len(p.valuePlans)), @@ -312,6 +329,47 @@ func validatePlanDigestText(name, value string, allowEmpty bool) error { return nil } +func (p *SSAPlan) canonicalDigestRoots() ([]planDigestRoot, error) { + if p.plan == nil { + return nil, fmt.Errorf("coro: CoroPlanDigest requires a base plan") + } + ret := make([]planDigestRoot, 0, len(p.roots)) + var previous FunctionID + for index, root := range p.roots { + if root.Function == nil { + return nil, fmt.Errorf("coro: SSA root plan %d has nil function", index) + } + if err := validateDigestFunctionID(root.ID); err != nil { + return nil, fmt.Errorf("coro: validate SSA root plan %d: %w", index, err) + } + if err := root.Demand.Validate(); err != nil { + return nil, fmt.Errorf("coro: validate SSA root plan %d demand: %w", index, err) + } + if root.Demand == NoDemand { + return nil, fmt.Errorf("coro: SSA root plan %d has no demand", index) + } + if index != 0 && previous >= root.ID { + return nil, fmt.Errorf("coro: SSA root plans are not in strict FunctionID order") + } + previous = root.ID + if got, ok := p.byFunction[root.Function]; !ok || got != root.ID { + return nil, fmt.Errorf("coro: missing forward root mapping for %q", root.ID) + } + if got, ok := p.byID[root.ID]; !ok || got != root.Function { + return nil, fmt.Errorf("coro: missing reverse root mapping for %q", root.ID) + } + plan, ok := p.plan.Lookup(root.ID) + if !ok { + return nil, fmt.Errorf("coro: root %q is absent from the base plan", root.ID) + } + if !plan.Demand.Contains(root.Demand) { + return nil, fmt.Errorf("coro: root %q demand %s is not contained in function demand %s", root.ID, root.Demand, plan.Demand) + } + ret = append(ret, planDigestRoot{Function: root.ID, Demand: uint8(root.Demand)}) + } + return ret, nil +} + func (p *SSAPlan) canonicalDigestFunctions() ([]planDigestFunction, error) { if p.plan == nil { return nil, fmt.Errorf("coro: CoroPlanDigest requires a base plan") diff --git a/internal/coro/plan_digest_test.go b/internal/coro/plan_digest_test.go index 153aaa4bef..50539b5450 100644 --- a/internal/coro/plan_digest_test.go +++ b/internal/coro/plan_digest_test.go @@ -97,6 +97,9 @@ func TestCoroPlanDigestDeterministicCompleteAndDomainSeparated(t *testing.T) { if len(document.Functions) != len(plainPlan.functions) { t.Fatalf("function records = %d, want %d", len(document.Functions), len(plainPlan.functions)) } + if len(document.Roots) != len(plainPlan.roots) || len(document.Roots) == 0 { + t.Fatalf("root records = %d, plan roots = %d", len(document.Roots), len(plainPlan.roots)) + } if len(document.Calls) != len(plainPlan.callPlans) || len(document.Calls) == 0 { t.Fatalf("call records = %d, map plans = %d", len(document.Calls), len(plainPlan.callPlans)) } @@ -180,6 +183,43 @@ func TestCoroPlanDigestCanonicalTargetsAndPlanMutations(t *testing.T) { } plan.callPlans[multiTargetCall] = originalCall + originalRoots := append([]SSARootPlan(nil), plan.roots...) + var addedRoot SSARootPlan + for _, function := range plan.functions { + isRoot := false + for _, root := range originalRoots { + isRoot = isRoot || root.ID == function.Plan.ID + } + if !isRoot && function.Plan.Demand != NoDemand { + addedRoot = SSARootPlan{Function: function.Function, ID: function.Plan.ID, Demand: function.Plan.Demand} + break + } + } + if addedRoot.Function == nil { + t.Fatal("test plan has no propagated non-root demand") + } + changedRoots := make([]SSARootPlan, 0, len(originalRoots)+1) + inserted := false + for _, root := range originalRoots { + if !inserted && addedRoot.ID < root.ID { + changedRoots = append(changedRoots, addedRoot) + inserted = true + } + changedRoots = append(changedRoots, root) + } + if !inserted { + changedRoots = append(changedRoots, addedRoot) + } + plan.roots = changedRoots + mutated, err = plan.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if mutated == baseline { + t.Fatal("explicit root mutation did not change digest") + } + plan.roots = originalRoots + var value ssa.Value var originalValue SSAValuePlan for candidate, valuePlan := range plan.valuePlans { @@ -274,6 +314,29 @@ func TestCoroPlanDigestFailsClosedOnCallAndValueCoverage(t *testing.T) { t.Fatalf("unreachable SSAValuePlan error = %v", err) } delete(plan.valuePlans, foreignValue) + + originalRoots := append([]SSARootPlan(nil), plan.roots...) + rootMutations := []struct { + name string + want string + mutate func() + }{ + {"nil function", "nil function", func() { plan.roots[0].Function = nil }}, + {"no demand", "has no demand", func() { plan.roots[0].Demand = NoDemand }}, + {"invalid demand", "unknown demand bits", func() { plan.roots[0].Demand = Demand(1 << 7) }}, + {"duplicate", "not in strict FunctionID order", func() { plan.roots = append(plan.roots, plan.roots[0]) }}, + {"foreign function", "missing forward root mapping", func() { plan.roots[0].Function = other.roots[0].Function }}, + } + for _, test := range rootMutations { + t.Run("root/"+test.name, func(t *testing.T) { + plan.roots = append([]SSARootPlan(nil), originalRoots...) + test.mutate() + if _, err := plan.CoroPlanDigest(metadata); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("root mutation error = %v, want %q", err, test.want) + } + }) + } + plan.roots = originalRoots } func TestCoroPlanDigestMetadataValidation(t *testing.T) { @@ -363,9 +426,8 @@ func TestCoroPlanDigestMetadataMutationsChangeDigest(t *testing.T) { } func TestCoroPlanDigestCanonicalEmptyArrays(t *testing.T) { - prog, pkg := buildCoroTestSSA(t, "empty.go", `package coroid; func root() {}`) - root := packageFunction(t, pkg, "root") - plan, err := AnalyzeSSA(prog, Roots{{Function: root, Demand: AsyncDemand}}, planDigestSSAConfig()) + prog, _ := buildCoroTestSSA(t, "empty.go", `package coroid; func root() {}`) + plan, err := AnalyzeSSA(prog, nil, planDigestSSAConfig()) if err != nil { t.Fatal(err) } @@ -378,13 +440,115 @@ func TestCoroPlanDigestCanonicalEmptyArrays(t *testing.T) { t.Fatal(err) } text := string(payload) - for _, field := range []string{`"calls":[]`, `"values":[]`} { + for _, field := range []string{`"roots":[]`, `"calls":[]`, `"values":[]`} { if !strings.Contains(text, field) { t.Fatalf("canonical document %s does not contain %s", text, field) } } } +func TestCoroPlanDigestDistinguishesExplicitAndPropagatedRoots(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "roots.go", `package coroid +func leaf(ch chan int) { <-ch } +func root(ch chan int) { leaf(ch) } +`) + root := packageFunction(t, pkg, "root") + leaf := packageFunction(t, pkg, "leaf") + config := planDigestSSAConfig() + propagated, err := AnalyzeSSA(prog, Roots{{Function: root, Demand: AsyncDemand}}, config) + if err != nil { + t.Fatal(err) + } + explicit, err := AnalyzeSSA(prog, Roots{ + {Function: root, Demand: AsyncDemand}, + {Function: leaf, Demand: AsyncDemand}, + }, config) + if err != nil { + t.Fatal(err) + } + permuted, err := AnalyzeSSA(prog, Roots{ + {Function: leaf, Demand: AsyncDemand}, + {Function: root, Demand: AsyncDemand}, + }, config) + if err != nil { + t.Fatal(err) + } + duplicated, err := AnalyzeSSA(prog, Roots{ + {Function: leaf, Demand: AsyncDemand}, + {Function: root, Demand: AsyncDemand}, + {Function: leaf, Demand: AsyncDemand}, + {Function: root, Demand: AsyncDemand}, + }, config) + if err != nil { + t.Fatal(err) + } + + if got := functionPlanFor(t, propagated, leaf).Demand; got != AsyncDemand { + t.Fatalf("propagated leaf demand = %s, want async", got) + } + if got, want := len(propagated.Roots()), 1; got != want { + t.Fatalf("propagated roots = %d, want %d", got, want) + } + if got, want := len(explicit.Roots()), 2; got != want { + t.Fatalf("explicit roots = %d, want %d", got, want) + } + for _, fn := range []*ssa.Function{root, leaf} { + left, leftOK := propagated.FunctionPlan(fn) + right, rightOK := explicit.FunctionPlan(fn) + if !leftOK || !rightOK || left != right { + t.Fatalf("function plan for %s differs: propagated=%+v,%v explicit=%+v,%v", fn.Name(), left, leftOK, right, rightOK) + } + } + + metadata := validPlanDigestMetadata() + propagatedDocument, err := propagated.canonicalPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + explicitDocument, err := explicit.canonicalPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + propagatedDocument.Roots = nil + explicitDocument.Roots = nil + propagatedPayload, err := json.Marshal(propagatedDocument) + if err != nil { + t.Fatal(err) + } + explicitPayload, err := json.Marshal(explicitDocument) + if err != nil { + t.Fatal(err) + } + if string(propagatedPayload) != string(explicitPayload) { + t.Fatalf("non-root digest plan changed:\npropagated %s\nexplicit %s", propagatedPayload, explicitPayload) + } + propagatedDigest, err := propagated.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + explicitDigest, err := explicit.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if explicitDigest == propagatedDigest { + t.Fatal("explicit Async root and propagated AsyncDemand produced the same digest") + } + permutedDigest, err := permuted.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if permutedDigest != explicitDigest { + t.Fatalf("root input order changed digest: %s != %s", permutedDigest, explicitDigest) + } + duplicatedDigest, err := duplicated.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if duplicatedDigest != explicitDigest { + t.Fatalf("duplicate roots changed digest: %s != %s", duplicatedDigest, explicitDigest) + } +} + func TestCoroPlanDigestProjectsDefinitionlessValueOccurrences(t *testing.T) { prog, pkg := buildCoroTestSSA(t, "occurrences.go", `package coroid func target() {} diff --git a/internal/coro/ssa_plan.go b/internal/coro/ssa_plan.go index 7347ba8b91..a5cd2b9968 100644 --- a/internal/coro/ssa_plan.go +++ b/internal/coro/ssa_plan.go @@ -141,10 +141,19 @@ type SSAFunctionPlan struct { Plan FunctionPlan } +// SSARootPlan records one canonical externally established entry demand. +// Duplicate and aliased input roots are joined before this record is created. +type SSARootPlan struct { + Function *ssa.Function + ID FunctionID + Demand Demand +} + // SSAPlan is the compilation-scoped whole-program result. Its maps remain // private so consumers cannot reconstruct identities from display strings. type SSAPlan struct { plan *Plan + roots []SSARootPlan functions []SSAFunctionPlan byFunction map[*ssa.Function]FunctionID byID map[FunctionID]*ssa.Function @@ -248,6 +257,15 @@ func (p *SSAPlan) Functions() []SSAFunctionPlan { return append([]SSAFunctionPlan(nil), p.functions...) } +// Roots returns canonical joined explicit roots in strict FunctionID order. +// The returned slice is a defensive copy. +func (p *SSAPlan) Roots() []SSARootPlan { + if p == nil { + return nil + } + return append([]SSARootPlan(nil), p.roots...) +} + // FunctionID returns the stable identity assigned to fn. func (p *SSAPlan) FunctionID(fn *ssa.Function) (FunctionID, bool) { if p == nil { @@ -479,6 +497,15 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err byID[id] = fn } sort.Slice(included, func(i, j int) bool { return ids[included[i]] < ids[included[j]] }) + canonicalRoots := make([]SSARootPlan, 0, len(rootDemand)) + for fn, demand := range rootDemand { + id, ok := ids[fn] + if !ok { + return nil, fmt.Errorf("coro: canonical root function %q has no FunctionID", fn.Name()) + } + canonicalRoots = append(canonicalRoots, SSARootPlan{Function: fn, ID: id, Demand: demand}) + } + sort.Slice(canonicalRoots, func(i, j int) bool { return canonicalRoots[i].ID < canonicalRoots[j].ID }) flow, err := analyzeSSAFunctionFlow(included, includedSet, ids, dynamicCandidates, config.DynamicResolution, canonicalizer) if err != nil { @@ -650,6 +677,7 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err } result := &SSAPlan{ plan: base, + roots: canonicalRoots, functions: make([]SSAFunctionPlan, 0, len(included)), byFunction: ids, byID: byID, diff --git a/internal/coro/ssa_plan_test.go b/internal/coro/ssa_plan_test.go index a7d9018c35..56c9b61a95 100644 --- a/internal/coro/ssa_plan_test.go +++ b/internal/coro/ssa_plan_test.go @@ -116,6 +116,92 @@ func send(ch chan int) { ch <- 1 } } } +func TestSSAPlanRootsCanonicalJoinedSortedAndDefensive(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "roots.go", `package coroid +func original() {} +func replacement() {} +func other() {} +`) + original := packageFunction(t, pkg, "original") + replacement := packageFunction(t, pkg, "replacement") + other := packageFunction(t, pkg, "other") + universe, err := NewSSAEmissionUniverse(prog, []*ssa.Function{other, replacement}) + if err != nil { + t.Fatal(err) + } + config := SSAConfig{ + EmissionUniverse: universe, + ResolveFunction: func(fn *ssa.Function) (*ssa.Function, bool, error) { + if fn == original { + return replacement, true, nil + } + return fn, universe.Contains(fn), nil + }, + } + inputs := Roots{ + {Function: original, Demand: SyncDemand}, + {Function: other, Demand: AsyncDemand}, + {Function: replacement, Demand: AsyncDemand}, + {Function: other, Demand: AsyncDemand}, + } + plan, err := AnalyzeSSA(prog, inputs, config) + if err != nil { + t.Fatal(err) + } + inputs[0] = Root{} + permuted, err := AnalyzeSSA(prog, Roots{ + {Function: replacement, Demand: BothDemand}, + {Function: other, Demand: AsyncDemand}, + }, config) + if err != nil { + t.Fatal(err) + } + + wantDemand := map[*ssa.Function]Demand{ + replacement: BothDemand, + other: AsyncDemand, + } + got := plan.Roots() + if len(got) != len(wantDemand) { + t.Fatalf("roots = %+v, want %d canonical roots", got, len(wantDemand)) + } + for index, root := range got { + if index != 0 && got[index-1].ID >= root.ID { + t.Fatalf("roots are not in strict FunctionID order: %+v", got) + } + if want, ok := wantDemand[root.Function]; !ok || root.Demand != want { + t.Fatalf("root %d = %+v, want one of %+v", index, root, wantDemand) + } + if id, ok := plan.FunctionID(root.Function); !ok || id != root.ID { + t.Fatalf("root %d ID = %q, FunctionID = %q, %v", index, root.ID, id, ok) + } + } + permutedRoots := permuted.Roots() + if len(permutedRoots) != len(got) { + t.Fatalf("permuted roots = %+v, want %+v", permutedRoots, got) + } + for index := range got { + if permutedRoots[index] != got[index] { + t.Fatalf("permuted root %d = %+v, want %+v", index, permutedRoots[index], got[index]) + } + } + if got := functionPlanFor(t, plan, replacement).Demand; got != BothDemand { + t.Fatalf("canonical replacement demand = %s, want both", got) + } + if _, ok := plan.FunctionPlan(original); ok { + t.Fatal("aliased root loser entered the plan") + } + + got[0] = SSARootPlan{} + if fresh := plan.Roots(); len(fresh) == 0 || fresh[0].Function == nil || fresh[0].ID == "" || fresh[0].Demand == NoDemand { + t.Fatalf("Roots did not return a defensive slice: %+v", fresh) + } + var nilPlan *SSAPlan + if roots := nilPlan.Roots(); roots != nil { + t.Fatalf("nil plan roots = %+v, want nil", roots) + } +} + func TestSSAPlanFunctionPlanUsesExactSSAFunction(t *testing.T) { const source = `package coroid func generic[T any](value T) T { return value } diff --git a/ssa/coro.go b/ssa/coro.go index bfb5a1790e..9342bae144 100644 --- a/ssa/coro.go +++ b/ssa/coro.go @@ -57,10 +57,12 @@ type CoroOptions struct { Promise Expr Frame CoroFrameOps // BeforeInitialSuspend runs after llvm.coro.begin has produced the handle - // and before the initial suspend is published. It may initialize the - // promise/header and register the handle, but must leave the builder in the - // same unterminated insertion block. - BeforeInitialSuspend func(b Builder, handle Expr) + // and before the initial suspend is published. storage is the allocation + // pointer passed to coro.begin (and may be null when allocation was elided). + // The callback may initialize the promise/header and register the + // handle/storage pair, but must leave the builder in the same unterminated + // insertion block. + BeforeInitialSuspend func(b Builder, handle, storage Expr) AllocationAlign uint32 } @@ -75,6 +77,22 @@ type CoroFrameDescriptorOptions struct { Result Type } +// CoroRootFactoryDescriptorOptions describes the target-specific constant +// used to create a root coroutine. ABIHash is computed by the frontend from +// the complete logical/physical root ABI. Factory must be a constant function +// declaration or function pointer in the same package module with the fixed +// (unsafe.Pointer, unsafe.Pointer, unsafe.Pointer) -> unsafe.Pointer ABI. +// Startup and Result are payload types from the package Program, not pointer +// types, and must be non-nil concrete types. +type CoroRootFactoryDescriptorOptions struct { + Version uint32 + ABIHash [16]byte + Flags uint32 + Factory Expr + Startup Type + Result Type +} + // NewCoroFrameDescriptor defines a link-once constant descriptor with layout: // // { version i32, flags i32, hashLo i64, hashHi i64, @@ -114,6 +132,108 @@ func (p Package) NewCoroFrameDescriptor(name string, opts CoroFrameDescriptorOpt return descriptor.Expr } +// NewCoroRootFactoryDescriptor defines a link-once constant descriptor with +// layout: +// +// { version i32, flags i32, hashLo i64, hashHi i64, factory ptr, +// startupSize uintptr, startupAlign uintptr, +// resultSize uintptr, resultAlign uintptr } +// +// The returned expression points at the descriptor. Size, alignment, and +// uintptr fields follow the package target data layout. The hash words use big +// endian byte order so their textual IR form is deterministic across hosts. +func (p Package) NewCoroRootFactoryDescriptor( + name string, opts CoroRootFactoryDescriptorOptions, +) Expr { + if name == "" { + panic("ssa: coroutine root factory descriptor requires a name") + } + if opts.Factory.IsNil() || + (opts.Factory.kind != vkFuncDecl && opts.Factory.kind != vkFuncPtr) || + opts.Factory.impl.IsAConstant().IsNil() || + !opts.Factory.impl.IsAConstantPointerNull().IsNil() { + panic("ssa: coroutine root factory descriptor requires a non-null constant function factory") + } + factoryFunction := coroRootFactoryFunction(opts.Factory.impl) + if factoryFunction.IsNil() || factoryFunction.GlobalParent().C != p.mod.C { + panic("ssa: coroutine root factory descriptor requires a factory from the same package module") + } + if !isCoroRootFactorySignature(opts.Factory.RawType()) { + panic("ssa: coroutine root factory descriptor requires factory signature (unsafe.Pointer, unsafe.Pointer, unsafe.Pointer) -> unsafe.Pointer") + } + if opts.Startup == nil || opts.Startup.kind == vkInvalid { + panic("ssa: coroutine root factory descriptor requires a concrete startup type") + } + if opts.Result == nil || opts.Result.kind == vkInvalid { + panic("ssa: coroutine root factory descriptor requires a concrete result type") + } + + prog := p.Prog + if opts.Startup.ll.Context().C != prog.ctx.C { + panic("ssa: coroutine root factory descriptor startup type belongs to another program") + } + if opts.Result.ll.Context().C != prog.ctx.C { + panic("ssa: coroutine root factory descriptor result type belongs to another program") + } + descriptorType := prog.Struct( + prog.Uint32(), + prog.Uint32(), + prog.Uint64(), + prog.Uint64(), + prog.VoidPtr(), + prog.Uintptr(), + prog.Uintptr(), + prog.Uintptr(), + prog.Uintptr(), + ) + descriptor := p.NewVarEx(name, prog.Pointer(descriptorType)) + factory := opts.Factory.impl + if factory.Type().C != prog.VoidPtr().ll.C { + factory = llvm.ConstBitCast(factory, prog.VoidPtr().ll) + } + fields := []llvm.Value{ + prog.IntVal(uint64(opts.Version), prog.Uint32()).impl, + prog.IntVal(uint64(opts.Flags), prog.Uint32()).impl, + prog.IntVal(binary.BigEndian.Uint64(opts.ABIHash[:8]), prog.Uint64()).impl, + prog.IntVal(binary.BigEndian.Uint64(opts.ABIHash[8:]), prog.Uint64()).impl, + factory, + prog.IntVal(prog.SizeOf(opts.Startup), prog.Uintptr()).impl, + prog.IntVal(uint64(prog.td.ABITypeAlignment(opts.Startup.ll)), prog.Uintptr()).impl, + prog.IntVal(prog.SizeOf(opts.Result), prog.Uintptr()).impl, + prog.IntVal(uint64(prog.td.ABITypeAlignment(opts.Result.ll)), prog.Uintptr()).impl, + } + descriptor.impl.SetInitializer(prog.ctx.ConstStruct(fields, false)) + descriptor.impl.SetGlobalConstant(true) + descriptor.impl.SetLinkage(llvm.LinkOnceODRLinkage) + descriptor.impl.SetUnnamedAddr(true) + // Root descriptors are runtime/linker discovery points and otherwise have + // no ordinary IR user. llvm.used preserves the descriptor through final-link + // dead stripping; its initializer keeps the typed wrapper reachable. + p.markLLVMRetained(descriptor.impl) + return descriptor.Expr +} + +func coroRootFactoryFunction(value llvm.Value) llvm.Value { + for !value.IsAConstantExpr().IsNil() && value.OperandsCount() == 1 { + value = value.Operand(0) + } + return value.IsAFunction() +} + +func isCoroRootFactorySignature(typ types.Type) bool { + sig, ok := typ.(*types.Signature) + if !ok || sig.Recv() != nil || sig.Variadic() || sig.Params().Len() != 3 || sig.Results().Len() != 1 { + return false + } + pointer := types.Typ[types.UnsafePointer] + for i := 0; i < sig.Params().Len(); i++ { + if !types.Identical(sig.Params().At(i).Type(), pointer) { + return false + } + } + return types.Identical(sig.Results().At(0).Type(), pointer) +} + // CoroBuilder owns the structured presplit control flow for one coroutine. // It does not define the promise, result, scheduler, or runtime frame ABI. type CoroBuilder struct { @@ -204,7 +324,7 @@ func (b Builder) BeginCoro(opts CoroOptions) *CoroBuilder { } if callback := opts.BeforeInitialSuspend; callback != nil { callbackPoint := captureCoroFrameCallbackPoint(b) - callback(b, coro.handle) + callback(b, coro.handle, storage.Expr) callbackPoint.ensureContinuation(b, "before-initial-suspend") } coro.initialResumeBlk = coro.emitSuspend(false) @@ -319,6 +439,81 @@ func (c *CoroBuilder) requireActive(operation string) { } } +// CoroPromise returns a typed pointer to the promise associated with handle. +// +// promise is the promise payload type, not a pointer type. The generated +// llvm.coro.promise call uses the target ABI alignment of that payload and the +// handle-to-promise direction (from=false). The handle must be a pointer-valued +// expression produced by llvm.coro.begin or otherwise supplied by the +// coroutine runtime. +func (b Builder) CoroPromise(handle Expr, promise Type) Expr { + b.requireCoroHandle("get promise for", handle) + if promise == nil || promise.kind == vkInvalid { + panic("ssa: coroutine promise requires a concrete payload type") + } + + prog := b.Prog + promisePtr := prog.Pointer(promise) + value := b.coroIntrinsic( + "llvm.coro.promise", + promisePtr.ll, + []llvm.Value{ + b.Convert(prog.VoidPtr(), handle).impl, + prog.IntVal(uint64(prog.td.ABITypeAlignment(promise.ll)), prog.Int32()).impl, + prog.BoolVal(false).impl, + }, + "coro.promise", + ) + return Expr{value, promisePtr} +} + +// CoroDone reports whether a suspended coroutine is at its final suspend. +// Calling it for a running coroutine or a coroutine without a final suspend is +// invalid according to LLVM's coroutine contract. +func (b Builder) CoroDone(handle Expr) Expr { + b.requireCoroHandle("query done for", handle) + prog := b.Prog + value := b.coroIntrinsic( + "llvm.coro.done", + prog.Bool().ll, + []llvm.Value{b.Convert(prog.VoidPtr(), handle).impl}, + "coro.done", + ) + return Expr{value, prog.Bool()} +} + +// CoroResume resumes a suspended coroutine. A final-suspended coroutine must +// be destroyed instead and must never be resumed. +func (b Builder) CoroResume(handle Expr) { + b.requireCoroHandle("resume", handle) + b.coroIntrinsic( + "llvm.coro.resume", + b.Prog.Void().ll, + []llvm.Value{b.Convert(b.Prog.VoidPtr(), handle).impl}, + "", + ) +} + +// CoroDestroy destroys a suspended coroutine exactly once. +func (b Builder) CoroDestroy(handle Expr) { + b.requireCoroHandle("destroy", handle) + b.coroIntrinsic( + "llvm.coro.destroy", + b.Prog.Void().ll, + []llvm.Value{b.Convert(b.Prog.VoidPtr(), handle).impl}, + "", + ) +} + +func (b Builder) requireCoroHandle(operation string, handle Expr) { + if b == nil || b.Func == nil || b.blk == nil { + panic("ssa: cannot " + operation + " coroutine without an active function block") + } + if handle.IsNil() || handle.kind != vkPtr { + panic("ssa: coroutine handle must be a pointer") + } +} + func validateCoroOptions(b Builder, opts CoroOptions) { if b == nil || b.Func == nil || b.blk == nil { panic("ssa: begin coroutine without an active function block") diff --git a/ssa/coro_test.go b/ssa/coro_test.go index 7d5ef1b630..2b791477b6 100644 --- a/ssa/coro_test.go +++ b/ssa/coro_test.go @@ -127,6 +127,71 @@ func TestCoroBuilderCoroSplit(t *testing.T) { } } +func TestCoroHandleIntrinsicsBeforeAndAfterCoroSplit(t *testing.T) { + fixture := newCoroTestFixture(t, nil, 32) + prog := fixture.prog + promiseType := prog.Struct(prog.Byte(), prog.Uint64()) + control := fixture.pkg.NewFunc("coro_control", functionSignature( + []types.Type{types.Typ[types.UnsafePointer]}, + []types.Type{types.Typ[types.Bool]}, + ), InC) + b := control.MakeBody(1) + handle := control.Param(0) + promise := b.CoroPromise(handle, promiseType) + if promise.kind != vkPtr || + !types.Identical(promise.RawType(), types.NewPointer(promiseType.RawType())) { + t.Fatalf("CoroPromise type = %v, want pointer to %v", promise.RawType(), promiseType.RawType()) + } + done := b.CoroDone(handle) + if done.kind != vkBool || !types.Identical(done.RawType(), types.Typ[types.Bool]) { + t.Fatalf("CoroDone type = %v, want bool", done.RawType()) + } + b.CoroResume(handle) + b.CoroDestroy(handle) + b.Return(done) + b.EndBuild() + b.Dispose() + + mod := fixture.pkg.Module() + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify coroutine handle intrinsics: %v\n%s", err, mod.String()) + } + pre := mod.String() + for _, intrinsic := range []string{ + "llvm.coro.promise", "llvm.coro.done", "llvm.coro.resume", "llvm.coro.destroy", + } { + if !hasCoroIntrinsicCall(pre, intrinsic) { + t.Fatalf("presplit module lacks %s call:\n%s", intrinsic, pre) + } + } + wantAlign := prog.td.ABITypeAlignment(promiseType.ll) + promiseCall := regexp.MustCompile(fmt.Sprintf( + `(?m)call ptr @llvm\.coro\.promise\(ptr [^,]+, i32 %d, i1 false\)`, wantAlign, + )) + if !promiseCall.MatchString(pre) { + t.Fatalf("llvm.coro.promise does not use payload ABI alignment %d and from=false:\n%s", wantAlign, pre) + } + + pipeline := "coro-early,cgscc(coro-split),coro-cleanup" + if llvmMajorVersion() == 14 { + pipeline = "function(coro-early),cgscc(coro-split),function(coro-cleanup)" + } + runCoroPasses(t, fixture, pipeline) + post := mod.String() + for _, intrinsic := range []string{ + "llvm.coro.promise", "llvm.coro.done", "llvm.coro.resume", "llvm.coro.destroy", + } { + if hasCoroIntrinsicCall(post, intrinsic) { + t.Fatalf("post-split module still calls %s:\n%s", intrinsic, post) + } + } + for _, suffix := range []string{".resume", ".destroy"} { + if mod.NamedFunction("coro_test" + suffix).IsNil() { + t.Fatalf("CoroSplit did not create coro_test%s:\n%s", suffix, post) + } + } +} + func TestCoroBuilderDefaultPipelineLLVM19(t *testing.T) { if llvmMajorVersion() != 19 { t.Skipf("production default smoke is specific to LLVM 19, using %s", llvm.Version) @@ -159,6 +224,288 @@ func TestCoroBuilderTargetUintptrIntrinsics(t *testing.T) { } } +func TestCoroPromiseUsesWasm32ABIAlignment(t *testing.T) { + Initialize(InitAll) + prog := NewProgram(&Target{GOOS: "wasip1", GOARCH: "wasm"}) + pkg := prog.NewPackage("coropromise", "coro/promise") + t.Cleanup(func() { + pkg.Module().Dispose() + prog.Dispose() + }) + + fn := pkg.NewFunc("coro_promise", functionSignature( + []types.Type{types.Typ[types.UnsafePointer]}, + []types.Type{types.Typ[types.Bool]}, + ), InC) + b := fn.MakeBody(1) + promiseType := prog.Uint64() + promise := b.CoroPromise(fn.Param(0), promiseType) + if promise.kind != vkPtr || + !types.Identical(promise.RawType(), types.NewPointer(promiseType.RawType())) { + t.Fatalf("CoroPromise type = %v, want *uint64", promise.RawType()) + } + b.Return(b.CoroDone(fn.Param(0))) + b.EndBuild() + b.Dispose() + + if got := prog.PointerSize(); got != 4 { + t.Fatalf("wasm pointer size = %d, want 4", got) + } + align := prog.td.ABITypeAlignment(promiseType.ll) + if align != 8 { + t.Fatalf("wasm uint64 ABI alignment = %d, want 8", align) + } + ir := pkg.String() + want := regexp.MustCompile( + `call ptr @llvm\.coro\.promise\(ptr [^,]+, i32 8, i1 false\)`, + ) + if !want.MatchString(ir) { + t.Fatalf("wasm llvm.coro.promise lacks i32 ABI alignment and from=false:\n%s", ir) + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify wasm coroutine promise accessor: %v\n%s", err, ir) + } +} + +func TestCoroRootFactoryDescriptorTargetLayout(t *testing.T) { + Initialize(InitAll) + tests := []struct { + name string + target *Target + pointerSize int + startupSize uint64 + startupAlign uint64 + resultSize uint64 + resultAlign uint64 + descriptorSize uint64 + startupSizeOffset uint64 + resultAlignOffset uint64 + }{ + { + name: "native", + pointerSize: 8, + startupSize: 16, + startupAlign: 8, + resultSize: 8, + resultAlign: 8, + descriptorSize: 64, + startupSizeOffset: 32, + resultAlignOffset: 56, + }, + { + name: "wasm32", + target: &Target{GOOS: "wasip1", GOARCH: "wasm"}, + pointerSize: 4, + startupSize: 8, + startupAlign: 4, + resultSize: 4, + resultAlign: 4, + descriptorSize: 48, + startupSizeOffset: 28, + resultAlignOffset: 40, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + prog := NewProgram(test.target) + pkg := prog.NewPackage("cororoot", "coro/root") + t.Cleanup(func() { + pkg.Module().Dispose() + prog.Dispose() + }) + + factory := pkg.NewFunc("coro_root_factory", coroRootFactoryTestSignature(), InC) + startup := prog.Struct(prog.VoidPtr(), prog.VoidPtr()) + result := prog.VoidPtr() + hash := [16]byte{ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + } + descriptor := pkg.NewCoroRootFactoryDescriptor( + "coro_root_descriptor", + CoroRootFactoryDescriptorOptions{ + Version: 7, + ABIHash: hash, + Flags: 0xa5, + Factory: factory.Expr, + Startup: startup, + Result: result, + }, + ) + + if got := prog.PointerSize(); got != test.pointerSize { + t.Fatalf("pointer size = %d, want %d", got, test.pointerSize) + } + if descriptor.kind != vkPtr { + t.Fatalf("descriptor kind = %d, want pointer", descriptor.kind) + } + if !descriptor.impl.IsGlobalConstant() { + t.Fatal("root factory descriptor is not a constant global") + } + if got := descriptor.impl.Linkage(); got != llvm.LinkOnceODRLinkage { + t.Fatalf("descriptor linkage = %v, want linkonce_odr", got) + } + + descriptorType := prog.Elem(descriptor.Type) + if got := prog.SizeOf(descriptorType); got != test.descriptorSize { + t.Fatalf("descriptor size = %d, want %d", got, test.descriptorSize) + } + if got := prog.OffsetOf(descriptorType, 5); got != test.startupSizeOffset { + t.Fatalf("startupSize offset = %d, want %d", got, test.startupSizeOffset) + } + if got := prog.OffsetOf(descriptorType, 8); got != test.resultAlignOffset { + t.Fatalf("resultAlign offset = %d, want %d", got, test.resultAlignOffset) + } + if got, want := descriptor.impl.Alignment(), + prog.td.ABITypeAlignment(descriptorType.ll); got != want { + t.Fatalf("descriptor alignment = %d, want target ABI alignment %d", got, want) + } + + initializer := descriptor.impl.Initializer() + if initializer.IsAConstantStruct().IsNil() { + t.Fatalf("descriptor initializer is not a constant struct: %v", initializer) + } + if got := initializer.OperandsCount(); got != 9 { + t.Fatalf("descriptor fields = %d, want 9", got) + } + wantFixed := []uint64{ + 7, + 0xa5, + 0x0102030405060708, + 0x090a0b0c0d0e0f10, + } + for i, want := range wantFixed { + if got := initializer.Operand(i).ZExtValue(); got != want { + t.Fatalf("descriptor field %d = %#x, want %#x", i, got, want) + } + } + factoryField := initializer.Operand(4) + if factoryField.Type().TypeKind() != llvm.PointerTypeKind || + !factoryField.IsAConstantPointerNull().IsNil() { + t.Fatalf("factory field is not a non-null constant pointer: %v", factoryField) + } + wantPayload := []uint64{ + test.startupSize, + test.startupAlign, + test.resultSize, + test.resultAlign, + } + for i, want := range wantPayload { + field := initializer.Operand(i + 5) + if got := field.Type().IntTypeWidth(); got != test.pointerSize*8 { + t.Fatalf("descriptor uintptr field %d width = %d, want %d", i+5, got, test.pointerSize*8) + } + if got := field.ZExtValue(); got != want { + t.Fatalf("descriptor field %d = %d, want %d", i+5, got, want) + } + } + + ir := pkg.String() + if !strings.Contains(ir, + "@coro_root_descriptor = linkonce_odr unnamed_addr constant") { + t.Fatalf("descriptor is not unnamed_addr linkonce_odr constant:\n%s", ir) + } + if !strings.Contains(ir, "@coro_root_factory") { + t.Fatalf("descriptor does not reference the root factory:\n%s", ir) + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify root factory descriptor: %v\n%s", err, ir) + } + }) + } +} + +func TestCoroRootFactoryDescriptorRejectsMisuse(t *testing.T) { + Initialize(InitAll) + prog := NewProgram(nil) + defer prog.Dispose() + pkg := prog.NewPackage("badcororoot", "bad/coro/root") + defer pkg.Module().Dispose() + factory := pkg.NewFunc("coro_root_factory", coroRootFactoryTestSignature(), InC) + startup := prog.Struct(prog.VoidPtr(), prog.VoidPtr()) + result := prog.VoidPtr() + valid := CoroRootFactoryDescriptorOptions{ + Factory: factory.Expr, + Startup: startup, + Result: result, + } + + mustPanicContains(t, "requires a name", func() { + pkg.NewCoroRootFactoryDescriptor("", valid) + }) + mustPanicContains(t, "constant function factory", func() { + bad := valid + bad.Factory = Nil + pkg.NewCoroRootFactoryDescriptor("missing_factory", bad) + }) + mustPanicContains(t, "constant function factory", func() { + bad := valid + bad.Factory = prog.IntVal(1, prog.Uintptr()) + pkg.NewCoroRootFactoryDescriptor("integer_factory", bad) + }) + mustPanicContains(t, "constant function factory", func() { + bad := valid + bad.Factory = prog.Nil(prog.rawType(coroHandleSignature())) + pkg.NewCoroRootFactoryDescriptor("null_factory", bad) + }) + mustPanicContains(t, "factory signature", func() { + bad := valid + bad.Factory = pkg.NewFunc("wrong_arity_factory", coroHandleSignature(), InC).Expr + pkg.NewCoroRootFactoryDescriptor("wrong_arity_descriptor", bad) + }) + mustPanicContains(t, "factory signature", func() { + bad := valid + bad.Factory = pkg.NewFunc("wrong_return_factory", functionSignature( + []types.Type{ + types.Typ[types.UnsafePointer], + types.Typ[types.UnsafePointer], + types.Typ[types.UnsafePointer], + }, + []types.Type{types.Typ[types.Bool]}, + ), InC).Expr + pkg.NewCoroRootFactoryDescriptor("wrong_return_descriptor", bad) + }) + foreignPkg := prog.NewPackage("foreigncororoot", "foreign/coro/root") + defer foreignPkg.Module().Dispose() + mustPanicContains(t, "same package module", func() { + bad := valid + bad.Factory = foreignPkg.NewFunc("foreign_factory", coroRootFactoryTestSignature(), InC).Expr + pkg.NewCoroRootFactoryDescriptor("foreign_factory_descriptor", bad) + }) + mustPanicContains(t, "concrete startup type", func() { + bad := valid + bad.Startup = nil + pkg.NewCoroRootFactoryDescriptor("missing_startup", bad) + }) + mustPanicContains(t, "concrete startup type", func() { + bad := valid + bad.Startup = prog.Void() + pkg.NewCoroRootFactoryDescriptor("void_startup", bad) + }) + mustPanicContains(t, "concrete result type", func() { + bad := valid + bad.Result = nil + pkg.NewCoroRootFactoryDescriptor("missing_result", bad) + }) + mustPanicContains(t, "concrete result type", func() { + bad := valid + bad.Result = prog.Void() + pkg.NewCoroRootFactoryDescriptor("void_result", bad) + }) + foreignProg := NewProgram(nil) + defer foreignProg.Dispose() + mustPanicContains(t, "startup type belongs to another program", func() { + bad := valid + bad.Startup = foreignProg.Struct(foreignProg.VoidPtr()) + pkg.NewCoroRootFactoryDescriptor("foreign_startup_descriptor", bad) + }) + mustPanicContains(t, "result type belongs to another program", func() { + bad := valid + bad.Result = foreignProg.VoidPtr() + pkg.NewCoroRootFactoryDescriptor("foreign_result_descriptor", bad) + }) +} + func TestCoroBuilderRejectsMisuse(t *testing.T) { fixture := newCoroTestFixture(t, nil, 0) mustPanicContains(t, "finished coroutine", func() { fixture.coro.Suspend() }) @@ -184,6 +531,43 @@ func TestCoroBuilderRejectsMisuse(t *testing.T) { }) b := fn.MakeBody(1) defer b.Dispose() + invalidHandles := []struct { + name string + handle Expr + }{ + {"nil expression", Nil}, + {"integer", prog.IntVal(1, prog.Uintptr())}, + {"function", fn.Expr}, + } + for _, test := range invalidHandles { + t.Run("reject handle "+test.name, func(t *testing.T) { + operations := []struct { + name string + call func() + }{ + {"promise", func() { b.CoroPromise(test.handle, prog.Byte()) }}, + {"done", func() { b.CoroDone(test.handle) }}, + {"resume", func() { b.CoroResume(test.handle) }}, + {"destroy", func() { b.CoroDestroy(test.handle) }}, + } + for _, operation := range operations { + t.Run(operation.name, func(t *testing.T) { + mustPanicContains(t, "handle must be a pointer", operation.call) + }) + } + }) + } + validHandle := prog.Nil(prog.VoidPtr()) + mustPanicContains(t, "concrete payload type", func() { + b.CoroPromise(validHandle, nil) + }) + mustPanicContains(t, "concrete payload type", func() { + b.CoroPromise(validHandle, prog.Void()) + }) + var nilBuilder Builder + mustPanicContains(t, "without an active function block", func() { + nilBuilder.CoroDone(validHandle) + }) mustPanicContains(t, "alignment", func() { b.BeginCoro(CoroOptions{ AllocationAlign: 3, @@ -246,7 +630,7 @@ func TestCoroBuilderRejectsCallbackControlFlow(t *testing.T) { }, Free: func(Builder, Expr, Expr, Expr) {}, }, - BeforeInitialSuspend: func(b Builder, _ Expr) { + BeforeInitialSuspend: func(b Builder, _, _ Expr) { b.Unreachable() }, }) @@ -270,6 +654,7 @@ func newCoroCallbackTestBuilder(t *testing.T) (Program, Builder) { func newCoroTestFixture(t *testing.T, target *Target, allocationAlign uint32) *coroTestFixture { t.Helper() + Initialize(InitAll) prog := NewProgram(target) pkg := prog.NewPackage("corotest", "coro/test") t.Cleanup(func() { @@ -304,7 +689,7 @@ func newCoroTestFixture(t *testing.T, target *Target, allocationAlign uint32) *c b.Call(free.Expr, frame, size, align) }, }, - BeforeInitialSuspend: func(b Builder, handle Expr) { + BeforeInitialSuspend: func(b Builder, handle, _ Expr) { if handle.IsNil() { t.Fatal("before-initial-suspend callback received a nil handle") } @@ -339,6 +724,17 @@ func coroHandleSignature() *types.Signature { return functionSignature(nil, []types.Type{types.Typ[types.UnsafePointer]}) } +func coroRootFactoryTestSignature() *types.Signature { + return functionSignature( + []types.Type{ + types.Typ[types.UnsafePointer], + types.Typ[types.UnsafePointer], + types.Typ[types.UnsafePointer], + }, + []types.Type{types.Typ[types.UnsafePointer]}, + ) +} + func runCoroPasses(t *testing.T, fixture *coroTestFixture, pipeline string) { t.Helper() mod := fixture.pkg.Module() @@ -388,6 +784,10 @@ func countCoroEndCalls(ir string) int { return strings.Count(ir, "call i1 @llvm.coro.end") + strings.Count(ir, "call void @llvm.coro.end") } +func hasCoroIntrinsicCall(ir, intrinsic string) bool { + return regexp.MustCompile(`call [^\n]*@` + regexp.QuoteMeta(intrinsic) + `\b`).MatchString(ir) +} + func frameAllocCallLine(ir string) string { for _, line := range strings.Split(ir, "\n") { if strings.Contains(line, "call") && strings.Contains(line, "@coro_frame_alloc") { diff --git a/ssa/package.go b/ssa/package.go index 620952630b..d7b291a65f 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -537,9 +537,10 @@ func (p Program) NewPackage(name, pkgPath string) Package { framePointerAttr: framePointerAttr, pyobjs: pyobjs, pymods: pymods, strs: strs, di: nil, cu: nil, glbDbgVars: glbDbgVars, - export: make(map[string]string), - preserveSyms: make(map[string]struct{}), - llvmUsedValues: make([]llvm.Value, 0, 4), + export: make(map[string]string), + preserveSyms: make(map[string]struct{}), + llvmUsedValues: make([]llvm.Value, 0, 4), + llvmRetainedValues: make([]llvm.Value, 0, 1), abiTypeFakeUseCache: make(map[llvm.Value][]llvm.Value), } @@ -814,9 +815,10 @@ type aPackage struct { MethodByIndex map[int]none MethodByName map[string]none - export map[string]string // pkgPath.nameInPkg => exportname - preserveSyms map[string]struct{} // set of exported symbol names - llvmUsedValues []llvm.Value + export map[string]string // pkgPath.nameInPkg => exportname + preserveSyms map[string]struct{} // set of exported symbol names + llvmUsedValues []llvm.Value + llvmRetainedValues []llvm.Value abiTypeFakeUseCache map[llvm.Value][]llvm.Value } @@ -848,13 +850,27 @@ func (p Package) markLLVMUsed(v llvm.Value) { p.llvmUsedValues = append(p.llvmUsedValues, llvm.ConstBitCast(v, elemTyp)) } +// markLLVMRetained preserves a linker-discoverable value through compiler +// optimization, object emission, and final-link section garbage collection. +// Unlike llvm.compiler.used, llvm.used is part of the linker retention +// contract and must be reserved for values that are discovered out of band. +func (p Package) markLLVMRetained(v llvm.Value) { + elemTyp := p.Prog.VoidPtr().ll + p.llvmRetainedValues = append(p.llvmRetainedValues, llvm.ConstBitCast(v, elemTyp)) +} + func (p Package) MaterializePreserveSyms() { - if len(p.llvmUsedValues) == 0 { + p.materializeLLVMUsed("llvm.compiler.used", p.llvmUsedValues) + p.materializeLLVMUsed("llvm.used", p.llvmRetainedValues) +} + +func (p Package) materializeLLVMUsed(name string, values []llvm.Value) { + if len(values) == 0 { return } elemTyp := p.Prog.VoidPtr().ll - init := llvm.ConstArray(elemTyp, p.llvmUsedValues) - global := llvm.AddGlobal(p.mod, init.Type(), "llvm.compiler.used") + init := llvm.ConstArray(elemTyp, values) + global := llvm.AddGlobal(p.mod, init.Type(), name) global.SetInitializer(init) global.SetLinkage(llvm.AppendingLinkage) global.SetSection("llvm.metadata")