From bbc0e13b1eb713400d5aa15cdba2598a51d48312 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 01:12:20 +0800 Subject: [PATCH] compiler: add structured LLVM coroutine builder --- .github/workflows/coroutine.yml | 26 ++- ssa/coro.go | 386 ++++++++++++++++++++++++++++++++ ssa/coro_test.go | 379 +++++++++++++++++++++++++++++++ 3 files changed, 786 insertions(+), 5 deletions(-) create mode 100644 ssa/coro.go create mode 100644 ssa/coro_test.go diff --git a/.github/workflows/coroutine.yml b/.github/workflows/coroutine.yml index 20883388cf..bc2cf88a0a 100644 --- a/.github/workflows/coroutine.yml +++ b/.github/workflows/coroutine.yml @@ -11,15 +11,22 @@ concurrency: jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + llvm: [14, 18, 19, 21] steps: - uses: actions/checkout@v7 - - name: Install dependencies - uses: ./.github/actions/setup-deps - with: - llvm-version: 19 + - name: Install LLVM + run: | + echo 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.llvm }} main' | sudo tee /etc/apt/sources.list.d/llvm.list + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo apt-get update + sudo apt-get install --no-install-recommends llvm-${{ matrix.llvm }}-dev clang-${{ matrix.llvm }} + echo '/usr/lib/llvm-${{ matrix.llvm }}/bin' >> "$GITHUB_PATH" - name: Set up Go uses: ./.github/actions/setup-go @@ -29,15 +36,22 @@ jobs: # Temporary while the stackless-coroutine slices are integrated. Restore # the full Go workflow, including macOS, before the upstream merge. - name: Test coroutine analysis + if: matrix.llvm == 19 run: go test -race -shuffle=on ./internal/coro - name: Test coroutine build integration + if: matrix.llvm == 19 run: go test ./internal/build -run 'Test(CoroPlanBuilderRunsBeforeCodegenWithoutChangingIR|BuildCoroPlanErrors)$' -count=1 - name: Test coroutine compiler integration + if: matrix.llvm == 19 run: go test ./cl -run '^TestCompilationCoroPlanObservationAndCacheRegistration$' -count=1 + - name: Test structured LLVM coroutine builder + run: go test -tags=llvm${{ matrix.llvm }} -v ./ssa -run '^TestCoroBuilder' -count=1 + - name: Test resolved LLVM target configuration + if: matrix.llvm == 19 run: | go test ./internal/xtool/llvm -run '^TestGetTarget(Spec|Triple)$' -count=1 go test ./internal/crosscompile -run '^Test(UseTarget|UseExportsResolvedLLVMConfig|ResolvedLLVMTargetSpecWASIThreads)$' -count=1 @@ -46,9 +60,11 @@ jobs: go test ./internal/build -run '^Test(NewLLSSATargetUsesResolvedLLVMConfig|LLVMCPUAndFeaturesAffectBuildFingerprint|DefaultTargetKeepsLegacyCacheIdentity|NonDefaultLLVMFeaturesEnterCacheIdentity|ResolvedTargetCompatibilityAudit)$' -count=1 - name: Check llgo-tag build + if: matrix.llvm == 19 run: go test -tags=llgo ./internal/coro - name: Vet coroutine analysis + if: matrix.llvm == 19 run: | go vet ./internal/coro ./internal/build ./internal/xtool/llvm ./internal/crosscompile ./internal/cabi # The SSA package has pre-existing sync.Map copylocks findings. Keep diff --git a/ssa/coro.go b/ssa/coro.go new file mode 100644 index 0000000000..7faf244b83 --- /dev/null +++ b/ssa/coro.go @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import ( + "fmt" + "go/types" + "strconv" + "strings" + + "github.com/xgo-dev/llvm" +) + +// CoroFrameOps emits target-independent coroutine frame allocation calls. +// +// The callbacks run at the builder's current insertion point. They deliberately +// receive both llvm.coro.size and the effective required allocation alignment +// so a later runtime can capture a frame descriptor without this package fixing +// that runtime's ABI. The alignment is at least llvm.coro.align and at least the +// guarantee declared by CoroOptions.AllocationAlign. Free is called only when +// llvm.coro.free returns a non-null allocation pointer. Each callback may append +// instructions but must leave the builder in the same unterminated basic block; +// CoroBuilder appends the required control-flow edge immediately afterwards. +// When the llvm.coro.alloc path executes, Alloc must return a non-null pointer; +// a target runtime must handle allocation failure before returning to the ramp. +type CoroFrameOps struct { + Alloc func(b Builder, size, align Expr) Expr + Free func(b Builder, frame, size, align Expr) +} + +// CoroOptions configures one LLVM switched-resume coroutine. +// +// Promise may be Nil when no promise is required. A non-Nil Promise must point +// to the alloca designated as the LLVM coroutine promise. +// +// AllocationAlign is the alignment guarantee passed to llvm.coro.id for memory +// returned by Frame.Alloc. Zero uses LLVM's default guarantee of twice the +// target pointer size. A non-zero value must be a power of two. Frame.Alloc is +// always passed an effective alignment that satisfies this guarantee as well as +// llvm.coro.align. +type CoroOptions struct { + Promise Expr + AllocationAlign uint32 + Frame CoroFrameOps +} + +// CoroBuilder owns the structured presplit control flow for one coroutine. +// It does not define the promise, result, scheduler, or runtime frame ABI. +type CoroBuilder struct { + b Builder + + id llvm.Value + handle Expr + frame CoroFrameOps + // allocationAlign is the literal guarantee supplied to llvm.coro.id. Zero + // retains LLVM's target-dependent 2*pointer default. + allocationAlign uint32 + + suspendBlk BasicBlock + cleanupBlk BasicBlock + finished bool +} + +// BeginCoro emits the coroutine allocation prologue and initial suspend. The +// enclosing function must return exactly one unsafe.Pointer coroutine handle. +// On return, b is positioned at the initial-resume body block. +func (b Builder) BeginCoro(opts CoroOptions) *CoroBuilder { + validateCoroOptions(b, opts) + markPresplitCoroutine(b.Func) + + prog := b.Prog + fn := b.Func + entryBlk := b.blk + allocBlk := fn.MakeBlock() + beginBlk := fn.MakeBlock() + suspendBlk := fn.MakeBlock() + cleanupBlk := fn.MakeBlock() + + promise := prog.Nil(prog.VoidPtr()) + if !opts.Promise.IsNil() { + promise = b.Convert(prog.VoidPtr(), opts.Promise) + } + null := prog.Nil(prog.VoidPtr()) + align := prog.IntVal(uint64(opts.AllocationAlign), prog.Int32()) + id := b.coroIntrinsic( + "llvm.coro.id", + prog.ctx.TokenType(), + []llvm.Value{align.impl, promise.impl, null.impl, null.impl}, + "coro.id", + ) + needAlloc := b.coroIntrinsic( + "llvm.coro.alloc", + prog.Bool().ll, + []llvm.Value{id}, + "coro.alloc", + ) + b.If(Expr{needAlloc, prog.Bool()}, allocBlk, beginBlk) + + b.SetBlock(allocBlk) + size, frameAlign := b.coroFrameLayout(opts.AllocationAlign) + allocCallbackPoint := captureCoroFrameCallbackPoint(b) + allocated := opts.Frame.Alloc(b, size, frameAlign) + allocCallbackPoint.ensureContinuation(b, "allocator") + if allocated.IsNil() || allocated.kind != vkPtr { + panic("ssa: coroutine frame allocator returned a non-pointer expression") + } + allocated = b.Convert(prog.VoidPtr(), allocated) + b.Jump(beginBlk) + + b.SetBlock(beginBlk) + storage := b.Phi(prog.VoidPtr()) + storage.AddIncoming(b, []BasicBlock{entryBlk, allocBlk}, func(i int, _ BasicBlock) Expr { + if i == 0 { + return null + } + return allocated + }) + handleValue := b.coroIntrinsic( + "llvm.coro.begin", + prog.VoidPtr().ll, + []llvm.Value{id, storage.impl}, + "coro.handle", + ) + + coro := &CoroBuilder{ + b: b, + id: id, + handle: Expr{handleValue, prog.VoidPtr()}, + frame: opts.Frame, + allocationAlign: opts.AllocationAlign, + suspendBlk: suspendBlk, + cleanupBlk: cleanupBlk, + } + coro.emitSuspend(false) + return coro +} + +// Handle returns the coroutine handle produced by llvm.coro.begin. +func (c *CoroBuilder) Handle() Expr { + if c == nil { + return Nil + } + return c.handle +} + +// Suspend emits a non-final stack cut and positions the builder at the newly +// created resume block. Scheduler state and suspend reasons must be published +// by the caller before invoking Suspend. +func (c *CoroBuilder) Suspend() BasicBlock { + c.requireActive("suspend") + return c.emitSuspend(false) +} + +// Finish emits the final suspend and completes the shared cleanup/return +// blocks. No further instructions may be emitted through c afterwards. +func (c *CoroBuilder) Finish() { + c.requireActive("finish") + c.finished = true + + b := c.b + prog := b.Prog + fn := b.Func + finalResult := c.suspendIntrinsic(true) + invalidResumeBlk := fn.MakeBlock() + switchValue := b.impl.CreateSwitch(finalResult, c.suspendBlk.first, 2) + switchValue.AddCase(llvm.ConstInt(prog.tyInt8(), 0, false), invalidResumeBlk.first) + switchValue.AddCase(llvm.ConstInt(prog.tyInt8(), 1, false), c.cleanupBlk.first) + + b.SetBlock(invalidResumeBlk) + b.coroIntrinsic("llvm.trap", prog.Void().ll, nil, "") + b.Unreachable() + + b.SetBlock(c.cleanupBlk) + frameValue := b.coroIntrinsic( + "llvm.coro.free", + prog.VoidPtr().ll, + []llvm.Value{c.id, c.handle.impl}, + "coro.frame", + ) + frame := Expr{frameValue, prog.VoidPtr()} + freeBlk := fn.MakeBlock() + afterFreeBlk := fn.MakeBlock() + nonNull := llvm.CreateICmp(b.impl, llvm.IntNE, frame.impl, prog.Nil(prog.VoidPtr()).impl) + b.If(Expr{nonNull, prog.Bool()}, freeBlk, afterFreeBlk) + + b.SetBlock(freeBlk) + size, align := b.coroFrameLayout(c.allocationAlign) + freeCallbackPoint := captureCoroFrameCallbackPoint(b) + c.frame.Free(b, frame, size, align) + freeCallbackPoint.ensureContinuation(b, "free") + b.Jump(afterFreeBlk) + + b.SetBlock(afterFreeBlk) + b.Jump(c.suspendBlk) + + // LLVM's canonical switched-resume shape sends every suspend default edge + // and the cleanup edge through one coro.end block. CoroSplit keeps the + // following handle return in the ramp and replaces coro.end with ret void in + // the resume/destroy functions. + b.SetBlock(c.suspendBlk) + b.coroEnd(c.handle) + b.Return(c.handle) +} + +func (c *CoroBuilder) emitSuspend(final bool) BasicBlock { + b := c.b + prog := b.Prog + resumeBlk := b.Func.MakeBlock() + result := c.suspendIntrinsic(final) + switchValue := b.impl.CreateSwitch(result, c.suspendBlk.first, 2) + switchValue.AddCase(llvm.ConstInt(prog.tyInt8(), 0, false), resumeBlk.first) + switchValue.AddCase(llvm.ConstInt(prog.tyInt8(), 1, false), c.cleanupBlk.first) + b.SetBlock(resumeBlk) + return resumeBlk +} + +func (c *CoroBuilder) suspendIntrinsic(final bool) llvm.Value { + b := c.b + return b.coroIntrinsic( + "llvm.coro.suspend", + b.Prog.Byte().ll, + []llvm.Value{b.Prog.ctx.ConstTokenNone(), b.Prog.BoolVal(final).impl}, + "coro.suspend", + ) +} + +func (c *CoroBuilder) requireActive(operation string) { + if c == nil { + panic("ssa: " + operation + " nil coroutine builder") + } + if c.finished { + panic("ssa: cannot " + operation + " finished coroutine") + } +} + +func validateCoroOptions(b Builder, opts CoroOptions) { + if b == nil || b.Func == nil || b.blk == nil { + panic("ssa: begin coroutine without an active function block") + } + sig, ok := b.Func.raw.Type.(*types.Signature) + if !ok || sig.Results().Len() != 1 || + !types.Identical(sig.Results().At(0).Type(), types.Typ[types.UnsafePointer]) { + panic("ssa: coroutine function must return exactly one unsafe.Pointer handle") + } + if opts.Frame.Alloc == nil || opts.Frame.Free == nil { + panic("ssa: coroutine frame allocator and free callbacks are required") + } + if opts.Promise.IsNil() { + // A nil promise is valid independently of the frame allocation guarantee. + } else if opts.Promise.kind != vkPtr { + panic("ssa: coroutine promise must be a pointer") + } + if opts.AllocationAlign != 0 && opts.AllocationAlign&(opts.AllocationAlign-1) != 0 { + panic("ssa: coroutine allocation alignment must be zero or a power of two") + } +} + +type coroFrameCallbackPoint struct { + blk BasicBlock + insert llvm.BasicBlock + instructions []llvm.Value +} + +func captureCoroFrameCallbackPoint(b Builder) coroFrameCallbackPoint { + insert := b.impl.GetInsertBlock() + return coroFrameCallbackPoint{ + blk: b.blk, + insert: insert, + instructions: coroBlockInstructions(insert), + } +} + +func (p coroFrameCallbackPoint) ensureContinuation(b Builder, callback string) { + if b.blk != p.blk || b.impl.GetInsertBlock().C != p.insert.C { + panic("ssa: coroutine frame " + callback + " callback changed insertion block") + } + current := coroBlockInstructions(p.insert) + if len(current) < len(p.instructions) { + panic("ssa: coroutine frame " + callback + " callback modified instructions before append point") + } + for i, instruction := range p.instructions { + if current[i].C != instruction.C { + panic("ssa: coroutine frame " + callback + " callback modified instructions before append point") + } + } + for _, inst := range current { + switch inst.InstructionOpcode() { + case llvm.Ret, llvm.Br, llvm.Switch, llvm.IndirectBr, llvm.Invoke, + llvm.Unreachable, llvm.Resume, llvm.CleanupRet, llvm.CatchRet, + llvm.CatchSwitch: + panic("ssa: coroutine frame " + callback + " callback terminated insertion block") + } + } + // The callbacks are append-only. Re-establish the insertion point at the + // end before CoroBuilder emits its own control-flow edge. + b.impl.SetInsertPointAtEnd(p.insert) +} + +func coroBlockInstructions(block llvm.BasicBlock) []llvm.Value { + var instructions []llvm.Value + for inst := block.FirstInstruction(); !inst.IsNil(); inst = llvm.NextInstruction(inst) { + instructions = append(instructions, inst) + } + return instructions +} + +func markPresplitCoroutine(fn Function) { + major := llvmMajorVersion() + ctx := fn.Pkg.mod.Context() + if major == 14 { + // LLVM 14's string attribute encodes a legacy state machine. Frontends + // must emit the unprepared "0" state before CoroEarly; "1" is reserved + // for a coroutine already prepared for a direct CoroSplit invocation. + fn.impl.AddFunctionAttr(ctx.CreateStringAttribute("coroutine.presplit", "0")) + return + } + kind := llvm.AttributeKindID("presplitcoroutine") + if kind == 0 { + panic(fmt.Sprintf("ssa: LLVM %s has no presplitcoroutine attribute", llvm.Version)) + } + fn.impl.AddFunctionAttr(ctx.CreateEnumAttribute(kind, 0)) +} + +func (b Builder) coroFrameLayout(allocationAlign uint32) (size, align Expr) { + typ := b.Prog.Uintptr() + sizeValue := b.coroIntrinsic("llvm.coro.size", typ.ll, nil, "coro.size") + alignValue := b.coroIntrinsic("llvm.coro.align", typ.ll, nil, "coro.align") + minimum := uint64(allocationAlign) + if minimum == 0 { + minimum = uint64(2 * b.Prog.PointerSize()) + } + minimumValue := llvm.ConstInt(typ.ll, minimum, false) + belowMinimum := llvm.CreateICmp(b.impl, llvm.IntULT, alignValue, minimumValue) + effectiveAlign := b.impl.CreateSelect(belowMinimum, minimumValue, alignValue, "coro.alloc.align") + return Expr{sizeValue, typ}, Expr{effectiveAlign, typ} +} + +func (b Builder) coroEnd(handle Expr) { + major := llvmMajorVersion() + args := []llvm.Value{handle.impl, b.Prog.BoolVal(false).impl} + if major >= 18 { + args = append(args, b.Prog.ctx.ConstTokenNone()) + } + ret := b.Prog.Bool().ll + name := "coro.end" + if major >= 22 { + ret = b.Prog.Void().ll + name = "" + } + b.coroIntrinsic("llvm.coro.end", ret, args, name) +} + +func (b Builder) coroIntrinsic(name string, ret llvm.Type, args []llvm.Value, resultName string) llvm.Value { + id := llvm.LookupIntrinsicID(name) + if id == 0 { + panic(fmt.Sprintf("ssa: LLVM %s has no %s intrinsic", llvm.Version, name)) + } + value := b.impl.CreateIntrinsic(ret, id, args, resultName) + if value.IsNil() { + panic(fmt.Sprintf("ssa: LLVM %s rejected %s intrinsic signature", llvm.Version, name)) + } + return value +} + +func llvmMajorVersion() int { + text, _, _ := strings.Cut(llvm.Version, ".") + major, err := strconv.Atoi(text) + if err != nil { + panic(fmt.Sprintf("ssa: parse LLVM version %q: %v", llvm.Version, err)) + } + return major +} diff --git a/ssa/coro_test.go b/ssa/coro_test.go new file mode 100644 index 0000000000..574fca12ad --- /dev/null +++ b/ssa/coro_test.go @@ -0,0 +1,379 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import ( + "fmt" + "go/token" + "go/types" + "regexp" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type coroTestFixture struct { + prog Program + pkg Package + fn Function + coro *CoroBuilder +} + +func TestCoroBuilderPresplitShape(t *testing.T) { + fixture := newCoroTestFixture(t, nil, 32) + mod := fixture.pkg.Module() + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify presplit coroutine: %v\n%s", err, mod.String()) + } + + ir := mod.String() + if major := llvmMajorVersion(); major == 14 { + if !strings.Contains(ir, `"coroutine.presplit"="0"`) { + t.Fatalf("LLVM 14 coroutine lacks unprepared frontend presplit state:\n%s", ir) + } + } else if !strings.Contains(ir, "presplitcoroutine") { + t.Fatalf("coroutine lacks enum presplit attribute:\n%s", ir) + } + if !strings.Contains(ir, "@llvm.coro.id(i32 32") { + t.Fatalf("coro.id lacks allocation alignment guarantee:\n%s", ir) + } + width := fixture.prog.PointerSize() * 8 + for _, intrinsic := range []string{"size", "align"} { + want := fmt.Sprintf("@llvm.coro.%s.i%d", intrinsic, width) + if !strings.Contains(ir, want) { + t.Fatalf("missing target-width %s intrinsic %q:\n%s", intrinsic, want, ir) + } + } + if got := strings.Count(ir, "call i8 @llvm.coro.suspend"); got != 3 { + t.Fatalf("coro.suspend calls = %d, want initial + ordinary + final:\n%s", got, ir) + } + if !strings.Contains(ir, "@llvm.coro.suspend(token none, i1 true)") { + t.Fatalf("missing final suspend:\n%s", ir) + } + if got := countCoroEndCalls(ir); got != 1 { + t.Fatalf("coro.end calls = %d, want one shared end block:\n%s", got, ir) + } + assertCoroSuspendDefaults(t, fixture) + + if !strings.Contains(ir, "icmp ne") || !strings.Contains(ir, "coro.frame") || !strings.Contains(ir, "br i1") { + t.Fatalf("coro.free result is not guarded by a non-null branch:\n%s", ir) + } + if !strings.Contains(ir, "call void @coro_frame_free") { + t.Fatalf("missing injected frame free callback:\n%s", ir) + } + for _, forbidden := range []string{"@malloc", "@free(", "runtime/internal/runtime", "CoroEnter", "CoroReschedule"} { + if strings.Contains(ir, forbidden) { + t.Fatalf("structured builder introduced forbidden runtime coupling %q:\n%s", forbidden, ir) + } + } +} + +func TestCoroBuilderCoroSplit(t *testing.T) { + fixture := newCoroTestFixture(t, nil, 32) + mod := fixture.pkg.Module() + pipeline := "coro-early,cgscc(coro-split),coro-cleanup" + if llvmMajorVersion() == 14 { + // LLVM 14 implicitly treats a pipeline beginning with coro-early as a + // function pipeline, so every pass-manager level must be explicit. + pipeline = "function(coro-early),cgscc(coro-split),function(coro-cleanup)" + } + runCoroPasses(t, fixture, pipeline) + + post := mod.String() + for _, suffix := range []string{".resume", ".destroy"} { + if mod.NamedFunction("coro_test" + suffix).IsNil() { + t.Fatalf("CoroSplit did not create coro_test%s:\n%s", suffix, post) + } + } + for _, intrinsic := range []string{"llvm.coro.id", "llvm.coro.begin", "llvm.coro.suspend"} { + if strings.Contains(post, "call ") && regexp.MustCompile(`call [^\n]*@`+regexp.QuoteMeta(intrinsic)+`\b`).MatchString(post) { + t.Fatalf("post-split module still calls %s:\n%s", intrinsic, post) + } + } + // The byte local is explicitly 64-byte aligned and live across an ordinary + // suspend. CoroSplit must therefore replace coro.align with a requirement of + // at least 64 in the injected allocation path. The exact select folding is + // intentionally left to later optimization passes. + if strings.Contains(post, "call i64 @llvm.coro.align") || strings.Contains(post, "call i32 @llvm.coro.align") { + t.Fatalf("CoroSplit did not lower coro.align:\n%s", post) + } + allocCall := frameAllocCallLine(post) + if !strings.Contains(allocCall, "%coro.alloc.align") { + t.Fatalf("frame allocator does not receive the normalized frame alignment:\n%s", post) + } + width := fixture.prog.PointerSize() * 8 + maxAlign := regexp.MustCompile(fmt.Sprintf( + `(?m)%%coro\.alloc\.align = select i1 %%[^,]+, i%d 32, i%d 64$`, width, width, + )) + if !maxAlign.MatchString(post) { + t.Fatalf("post-split frame alignment is not max(coro.align=64, allocation guarantee=32):\n%s", post) + } +} + +func TestCoroBuilderDefaultPipelineLLVM19(t *testing.T) { + if llvmMajorVersion() != 19 { + t.Skipf("production default smoke is specific to LLVM 19, using %s", llvm.Version) + } + fixture := newCoroTestFixture(t, nil, 0) + runCoroPasses(t, fixture, "default") + post := fixture.pkg.String() + if fixture.pkg.Module().NamedFunction("coro_test.resume").IsNil() || + fixture.pkg.Module().NamedFunction("coro_test.destroy").IsNil() { + t.Fatalf("default did not split coroutine:\n%s", post) + } +} + +func TestCoroBuilderTargetUintptrIntrinsics(t *testing.T) { + Initialize(InitAll) + fixture := newCoroTestFixture(t, &Target{GOOS: "wasip1", GOARCH: "wasm"}, 0) + if got := fixture.prog.PointerSize(); got != 4 { + t.Fatalf("wasm pointer size = %d, want 4", got) + } + ir := fixture.pkg.String() + for _, intrinsic := range []string{"size", "align"} { + if !strings.Contains(ir, "@llvm.coro."+intrinsic+".i32") { + t.Fatalf("wasm coroutine uses non-i32 %s intrinsic:\n%s", intrinsic, ir) + } + } + // AllocationAlign=0 means llvm.coro.id keeps LLVM's 2*pointer guarantee; + // the callback still receives an effective alignment of at least 8. + if !strings.Contains(ir, "@llvm.coro.id(i32 0") || !strings.Contains(ir, "i32 8") { + t.Fatalf("wasm default allocation alignment is not 2*pointer:\n%s", ir) + } +} + +func TestCoroBuilderRejectsMisuse(t *testing.T) { + fixture := newCoroTestFixture(t, nil, 0) + mustPanicContains(t, "finished coroutine", func() { fixture.coro.Suspend() }) + mustPanicContains(t, "finished coroutine", func() { fixture.coro.Finish() }) + if (*CoroBuilder)(nil).Handle() != Nil { + t.Fatal("nil coroutine builder returned a non-nil handle") + } + + prog := NewProgram(nil) + defer prog.Dispose() + pkg := prog.NewPackage("badcoro", "bad/coro") + defer pkg.Module().Dispose() + fn := pkg.NewFunc("bad_alignment", coroHandleSignature(), InC) + b := fn.MakeBody(1) + defer b.Dispose() + mustPanicContains(t, "alignment", func() { + b.BeginCoro(CoroOptions{ + AllocationAlign: 3, + Frame: CoroFrameOps{ + Alloc: func(Builder, Expr, Expr) Expr { return prog.Nil(prog.VoidPtr()) }, + Free: func(Builder, Expr, Expr, Expr) {}, + }, + }) + }) +} + +func TestCoroBuilderRejectsCallbackControlFlow(t *testing.T) { + t.Run("allocator changes LLVM insertion block", func(t *testing.T) { + prog, b := newCoroCallbackTestBuilder(t) + mustPanicContains(t, "allocator callback changed insertion block", func() { + b.BeginCoro(CoroOptions{Frame: CoroFrameOps{ + Alloc: func(b Builder, _, _ Expr) Expr { + b.SetBlockEx(b.Func.MakeBlock(), AtEnd, false) + return prog.Nil(prog.VoidPtr()) + }, + Free: func(Builder, Expr, Expr, Expr) {}, + }}) + }) + }) + + t.Run("allocator inserts before append point", func(t *testing.T) { + prog, b := newCoroCallbackTestBuilder(t) + mustPanicContains(t, "allocator callback modified instructions before append point", func() { + b.BeginCoro(CoroOptions{Frame: CoroFrameOps{ + Alloc: func(b Builder, _, _ Expr) Expr { + b.SetBlockEx(b.blk, AtStart, false) + b.Unreachable() + return prog.Nil(prog.VoidPtr()) + }, + Free: func(Builder, Expr, Expr, Expr) {}, + }}) + }) + }) + + t.Run("free terminates block", func(t *testing.T) { + prog, b := newCoroCallbackTestBuilder(t) + coro := b.BeginCoro(CoroOptions{Frame: CoroFrameOps{ + Alloc: func(Builder, Expr, Expr) Expr { + return prog.Nil(prog.VoidPtr()) + }, + Free: func(b Builder, _, _, _ Expr) { + b.Unreachable() + }, + }}) + mustPanicContains(t, "free callback terminated insertion block", coro.Finish) + }) +} + +func newCoroCallbackTestBuilder(t *testing.T) (Program, Builder) { + t.Helper() + prog := NewProgram(nil) + pkg := prog.NewPackage("badcorocallback", "bad/coro/callback") + fn := pkg.NewFunc("bad_callback", coroHandleSignature(), InC) + b := fn.MakeBody(1) + t.Cleanup(func() { + b.Dispose() + pkg.Module().Dispose() + prog.Dispose() + }) + return prog, b +} + +func newCoroTestFixture(t *testing.T, target *Target, allocationAlign uint32) *coroTestFixture { + t.Helper() + prog := NewProgram(target) + pkg := prog.NewPackage("corotest", "coro/test") + t.Cleanup(func() { + pkg.Module().Dispose() + prog.Dispose() + }) + + alloc := pkg.NewFunc("coro_frame_alloc", functionSignature( + []types.Type{types.Typ[types.Uintptr], types.Typ[types.Uintptr]}, + []types.Type{types.Typ[types.UnsafePointer]}, + ), InC) + free := pkg.NewFunc("coro_frame_free", functionSignature( + []types.Type{types.Typ[types.UnsafePointer], types.Typ[types.Uintptr], types.Typ[types.Uintptr]}, + nil, + ), InC) + sink := pkg.NewFunc("coro_value_sink", functionSignature([]types.Type{types.Typ[types.Uint8]}, nil), InC) + + fn := pkg.NewFunc("coro_test", coroHandleSignature(), InGo) + b := fn.MakeBody(1) + promise := b.AllocaT(prog.Byte()) + // Keep promise alignment distinct from AllocationAlign so the test guards + // llvm.coro.id's allocator-guarantee semantics rather than conflating them. + promise.impl.SetAlignment(16) + coro := b.BeginCoro(CoroOptions{ + Promise: promise, + AllocationAlign: allocationAlign, + Frame: CoroFrameOps{ + Alloc: func(b Builder, size, align Expr) Expr { + return b.Call(alloc.Expr, size, align) + }, + Free: func(b Builder, frame, size, align Expr) { + b.Call(free.Expr, frame, size, align) + }, + }, + }) + + live := b.AllocaT(prog.Byte()) + live.impl.SetAlignment(64) + b.Store(live, prog.IntVal(7, prog.Byte())) + coro.Suspend() + b.Call(sink.Expr, b.Load(live)) + coro.Finish() + b.EndBuild() + b.Dispose() + + return &coroTestFixture{prog: prog, pkg: pkg, fn: fn, coro: coro} +} + +func functionSignature(params, results []types.Type) *types.Signature { + makeTuple := func(values []types.Type) *types.Tuple { + vars := make([]*types.Var, len(values)) + for i, value := range values { + vars[i] = types.NewVar(token.NoPos, nil, "", value) + } + return types.NewTuple(vars...) + } + return types.NewSignatureType(nil, nil, nil, makeTuple(params), makeTuple(results), false) +} + +func coroHandleSignature() *types.Signature { + return functionSignature(nil, []types.Type{types.Typ[types.UnsafePointer]}) +} + +func runCoroPasses(t *testing.T, fixture *coroTestFixture, pipeline string) { + t.Helper() + mod := fixture.pkg.Module() + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify before %s: %v\n%s", pipeline, err, mod.String()) + } + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + options.SetVerifyEach(true) + if err := mod.RunPasses(pipeline, fixture.prog.TargetMachine(), options); err != nil { + t.Fatalf("run %s: %v\n%s", pipeline, err, mod.String()) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify after %s: %v\n%s", pipeline, err, mod.String()) + } +} + +func assertCoroSuspendDefaults(t *testing.T, fixture *coroTestFixture) { + t.Helper() + suspendID := llvm.LookupIntrinsicID("llvm.coro.suspend") + count := 0 + for _, block := range fixture.fn.impl.BasicBlocks() { + terminator := block.LastInstruction() + if terminator.IsNil() || terminator.IsASwitchInst().IsNil() { + continue + } + condition := terminator.Operand(0) + if condition.IsACallInst().IsNil() || condition.CalledValue().IntrinsicID() != suspendID { + continue + } + count++ + defaultBlock := terminator.Operand(1).AsBasicBlock() + if defaultBlock.C != fixture.coro.suspendBlk.first.C { + t.Fatalf("coro.suspend switch %d has a non-shared default block", count) + } + } + if count != 3 { + t.Fatalf("structured coro.suspend switches = %d, want 3", count) + } + cleanupTerminator := fixture.coro.cleanupBlk.last.LastInstruction() + if cleanupTerminator.IsNil() || cleanupTerminator.InstructionOpcode() != llvm.Br { + t.Fatal("coroutine cleanup block lacks its guarded-free branch") + } +} + +func countCoroEndCalls(ir string) int { + return strings.Count(ir, "call i1 @llvm.coro.end") + strings.Count(ir, "call void @llvm.coro.end") +} + +func frameAllocCallLine(ir string) string { + for _, line := range strings.Split(ir, "\n") { + if strings.Contains(line, "call") && strings.Contains(line, "@coro_frame_alloc") { + return line + } + } + return "" +} + +func mustPanicContains(t *testing.T, want string, fn func()) { + t.Helper() + defer func() { + got := recover() + if got == nil { + t.Fatalf("operation did not panic with %q", want) + } + if text := fmt.Sprint(got); !strings.Contains(text, want) { + t.Fatalf("panic = %q, want substring %q", text, want) + } + }() + fn() +}