diff --git a/.github/workflows/llgo.yml b/.github/workflows/llgo.yml index dadff854d0..df0f26d991 100644 --- a/.github/workflows/llgo.yml +++ b/.github/workflows/llgo.yml @@ -489,6 +489,15 @@ jobs: run_wasm_scheduler "$RUNNER_TEMP/wasm-scheduler.mjs" GOOS=wasip1 GOARCH=wasm llgo build -o "$RUNNER_TEMP/wasm-scheduler-wasip1.wasm" ./internal/build/testdata/wasm-scheduler run_wasi_scheduler "$RUNNER_TEMP/wasm-scheduler-wasip1.wasm" + LLGO_WASM_RESUME=1 GOOS=js GOARCH=wasm llgo build \ + -o "$RUNNER_TEMP/wasm-resume-scheduler-go.mjs" ./internal/build/testdata/wasm-scheduler + run_wasm_scheduler "$RUNNER_TEMP/wasm-resume-scheduler-go.mjs" + LLGO_WASM_RESUME=1 llgo build -target wasm \ + -o "$RUNNER_TEMP/wasm-resume-scheduler.mjs" ./internal/build/testdata/wasm-scheduler + run_wasm_scheduler "$RUNNER_TEMP/wasm-resume-scheduler.mjs" + LLGO_WASM_RESUME=1 GOOS=wasip1 GOARCH=wasm llgo build \ + -o "$RUNNER_TEMP/wasm-resume-scheduler-wasip1.wasm" ./internal/build/testdata/wasm-scheduler + run_wasi_scheduler "$RUNNER_TEMP/wasm-resume-scheduler-wasip1.wasm" file "$RUNNER_TEMP/runtime-js.wasm" \ "$RUNNER_TEMP/runtime-wasip1.wasm" \ "$RUNNER_TEMP/runtime-wasip1-threads.wasm" diff --git a/internal/build/build.go b/internal/build/build.go index f0b08fd43d..8784c32896 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -385,6 +385,9 @@ func Build(inv Invocation) ([]Package, error) { if conf.Target != "" && export.GOARCH != "" { conf.Goarch = export.GOARCH } + if err := configureWasmResume(conf, &export); err != nil { + return nil, err + } if conf.AppExt == "" { conf.AppExt = defaultAppExt(conf) } @@ -441,6 +444,7 @@ func Build(inv Invocation) ([]Package, error) { } prog := llssa.NewProgram(target) + prog.EnableWasmResumeABI(IsWasmResumeEnabled()) prog.DisableBoundsChecks(conf.DisableBoundsChecks) if conf.Mode != ModeGen { // ModeGen callers (llgen and the golden suites) read LPkg.String() @@ -1382,6 +1386,9 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa pcLineInfo: pcLineInfo, funcInfoStubs: funcInfoStubs, }) + if err := lowerWasmResumeModule(ctx, entryPkg.LPkg.Module()); err != nil { + return fmt.Errorf("entry main: %w", err) + } entryObjFile, err := exportObject(ctx, "entry_main", entryPkg.ExportFile, entryPkg.LPkg) if err != nil { return err @@ -1785,6 +1792,9 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { return nil } + if err := lowerWasmResumeModule(ctx, ret.Module()); err != nil { + return fmt.Errorf("%s: %w", pkgPath, err) + } ctx.cTransformer.SetSkipFuncs(cabiSkipFuncsForPlan9Asm(ctx, pkgPath, ret.Module())) llabi.LowerLargeAggregates(ctx.prog.TargetData(), ret.Module()) ctx.cTransformer.TransformModule(ret.Path(), ret.Module()) @@ -2365,6 +2375,7 @@ const llgoFuncInfoSites = "LLGO_FUNCINFO_SITES" const llgoTrace = "LLGO_TRACE" const llgoOptimize = "LLGO_OPTIMIZE" const llgoWasmRuntime = "LLGO_WASM_RUNTIME" +const llgoWasmResume = "LLGO_WASM_RESUME" const llgoWasiThreads = "LLGO_WASI_THREADS" const llgoStdioNobuf = "LLGO_STDIO_NOBUF" const llgoFullRpath = "LLGO_FULL_RPATH" @@ -2446,6 +2457,10 @@ func IsWasiThreadsEnabled() bool { return isEnvOn(llgoWasiThreads, false) } +func IsWasmResumeEnabled() bool { + return isEnvOn(llgoWasmResume, false) +} + func IsFullRpathEnabled() bool { return isEnvOn(llgoFullRpath, true) } diff --git a/internal/build/collect.go b/internal/build/collect.go index 66da887c21..9773ece09d 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -85,6 +85,7 @@ func (c *context) collectEnvInputs(m *manifestBuilder) { llgoTrace, llgoOptimize, llgoWasmRuntime, + llgoWasmResume, llgoWasiThreads, llgoStdioNobuf, llgoFullRpath, diff --git a/internal/build/main_module.go b/internal/build/main_module.go index 02d8ee9e84..7b09fb30ab 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -87,8 +87,9 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g pyFinalize = declareNoArgFunc(mainPkg, "Py_Finalize") } + wasmScheduler := ctx.crossCompile.WasmPostLink.Asyncify || ctx.prog.WasmResumeABIEnabled() var rtInit llssa.Function - if cfg.rtInit || ctx.crossCompile.WasmPostLink.Asyncify { + if cfg.rtInit || wasmScheduler { rtInit = declareNoArgFunc(mainPkg, rtPkgPath+".init") } @@ -109,8 +110,12 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g pkgPath = pkg.PkgPath } - mainInit := declareNoArgFunc(mainPkg, pkgPath+".init") - mainMain := declareNoArgFunc(mainPkg, pkgPath+".main") + mainBackground := llssa.InC + if ctx.prog.WasmResumeABIEnabled() { + mainBackground = llssa.InGo + } + mainInit := mainPkg.NewFunc(pkgPath+".init", llssa.NoArgsNoRet, mainBackground) + mainMain := mainPkg.NewFunc(pkgPath+".main", llssa.NoArgsNoRet, mainBackground) if ctx.buildConf.BuildMode != BuildModeExe { initArraySection := "" @@ -128,8 +133,8 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g } var wasmRunMain llssa.Function - if ctx.crossCompile.WasmPostLink.Asyncify { - defineWasmMainTask(mainPkg, mainInit, mainMain) + if wasmScheduler { + defineWasmMainTask(mainPkg, mainInit, mainMain, ctx.prog.WasmResumeABIEnabled()) wasmRunMain = declareNoArgFunc(mainPkg, rtPkgPath+".RunWasmMain") } entryFn := defineEntryFunction(ctx, mainPkg, argcVar, argvVar, argvValueType, entryFunctions{ @@ -295,13 +300,17 @@ func defineEntryFunction(ctx *context, pkg llssa.Package, argcVar, argvVar llssa return fn } -func defineWasmMainTask(pkg llssa.Package, mainInit, mainMain llssa.Function) { +func defineWasmMainTask(pkg llssa.Package, mainInit, mainMain llssa.Function, resumable bool) { prog := pkg.Prog sig := newSignature( []types.Type{types.Typ[types.UnsafePointer]}, []types.Type{types.Typ[types.UnsafePointer]}, ) - fn := pkg.NewFunc("__llgo_wasm_main", sig, llssa.InC) + background := llssa.InC + if resumable { + background = llssa.InGo + } + fn := pkg.NewFunc("__llgo_wasm_main", sig, background) fnVal := pkg.Module().NamedFunction("__llgo_wasm_main") fnVal.SetVisibility(llvm.HiddenVisibility) b := fn.MakeBody(1) diff --git a/internal/build/main_module_test.go b/internal/build/main_module_test.go index b577b12cdd..36bd82fa85 100644 --- a/internal/build/main_module_test.go +++ b/internal/build/main_module_test.go @@ -99,6 +99,40 @@ func TestGenMainModuleWASIAsyncifyEntry(t *testing.T) { } } +func TestGenMainModuleWasmResumeEntry(t *testing.T) { + llvm.InitializeAllTargets() + t.Setenv(llgoStdioNobuf, "") + prog := llssa.NewProgram(&llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + prog.EnableWasmResumeABI(true) + ctx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "wasip1", + Goarch: "wasm", + }, + } + pkg := &packages.Package{PkgPath: "example.com/foo", ExportFile: "foo.a"} + mod := genMainModule(ctx, llssa.PkgRuntime, pkg, &genConfig{}) + if err := lowerWasmResumeModule(ctx, mod.LPkg.Module()); err != nil { + t.Fatal(err) + } + ir := mod.LPkg.String() + for _, want := range []string{ + `define ptr @__llgo_wasm_start.__llgo_wasm_main`, + `define internal i8 @__llgo_wasm_resume.__llgo_wasm_main`, + `@"__llgo_wasm_resume_desc.example.com/foo.init" = external global`, + `@"__llgo_wasm_resume_desc.example.com/foo.main" = external global`, + `define hidden ptr @__llgo_wasm_main(ptr %0)`, + `call void @"github.com/goplus/llgo/runtime/internal/runtime.RunWasmMain"()`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("resumable main module IR missing %q:\n%s", want, ir) + } + } +} + func TestGenMainModuleLibrary(t *testing.T) { llvm.InitializeAllTargets() t.Setenv(llgoStdioNobuf, "") diff --git a/internal/build/wasm_postlink.go b/internal/build/wasm_postlink.go index b460f153b1..2a90340f71 100644 --- a/internal/build/wasm_postlink.go +++ b/internal/build/wasm_postlink.go @@ -29,17 +29,25 @@ import ( func needsWasmPostLink(conf *Config, target *crosscompile.Export) bool { return conf != nil && conf.BuildMode == BuildModeExe && - target != nil && target.WasmPostLink.Asyncify + target != nil && + (target.WasmPostLink.Asyncify || target.WasmPostLink.TranslateToExnref) } func wasmPostLinkArgs(target *crosscompile.Export, input, output string, debug bool) []string { - if target == nil || !target.WasmPostLink.Asyncify { + if target == nil || + (!target.WasmPostLink.Asyncify && !target.WasmPostLink.TranslateToExnref) { return nil } - // LLVM 19 lowers Wasm SjLj through the legacy EH encoding. Asyncify - // understands that form; translate it only after instrumentation so the - // final module uses the standardized exnref-based EH instructions. - args := []string{"--asyncify", "--translate-to-exnref"} + var args []string + if target.WasmPostLink.Asyncify { + args = append(args, "--asyncify") + } + // LLVM 19 lowers Wasm SjLj through the legacy EH encoding. When Asyncify + // is enabled, translate only after instrumentation so the final module + // uses the standardized exnref-based EH instructions. + if target.WasmPostLink.TranslateToExnref { + args = append(args, "--translate-to-exnref") + } if debug { args = append(args, "-g") } @@ -89,7 +97,7 @@ func postLinkWasm(ctx *context, input, output string, verbose bool) error { } resolved, err := exec.LookPath(wasmOpt) if err != nil { - return fmt.Errorf("WebAssembly Asyncify requires wasm-opt; install Binaryen or set WASMOPT: %w", err) + return fmt.Errorf("WebAssembly post-link requires wasm-opt; install Binaryen or set WASMOPT: %w", err) } tmpName, err := createClosedTemp( @@ -114,7 +122,7 @@ func postLinkWasm(ctx *context, input, output string, verbose bool) error { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - return fmt.Errorf("wasm-opt Asyncify failed: %w", err) + return fmt.Errorf("wasm-opt post-link failed: %w", err) } if err := os.Rename(tmpName, output); err != nil { return err diff --git a/internal/build/wasm_postlink_test.go b/internal/build/wasm_postlink_test.go index 0a00327425..406183dd38 100644 --- a/internal/build/wasm_postlink_test.go +++ b/internal/build/wasm_postlink_test.go @@ -33,7 +33,10 @@ func wasmPostLinkTestContext() *context { return &context{ buildConf: &Config{LinkOptions: LinkOptions{DWARF: DWARFOmit}}, crossCompile: crosscompile.Export{ - WasmPostLink: crosscompile.WasmPostLink{Asyncify: true}, + WasmPostLink: crosscompile.WasmPostLink{ + Asyncify: true, + TranslateToExnref: true, + }, }, } } @@ -51,7 +54,10 @@ func writeWasmOptTestTool(t *testing.T, dir, script string) string { } func TestWasmPostLinkArgs(t *testing.T) { - target := &crosscompile.Export{WasmPostLink: crosscompile.WasmPostLink{Asyncify: true}} + target := &crosscompile.Export{WasmPostLink: crosscompile.WasmPostLink{ + Asyncify: true, + TranslateToExnref: true, + }} if got, want := wasmPostLinkArgs(target, "in.wasm", "out.wasm", false), []string{"--asyncify", "--translate-to-exnref", "in.wasm", "-o", "out.wasm"}; !reflect.DeepEqual(got, want) { t.Fatalf("wasmPostLinkArgs() = %v, want %v", got, want) @@ -63,6 +69,11 @@ func TestWasmPostLinkArgs(t *testing.T) { if got := wasmPostLinkArgs(&crosscompile.Export{}, "in", "out", false); got != nil { t.Fatalf("wasmPostLinkArgs(disabled) = %v, want nil", got) } + target.WasmPostLink.Asyncify = false + if got, want := wasmPostLinkArgs(target, "in.wasm", "out.wasm", false), + []string{"--translate-to-exnref", "in.wasm", "-o", "out.wasm"}; !reflect.DeepEqual(got, want) { + t.Fatalf("wasmPostLinkArgs(translate only) = %v, want %v", got, want) + } } func TestNeedsWasmPostLink(t *testing.T) { @@ -87,6 +98,11 @@ func TestNeedsWasmPostLink(t *testing.T) { if needsWasmPostLink(&Config{BuildMode: BuildModeExe}, nil) { t.Fatal("needsWasmPostLink() enabled for a nil target") } + target.WasmPostLink.Asyncify = false + target.WasmPostLink.TranslateToExnref = true + if !needsWasmPostLink(&Config{BuildMode: BuildModeExe}, target) { + t.Fatal("needsWasmPostLink() disabled for exnref translation") + } } func TestPrepareWasmLinkOutput(t *testing.T) { @@ -196,7 +212,7 @@ func TestPostLinkWasmReportsToolFailure(t *testing.T) { ctx := wasmPostLinkTestContext() err := postLinkWasm(ctx, input, output, false) - if err == nil || !strings.Contains(err.Error(), "wasm-opt Asyncify failed") { + if err == nil || !strings.Contains(err.Error(), "wasm-opt post-link failed") { t.Fatalf("postLinkWasm() error = %v", err) } if data, err := os.ReadFile(output); err != nil || string(data) != "old" { @@ -223,7 +239,7 @@ func TestPostLinkWasmReportsPublishFailure(t *testing.T) { if err == nil { t.Fatal("postLinkWasm succeeded when the final output was a directory") } - if strings.Contains(err.Error(), "wasm-opt Asyncify failed") { + if strings.Contains(err.Error(), "wasm-opt post-link failed") { t.Fatalf("postLinkWasm failed before publishing output: %v", err) } } diff --git a/internal/build/wasm_resume.go b/internal/build/wasm_resume.go new file mode 100644 index 0000000000..16d0aa0d52 --- /dev/null +++ b/internal/build/wasm_resume.go @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "fmt" + "slices" + + "github.com/goplus/llgo/internal/crosscompile" + "github.com/goplus/llgo/internal/wasmresume" + "github.com/xgo-dev/llvm" +) + +const wasmResumeBuildTag = "llgo.wasm_resume" + +func configureWasmResume(conf *Config, export *crosscompile.Export) error { + if !IsWasmResumeEnabled() { + return nil + } + if conf == nil || conf.Goarch != "wasm" { + return fmt.Errorf("%s requires GOARCH=wasm", llgoWasmResume) + } + if conf.Goos != "js" && conf.Goos != "wasip1" { + return fmt.Errorf("%s does not support GOOS=%s", llgoWasmResume, conf.Goos) + } + if IsWasiThreadsEnabled() { + return fmt.Errorf("%s is incompatible with %s", llgoWasmResume, llgoWasiThreads) + } + if !slices.Contains(export.BuildTags, wasmResumeBuildTag) { + export.BuildTags = append(export.BuildTags, wasmResumeBuildTag) + } + export.WasmPostLink.Asyncify = false + export.LDFLAGS = slices.DeleteFunc(export.LDFLAGS, func(flag string) bool { + return flag == "-sASYNCIFY=1" + }) + return nil +} + +func lowerWasmResumeModule(ctx *context, mod llvm.Module) error { + if ctx == nil || !ctx.prog.WasmResumeABIEnabled() { + return nil + } + if err := wasmresume.Lower(mod, ctx.prog.TargetData()); err != nil { + return fmt.Errorf("lower WebAssembly resumable ABI: %w", err) + } + return nil +} diff --git a/internal/build/wasm_resume_test.go b/internal/build/wasm_resume_test.go new file mode 100644 index 0000000000..478575da1f --- /dev/null +++ b/internal/build/wasm_resume_test.go @@ -0,0 +1,134 @@ +package build + +import ( + "slices" + "strings" + "testing" + + "github.com/goplus/llgo/internal/crosscompile" + llssa "github.com/goplus/llgo/ssa" + "github.com/xgo-dev/llvm" +) + +func TestConfigureWasmResume(t *testing.T) { + t.Setenv(llgoWasmResume, "1") + t.Setenv(llgoWasiThreads, "") + export := crosscompile.Export{ + BuildTags: []string{"existing"}, + LDFLAGS: []string{"before", "-sASYNCIFY=1", "after"}, + WasmPostLink: crosscompile.WasmPostLink{ + Asyncify: true, + TranslateToExnref: true, + }, + } + conf := &Config{Goos: "wasip1", Goarch: "wasm"} + if err := configureWasmResume(conf, &export); err != nil { + t.Fatal(err) + } + if !slices.Contains(export.BuildTags, wasmResumeBuildTag) { + t.Fatalf("build tags = %v", export.BuildTags) + } + if export.WasmPostLink.Asyncify || slices.Contains(export.LDFLAGS, "-sASYNCIFY=1") { + t.Fatalf("Asyncify remains enabled: %+v", export) + } + if !export.WasmPostLink.TranslateToExnref { + t.Fatal("resumable WASI build disabled SjLj exception translation") + } + if err := configureWasmResume(conf, &export); err != nil { + t.Fatal(err) + } + count := 0 + for _, tag := range export.BuildTags { + if tag == wasmResumeBuildTag { + count++ + } + } + if count != 1 { + t.Fatalf("resumable build tag count = %d, tags = %v", count, export.BuildTags) + } +} + +func TestConfigureWasmResumeRejectsUnsupportedModes(t *testing.T) { + t.Setenv(llgoWasmResume, "1") + for _, test := range []struct { + name string + conf Config + threads bool + want string + }{ + {name: "native", conf: Config{Goos: "linux", Goarch: "amd64"}, want: "requires GOARCH=wasm"}, + {name: "host", conf: Config{Goos: "linux", Goarch: "wasm"}, want: "does not support GOOS=linux"}, + {name: "threads", conf: Config{Goos: "wasip1", Goarch: "wasm"}, threads: true, want: llgoWasiThreads}, + } { + t.Run(test.name, func(t *testing.T) { + if test.threads { + t.Setenv(llgoWasiThreads, "1") + } else { + t.Setenv(llgoWasiThreads, "") + } + err := configureWasmResume(&test.conf, &crosscompile.Export{}) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("configureWasmResume error = %v, want %q", err, test.want) + } + }) + } +} + +func TestLowerWasmResumeModule(t *testing.T) { + llvm.InitializeAllTargets() + prog := llssa.NewProgram(&llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + prog.EnableWasmResumeABI(true) + pkg := prog.NewPackage("p", "example.com/p") + callee := pkg.NewFunc("callee", llssa.NoArgsNoRet, llssa.InGo) + callee.MakeBody(1).Return() + caller := pkg.NewFunc("caller", llssa.NoArgsNoRet, llssa.InGo) + b := caller.MakeBody(1) + b.Call(callee.Expr) + b.Return() + + ctx := &context{prog: prog} + if err := lowerWasmResumeModule(ctx, pkg.Module()); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify lowered module: %v\n%s", err, pkg.String()) + } + for _, want := range []string{ + "define internal i8 @__llgo_wasm_resume.caller", + "define void @caller()", + "define ptr @__llgo_wasm_start.caller", + } { + if !strings.Contains(pkg.String(), want) { + t.Fatalf("lowered module is missing %q:\n%s", want, pkg.String()) + } + } +} + +func TestLowerWasmResumeModuleDisabled(t *testing.T) { + if err := lowerWasmResumeModule(nil, llvm.Module{}); err != nil { + t.Fatal(err) + } + prog := llssa.NewProgram(nil) + defer prog.Dispose() + ctx := &context{prog: prog} + if err := lowerWasmResumeModule(ctx, prog.NewPackage("p", "example.com/p").Module()); err != nil { + t.Fatal(err) + } +} + +func TestLowerWasmResumeModuleReportsLoweringError(t *testing.T) { + prog := llssa.NewProgram(&llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + prog.EnableWasmResumeABI(true) + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("native") + defer mod.Dispose() + mod.SetTarget("aarch64-apple-darwin") + + err := lowerWasmResumeModule(&context{prog: prog}, mod) + if err == nil || !strings.Contains(err.Error(), "target") { + t.Fatalf("lowerWasmResumeModule error = %v", err) + } +} diff --git a/internal/crosscompile/crosscompile.go b/internal/crosscompile/crosscompile.go index 69a42b93c6..6cab8e6076 100644 --- a/internal/crosscompile/crosscompile.go +++ b/internal/crosscompile/crosscompile.go @@ -51,7 +51,8 @@ type Export struct { // WasmPostLink describes transformations required after the core module is // linked. Build orchestration owns tool discovery and atomic output handling. type WasmPostLink struct { - Asyncify bool + Asyncify bool + TranslateToExnref bool } // DebugInfoPolicy describes how a selected linker handles debug information. @@ -434,6 +435,7 @@ func useWithJSWasm32(goos, goarch string, wasiThreads, forceEspClang bool, level ) } else { export.WasmPostLink.Asyncify = true + export.WasmPostLink.TranslateToExnref = true } case "js": diff --git a/internal/crosscompile/crosscompile_test.go b/internal/crosscompile/crosscompile_test.go index f811bf3e9b..f46b13c7b1 100644 --- a/internal/crosscompile/crosscompile_test.go +++ b/internal/crosscompile/crosscompile_test.go @@ -132,6 +132,9 @@ func TestUseCrossCompileSDK(t *testing.T) { if !export.WasmPostLink.Asyncify { t.Error("WASI target does not request Asyncify post-link processing") } + if !export.WasmPostLink.TranslateToExnref { + t.Error("WASI target does not request standardized exception encoding") + } if slices.Contains(export.LDFLAGS, "-Wl,--import-memory") { t.Errorf("single-worker WASI imports host memory: %v", export.LDFLAGS) } diff --git a/internal/wasmresume/abi.go b/internal/wasmresume/abi.go new file mode 100644 index 0000000000..37ea167cc7 --- /dev/null +++ b/internal/wasmresume/abi.go @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import ( + "fmt" + + "github.com/xgo-dev/llvm" +) + +const ( + resumeEntryPrefix = "__llgo_wasm_resume." + startEntryPrefix = "__llgo_wasm_start." + descriptorPrefix = "__llgo_wasm_resume_desc." + frameCloseName = "__llgo_wasm_resume_close" + actionContinue = 0 + actionReturn = 1 + actionSuspend = 2 +) + +// StartSymbol returns the resumable start entry for a Go function symbol. +func StartSymbol(function string) string { + return startEntryPrefix + function +} + +type resumeABI struct { + ctx llvm.Context + ptr llvm.Type + uintptrType llvm.Type + entryType llvm.Type + descriptorType llvm.Type + contextType llvm.Type +} + +func newResumeABI(ctx llvm.Context, targetData llvm.TargetData) resumeABI { + ptr := llvm.PointerType(ctx.Int8Type(), 0) + uintptrType := ctx.IntType(targetData.PointerSize() * 8) + return resumeABI{ + ctx: ctx, + ptr: ptr, + uintptrType: uintptrType, + entryType: llvm.FunctionType(ctx.Int8Type(), []llvm.Type{ptr, ptr}, false), + descriptorType: ctx.StructType([]llvm.Type{ + ptr, + uintptrType, + uintptrType, + uintptrType, + ctx.Int32Type(), + }, false), + // The first two fields are the public dispatch ABI. The trailing pointer + // is runtime-owned per-context frame storage. + contextType: ctx.StructType([]llvm.Type{ptr, ptr, ptr}, false), + } +} + +func (abi resumeABI) defineEntryAndDescriptor( + mod llvm.Module, layout frameLayout, +) (entry, descriptor llvm.Value, err error) { + fn := layout.plan.function + entryName := resumeEntryPrefix + fn.Name() + descriptorName := descriptorPrefix + fn.Name() + if !mod.NamedFunction(entryName).IsNil() || !mod.NamedGlobal(descriptorName).IsNil() { + return llvm.Value{}, llvm.Value{}, fmt.Errorf("%s: duplicate resumable descriptor", fn.Name()) + } + + entry = llvm.AddFunction(mod, entryName, abi.entryType) + entry.SetLinkage(llvm.InternalLinkage) + descriptor = llvm.AddGlobal(mod, abi.descriptorType, descriptorName) + descriptor.SetLinkage(fn.Linkage()) + descriptor.SetGlobalConstant(true) + descriptor.SetInitializer(abi.ctx.ConstStruct([]llvm.Value{ + entry, + llvm.ConstInt(abi.uintptrType, layout.size, false), + llvm.ConstInt(abi.uintptrType, uint64(layout.alignment), false), + llvm.ConstInt(abi.uintptrType, layout.unwindOffset, false), + llvm.ConstInt(abi.ctx.Int32Type(), uint64(layout.plan.unwindPC), false), + }, false)) + return entry, descriptor, nil +} diff --git a/internal/wasmresume/abi_test.go b/internal/wasmresume/abi_test.go new file mode 100644 index 0000000000..8a987bb17a --- /dev/null +++ b/internal/wasmresume/abi_test.go @@ -0,0 +1,131 @@ +package wasmresume + +import ( + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestDescriptorLinksAcrossModules(t *testing.T) { + llvm.InitializeAllTargetInfos() + llvm.InitializeAllTargets() + llvm.InitializeAllTargetMCs() + llvm.InitializeAllAsmPrinters() + + for _, triple := range []string{"wasm32-unknown-unknown", "wasm64-unknown-unknown"} { + t.Run(triple, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + + target, err := llvm.GetTargetFromTriple(triple) + if err != nil { + t.Fatal(err) + } + machine := target.CreateTargetMachine( + triple, "", "", llvm.CodeGenLevelNone, llvm.RelocDefault, llvm.CodeModelDefault, + ) + defer machine.Dispose() + targetData := machine.CreateTargetData() + defer targetData.Dispose() + + producer := ctx.NewModule("producer") + producerOwned := true + defer func() { + if producerOwned { + producer.Dispose() + } + }() + configureWasmModule(producer, triple, targetData) + + i32 := ctx.Int32Type() + sig := llvm.FunctionType(i32, []llvm.Type{i32}, false) + callee := llvm.AddFunction(producer, "example.com/dep.callee", sig) + callee.SetLinkage(llvm.LinkOnceAnyLinkage) + markFunction(ctx, callee) + block := ctx.AddBasicBlock(callee, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + builder.CreateRet(callee.Param(0)) + abi := newResumeABI(ctx, targetData) + startType := llvm.FunctionType( + abi.ptr, []llvm.Type{abi.ptr, i32}, false, + ) + startDeclaration := llvm.AddFunction( + producer, startEntryPrefix+callee.Name(), startType, + ) + + if _, err := lowerPrototype(producer, targetData); err != nil { + t.Fatal(err) + } + descriptorName := descriptorPrefix + callee.Name() + definedDescriptor := producer.NamedGlobal(descriptorName) + if definedDescriptor.IsNil() || definedDescriptor.Initializer().IsNil() { + t.Fatal("producer descriptor is not defined") + } + if got := definedDescriptor.Linkage(); got != llvm.LinkOnceAnyLinkage { + t.Fatalf("producer descriptor linkage = %v, want linkonce", got) + } + if startDeclaration.IsDeclaration() || + startDeclaration.Linkage() != llvm.LinkOnceAnyLinkage { + t.Fatal("producer did not define its predeclared start entry") + } + + consumer := ctx.NewModule("consumer") + defer consumer.Dispose() + configureWasmModule(consumer, triple, targetData) + calleeDeclaration := llvm.AddFunction(consumer, callee.Name(), sig) + markFunction(ctx, calleeDeclaration) + caller := llvm.AddFunction(consumer, "example.com/main.caller", sig) + markFunction(ctx, caller) + block = ctx.AddBasicBlock(caller, "entry") + builder.SetInsertPointAtEnd(block) + call := builder.CreateCall(sig, calleeDeclaration, []llvm.Value{caller.Param(0)}, "called") + markCall(ctx, call) + builder.CreateRet(call) + + if _, err := lowerPrototype(consumer, targetData); err != nil { + t.Fatal(err) + } + referencedDescriptor := consumer.NamedGlobal(descriptorName) + if referencedDescriptor.IsNil() || !referencedDescriptor.Initializer().IsNil() { + t.Fatal("consumer descriptor is not an external declaration") + } + requireWasmObject(t, machine, producer) + requireWasmObject(t, machine, consumer) + + if err := llvm.LinkModules(consumer, producer); err != nil { + t.Fatal(err) + } + producerOwned = false + linkedDescriptor := consumer.NamedGlobal(descriptorName) + if linkedDescriptor.IsNil() || linkedDescriptor.Initializer().IsNil() { + t.Fatal("linked descriptor remains unresolved") + } + if got := linkedDescriptor.Linkage(); got != llvm.LinkOnceAnyLinkage { + t.Fatalf("linked descriptor linkage = %v, want linkonce", got) + } + if err := llvm.VerifyModule(consumer, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify linked module: %v\n%s", err, consumer.String()) + } + requireWasmObject(t, machine, consumer) + }) + } +} + +func configureWasmModule(mod llvm.Module, triple string, targetData llvm.TargetData) { + mod.SetTarget(triple) + mod.SetDataLayout(targetData.String()) +} + +func requireWasmObject(t *testing.T, machine llvm.TargetMachine, mod llvm.Module) { + t.Helper() + object, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit %s: %v\n%s", mod.Target(), err, mod.String()) + } + defer object.Dispose() + if data := object.Bytes(); len(data) < 4 || string(data[:4]) != "\x00asm" { + t.Fatalf("%s object does not have the WebAssembly header", mod.Target()) + } +} diff --git a/internal/wasmresume/blockaddress.go b/internal/wasmresume/blockaddress.go new file mode 100644 index 0000000000..be0a782458 --- /dev/null +++ b/internal/wasmresume/blockaddress.go @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import "github.com/xgo-dev/llvm" + +type movedBlockAddress struct { + value llvm.Value + block llvm.BasicBlock +} + +func collectMovedBlockAddresses(function llvm.Value, blocks []llvm.BasicBlock) []movedBlockAddress { + found := make(map[llvm.Value]llvm.BasicBlock) + seen := make(map[llvm.Value]struct{}) + var visit func(llvm.Value) + visit = func(value llvm.Value) { + if value.IsNil() { + return + } + if _, ok := seen[value]; ok { + return + } + seen[value] = struct{}{} + if value.IsAUser().IsNil() { + return + } + if value.OperandsCount() == 2 && + value.Operand(0) == function && + value.Operand(1).IsBasicBlock() { + found[value] = value.Operand(1).AsBasicBlock() + return + } + if value.IsAConstant().IsNil() { + return + } + for i := 0; i < value.OperandsCount(); i++ { + visit(value.Operand(i)) + } + } + for _, block := range blocks { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + for i := 0; i < instruction.OperandsCount(); i++ { + visit(instruction.Operand(i)) + } + } + } + + addresses := make([]movedBlockAddress, 0, len(found)) + for value, block := range found { + addresses = append(addresses, movedBlockAddress{value: value, block: block}) + } + return addresses +} + +func remapMovedBlockAddresses(function llvm.Value, addresses []movedBlockAddress) { + for _, address := range addresses { + address.value.ReplaceAllUsesWith(llvm.BlockAddress(function, address.block)) + } +} diff --git a/internal/wasmresume/blockaddress_test.go b/internal/wasmresume/blockaddress_test.go new file mode 100644 index 0000000000..07b843b327 --- /dev/null +++ b/internal/wasmresume/blockaddress_test.go @@ -0,0 +1,50 @@ +package wasmresume + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestLowerRemapsBlockAddressesToResumeEntry(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("block-address") + defer mod.Dispose() + mod.SetTarget("wasm32-unknown-unknown") + targetData := llvm.NewTargetData("e-m:e-p:32:32-i64:64-n32:64-S128") + defer targetData.Dispose() + + callee := llvm.AddFunction(mod, "callee", llvm.FunctionType(ctx.VoidType(), nil, false)) + markFunction(ctx, callee) + calleeBlock := ctx.AddBasicBlock(callee, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(calleeBlock) + builder.CreateRetVoid() + + caller := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), nil, false)) + markFunction(ctx, caller) + entry := ctx.AddBasicBlock(caller, "entry") + target := ctx.AddBasicBlock(caller, "target") + builder.SetInsertPointAtEnd(entry) + call := builder.CreateCall(callee.GlobalValueType(), callee, nil, "") + markCall(ctx, call) + indirect := builder.CreateIndirectBr(llvm.BlockAddress(caller, target), 1) + indirect.AddDest(target) + builder.SetInsertPointAtEnd(target) + builder.CreateRetVoid() + + if err := Lower(mod, targetData); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify lowered module: %v\n%s", err, mod.String()) + } + ir := mod.String() + if strings.Contains(ir, "blockaddress(@caller,") || + !strings.Contains(ir, "blockaddress(@__llgo_wasm_resume.caller,") { + t.Fatalf("block address was not remapped to the resume entry:\n%s", ir) + } +} diff --git a/internal/wasmresume/boundary.go b/internal/wasmresume/boundary.go new file mode 100644 index 0000000000..2ee817d3c8 --- /dev/null +++ b/internal/wasmresume/boundary.go @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import "strings" + +const ( + runtimeResumePrefix = "github.com/goplus/llgo/runtime/internal/wasmresume." + runtimeAllocRoot = "github.com/goplus/llgo/runtime/internal/runtime.AllocRoot" + runtimeFreeRoot = "github.com/goplus/llgo/runtime/internal/runtime.FreeRoot" + runtimeRunWasmMain = "github.com/goplus/llgo/runtime/internal/runtime.RunWasmMain" + runtimeFrameAlloc = "__llgo_wasm_resume_alloc" + runtimeDynamicAlloc = "__llgo_wasm_resume_alloc_dynamic" + runtimeFrameFree = "__llgo_wasm_resume_free" + runtimeFrameClose = "__llgo_wasm_resume_close" +) + +// IsRuntimeABIImplementation reports functions which implement the resumable +// ABI itself and therefore cannot be lowered through that same ABI. +func IsRuntimeABIImplementation(name string) bool { + return strings.HasPrefix(name, runtimeResumePrefix) +} + +// IsNonSuspendingBoundary reports leaf runtime entry points which remain +// callable without allocating a resumable frame. +func IsNonSuspendingBoundary(name string) bool { + switch name { + case "github.com/goplus/llgo/runtime/internal/runtime.ClearThreadDefer", + "github.com/goplus/llgo/runtime/internal/runtime.FreeDeferNode", + "github.com/goplus/llgo/runtime/internal/runtime.GetThreadDefer", + "github.com/goplus/llgo/runtime/internal/runtime.Goexit", + "github.com/goplus/llgo/runtime/internal/runtime.Panic", + "github.com/goplus/llgo/runtime/internal/runtime.Recover", + "github.com/goplus/llgo/runtime/internal/runtime.Rethrow", + "github.com/goplus/llgo/runtime/internal/runtime.SetThreadDefer", + "runtime.Goexit": + return true + } + return (IsRuntimeABIImplementation(name) && name != SuspendSymbol) || + name == runtimeAllocRoot || + name == runtimeFreeRoot || + name == runtimeRunWasmMain || + name == runtimeFrameAlloc || + name == runtimeDynamicAlloc || + name == runtimeFrameFree || + name == runtimeFrameClose +} diff --git a/internal/wasmresume/boundary_test.go b/internal/wasmresume/boundary_test.go new file mode 100644 index 0000000000..9c445383f1 --- /dev/null +++ b/internal/wasmresume/boundary_test.go @@ -0,0 +1,44 @@ +package wasmresume + +import "testing" + +func TestRuntimeBoundaries(t *testing.T) { + for _, name := range []string{ + runtimeResumePrefix + "Context.Run", + runtimeResumePrefix + "Context.AllocateFrame", + } { + if !IsRuntimeABIImplementation(name) || !IsNonSuspendingBoundary(name) { + t.Fatalf("%q is not a non-suspending ABI implementation", name) + } + } + if !IsRuntimeABIImplementation(SuspendSymbol) { + t.Fatal("SuspendCurrent is not recognized as an ABI implementation") + } + if IsNonSuspendingBoundary(SuspendSymbol) { + t.Fatal("SuspendCurrent was classified as non-suspending") + } + for _, name := range []string{ + runtimeAllocRoot, + runtimeFreeRoot, + runtimeRunWasmMain, + runtimeFrameAlloc, + runtimeDynamicAlloc, + runtimeFrameFree, + runtimeFrameClose, + "github.com/goplus/llgo/runtime/internal/runtime.GetThreadDefer", + "github.com/goplus/llgo/runtime/internal/runtime.SetThreadDefer", + "github.com/goplus/llgo/runtime/internal/runtime.Panic", + "github.com/goplus/llgo/runtime/internal/runtime.Recover", + "github.com/goplus/llgo/runtime/internal/runtime.Rethrow", + "github.com/goplus/llgo/runtime/internal/runtime.Goexit", + "runtime.Goexit", + } { + if !IsNonSuspendingBoundary(name) { + t.Fatalf("%q is not a non-suspending boundary", name) + } + } + if IsRuntimeABIImplementation("example.com/p.Run") || + IsNonSuspendingBoundary("example.com/p.Run") { + t.Fatal("ordinary Go function was classified as a runtime boundary") + } +} diff --git a/internal/wasmresume/compat.go b/internal/wasmresume/compat.go new file mode 100644 index 0000000000..846fe12da9 --- /dev/null +++ b/internal/wasmresume/compat.go @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import ( + "fmt" + + "github.com/xgo-dev/llvm" +) + +// emitCompatibilityWrapper keeps the original Go symbol callable from +// non-resumable runtime and C boundaries. Such a call owns a temporary context +// and must run to completion; observing Suspend is a boundary violation. +func emitCompatibilityWrapper( + mod llvm.Module, targetData llvm.TargetData, abi resumeABI, lowered *loweredState, +) error { + fn := lowered.layout.plan.function + if !fn.IsDeclaration() { + return fmt.Errorf("%s: compatibility wrapper still has a body", fn.Name()) + } + + ctx := mod.Context() + entry := ctx.AddBasicBlock(fn, "wasm.resume.compat") + dispatch := ctx.AddBasicBlock(fn, "wasm.resume.dispatch") + resume := ctx.AddBasicBlock(fn, "wasm.resume.call") + continued := ctx.AddBasicBlock(fn, "wasm.resume.continue") + returned := ctx.AddBasicBlock(fn, "wasm.resume.return") + finished := ctx.AddBasicBlock(fn, "wasm.resume.finished") + suspended := ctx.AddBasicBlock(fn, "wasm.resume.suspended") + invalid := ctx.AddBasicBlock(fn, "wasm.resume.invalid") + + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(entry) + context := builder.CreateAlloca(abi.contextType, "resume.context") + context.SetAlignment(targetData.ABITypeAlignment(abi.contextType)) + root := builder.CreateAlloca(lowered.layout.typ, "resume.root") + root.SetAlignment(lowered.layout.alignment) + builder.CreateIntrinsic(ctx.VoidType(), llvm.LookupIntrinsicID("llvm.memset"), []llvm.Value{ + context, + llvm.ConstInt(ctx.Int8Type(), 0, false), + llvm.ConstInt(abi.uintptrType, targetData.TypeAllocSize(abi.contextType), false), + llvm.ConstInt(ctx.Int1Type(), 0, false), + }, "") + builder.CreateIntrinsic(ctx.VoidType(), llvm.LookupIntrinsicID("llvm.memset"), []llvm.Value{ + root, + llvm.ConstInt(ctx.Int8Type(), 0, false), + llvm.ConstInt(abi.uintptrType, lowered.layout.size, false), + llvm.ConstInt(ctx.Int1Type(), 0, false), + }, "") + builder.CreateStore( + llvm.ConstNull(abi.ptr), + builder.CreateStructGEP(lowered.layout.typ, root, 0, ""), + ) + builder.CreateStore( + lowered.descriptor, + builder.CreateStructGEP(lowered.layout.typ, root, 1, ""), + ) + for _, slot := range lowered.layout.plan.slots { + if slot.kind != slotParameter { + continue + } + builder.CreateStore( + fn.Param(parameterIndex(lowered.layout.plan, slot.id)), + builder.CreateStructGEP( + lowered.layout.typ, root, lowered.layout.fieldIndex(slot.id), "", + ), + ) + } + topField := builder.CreateStructGEP(abi.contextType, context, 0, "") + returnedField := builder.CreateStructGEP(abi.contextType, context, 1, "") + builder.CreateStore(root, topField) + builder.CreateBr(dispatch) + + framePrefix := ctx.StructType([]llvm.Type{abi.ptr, abi.ptr, ctx.Int32Type()}, false) + builder.SetInsertPointAtEnd(dispatch) + top := builder.CreateLoad(abi.ptr, topField, "top") + builder.CreateCondBr( + builder.CreateICmp(llvm.IntNE, top, llvm.ConstNull(abi.ptr), ""), + resume, + finished, + ) + + builder.SetInsertPointAtEnd(resume) + descriptor := builder.CreateLoad( + abi.ptr, builder.CreateStructGEP(framePrefix, top, 1, ""), "descriptor", + ) + resumeEntry := builder.CreateLoad( + abi.ptr, + builder.CreateStructGEP(abi.descriptorType, descriptor, 0, ""), + "resume.entry", + ) + action := builder.CreateCall(abi.entryType, resumeEntry, []llvm.Value{context, top}, "action") + actionSwitch := builder.CreateSwitch(action, invalid, 3) + actionSwitch.AddCase(llvm.ConstInt(ctx.Int8Type(), actionContinue, false), continued) + actionSwitch.AddCase(llvm.ConstInt(ctx.Int8Type(), actionReturn, false), returned) + actionSwitch.AddCase(llvm.ConstInt(ctx.Int8Type(), actionSuspend, false), suspended) + + builder.SetInsertPointAtEnd(continued) + builder.CreateBr(dispatch) + + builder.SetInsertPointAtEnd(returned) + parent := builder.CreateLoad( + abi.ptr, builder.CreateStructGEP(framePrefix, top, 0, ""), "parent", + ) + builder.CreateStore(parent, topField) + builder.CreateStore(top, returnedField) + builder.CreateBr(dispatch) + + builder.SetInsertPointAtEnd(finished) + builder.CreateCall( + declareFrameClose(mod, abi).GlobalValueType(), + declareFrameClose(mod, abi), + []llvm.Value{context}, + "", + ) + if lowered.layout.plan.resultSlot == 0 { + builder.CreateRetVoid() + } else { + result := builder.CreateLoad( + fn.GlobalValueType().ReturnType(), + builder.CreateStructGEP( + lowered.layout.typ, + root, + lowered.layout.fieldIndex(lowered.layout.plan.resultSlot), + "", + ), + "result", + ) + builder.CreateRet(result) + } + + builder.SetInsertPointAtEnd(suspended) + builder.CreateIntrinsic(ctx.VoidType(), llvm.LookupIntrinsicID("llvm.trap"), nil, "") + builder.CreateUnreachable() + builder.SetInsertPointAtEnd(invalid) + builder.CreateUnreachable() + return nil +} + +func parameterIndex(plan framePlan, slotID uint32) int { + index := 0 + for _, slot := range plan.slots { + if slot.kind != slotParameter { + continue + } + if slot.id == slotID { + return index + } + index++ + } + return -1 +} diff --git a/internal/wasmresume/dynamic.go b/internal/wasmresume/dynamic.go new file mode 100644 index 0000000000..f45a10097f --- /dev/null +++ b/internal/wasmresume/dynamic.go @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import ( + "fmt" + "strings" + + "github.com/xgo-dev/llvm" +) + +func lowerDynamicAlloca( + mod llvm.Module, + targetData llvm.TargetData, + abi resumeABI, + entry llvm.Value, + alloca llvm.Value, + field llvm.Value, +) error { + if alloca.IsAAllocaInst().IsNil() || alloca.OperandsCount() == 0 { + return fmt.Errorf("invalid dynamic alloca %q", alloca.Name()) + } + + ctx := mod.Context() + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointBefore(alloca) + + count := alloca.Operand(0) + switch { + case count.Type().IntTypeWidth() < abi.uintptrType.IntTypeWidth(): + count = builder.CreateZExt(count, abi.uintptrType, "alloca.count") + case count.Type().IntTypeWidth() > abi.uintptrType.IntTypeWidth(): + count = builder.CreateTrunc(count, abi.uintptrType, "alloca.count") + } + size := count + if elementSize := targetData.TypeAllocSize(alloca.AllocatedType()); elementSize != 1 { + size = builder.CreateMul( + count, + llvm.ConstInt(abi.uintptrType, elementSize, false), + "alloca.size", + ) + } + one := llvm.ConstInt(abi.uintptrType, 1, false) + size = builder.CreateSelect( + builder.CreateICmp(llvm.IntEQ, size, llvm.ConstNull(abi.uintptrType), ""), + one, + size, + "alloca.nonzero.size", + ) + align := targetData.ABITypeAlignment(alloca.AllocatedType()) + if alloca.Alignment() > align { + align = alloca.Alignment() + } + allocate := declareDynamicAllocator(mod, abi) + value := builder.CreateCall(allocate.GlobalValueType(), allocate, []llvm.Value{ + entry.Param(0), + size, + llvm.ConstInt(abi.uintptrType, uint64(align), false), + }, alloca.Name()+".frame") + store := builder.CreateStore(value, field) + replaceValueUsesWithLoads(ctx, alloca, field, store) + alloca.EraseFromParentAsInstruction() + return nil +} + +func isStackSave(value llvm.Value) bool { + return isCallToIntrinsic(value, "llvm.stacksave") +} + +func lowerPersistentStackSave(save llvm.Value) error { + var restores []llvm.Value + seen := make(map[llvm.Value]struct{}) + for use := save.FirstUse(); !use.IsNil(); use = use.NextUse() { + user := use.User() + if !isCallToIntrinsic(user, "llvm.stackrestore") { + return fmt.Errorf("persistent stacksave has unsupported use %q", user.Name()) + } + if _, ok := seen[user]; !ok { + seen[user] = struct{}{} + restores = append(restores, user) + } + } + for _, restore := range restores { + restore.EraseFromParentAsInstruction() + } + save.EraseFromParentAsInstruction() + return nil +} + +func isCallToIntrinsic(value llvm.Value, name string) bool { + if value.IsNil() || value.IsAInstruction().IsNil() || + value.InstructionOpcode() != llvm.Call { + return false + } + callee := value.CalledValue() + return !callee.IsAFunction().IsNil() && + (callee.Name() == name || strings.HasPrefix(callee.Name(), name+".")) +} + +func declareDynamicAllocator(mod llvm.Module, abi resumeABI) llvm.Value { + fn := mod.NamedFunction(frameDynamicAllocName) + if fn.IsNil() { + fn = llvm.AddFunction(mod, frameDynamicAllocName, llvm.FunctionType( + abi.ptr, []llvm.Type{abi.ptr, abi.uintptrType, abi.uintptrType}, false, + )) + } + return fn +} diff --git a/internal/wasmresume/dynamic_test.go b/internal/wasmresume/dynamic_test.go new file mode 100644 index 0000000000..a4e05afc53 --- /dev/null +++ b/internal/wasmresume/dynamic_test.go @@ -0,0 +1,99 @@ +package wasmresume + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestLowerMovesPersistentDynamicAllocaIntoContextStorage(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("dynamic-alloca") + defer mod.Dispose() + mod.SetTarget("wasm32-unknown-unknown") + targetData := llvm.NewTargetData("e-m:e-p:32:32-i64:64-n32:64-S128") + defer targetData.Dispose() + + i8 := ctx.Int8Type() + ptr := llvm.PointerType(i8, 0) + calleeType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr}, false) + callee := llvm.AddFunction(mod, "callee", calleeType) + markFunction(ctx, callee) + calleeBlock := ctx.AddBasicBlock(callee, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(calleeBlock) + builder.CreateRetVoid() + + caller := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), []llvm.Type{ctx.Int32Type()}, false)) + markFunction(ctx, caller) + block := ctx.AddBasicBlock(caller, "entry") + builder.SetInsertPointAtEnd(block) + buffer := builder.CreateArrayAlloca(i8, caller.Param(0), "buffer") + call := builder.CreateCall(calleeType, callee, []llvm.Value{buffer}, "") + markCall(ctx, call) + builder.CreateRetVoid() + + if err := Lower(mod, targetData); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify lowered module: %v\n%s", err, mod.String()) + } + ir := mod.String() + if strings.Contains(ir, "%buffer = alloca") || + !strings.Contains(ir, "call ptr @__llgo_wasm_resume_alloc_dynamic") { + t.Fatalf("dynamic alloca was not moved into context storage:\n%s", ir) + } +} + +func TestLowerRemovesStackLifetimeAcrossResume(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("stack-lifetime") + defer mod.Dispose() + mod.SetTarget("wasm32-unknown-unknown") + targetData := llvm.NewTargetData("e-m:e-p:32:32-i64:64-n32:64-S128") + defer targetData.Dispose() + + callee := llvm.AddFunction(mod, "callee", llvm.FunctionType(ctx.VoidType(), nil, false)) + markFunction(ctx, callee) + calleeBlock := ctx.AddBasicBlock(callee, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(calleeBlock) + builder.CreateRetVoid() + + caller := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), nil, false)) + markFunction(ctx, caller) + block := ctx.AddBasicBlock(caller, "entry") + builder.SetInsertPointAtEnd(block) + saved := builder.CreateIntrinsic( + llvm.PointerType(ctx.Int8Type(), 0), + llvm.LookupIntrinsicID("llvm.stacksave"), + nil, + "", + ) + call := builder.CreateCall(callee.GlobalValueType(), callee, nil, "") + markCall(ctx, call) + builder.CreateIntrinsic( + ctx.VoidType(), + llvm.LookupIntrinsicID("llvm.stackrestore"), + []llvm.Value{saved}, + "", + ) + builder.CreateRetVoid() + + if err := Lower(mod, targetData); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify lowered module: %v\n%s", err, mod.String()) + } + if ir := mod.String(); strings.Contains(ir, "call ptr @llvm.stacksave") || + strings.Contains(ir, "call void @llvm.stackrestore") { + t.Fatalf("native stack lifetime crosses a resume point:\n%s", ir) + } +} diff --git a/internal/wasmresume/execution_test.go b/internal/wasmresume/execution_test.go new file mode 100644 index 0000000000..0b8d31f869 --- /dev/null +++ b/internal/wasmresume/execution_test.go @@ -0,0 +1,179 @@ +package wasmresume + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestLowerExecutesRequiredWasmProfiles(t *testing.T) { + wasmLD, err := exec.LookPath("wasm-ld") + if err != nil { + t.Skip("wasm-ld is not installed") + } + node, nodeErr := exec.LookPath("node") + wasmtime, wasmtimeErr := exec.LookPath("wasmtime") + + llvm.InitializeAllTargetInfos() + llvm.InitializeAllTargets() + llvm.InitializeAllTargetMCs() + llvm.InitializeAllAsmPrinters() + + tests := []struct { + name string + triple string + run func(*testing.T, string) ([]byte, error) + }{ + { + name: "J32", + triple: "wasm32-unknown-emscripten", + run: func(t *testing.T, wasm string) ([]byte, error) { + if nodeErr != nil { + t.Skip("node is not installed") + } + script := `const fs=require("fs");WebAssembly.instantiate(fs.readFileSync(process.argv[1])).then(({instance})=>console.log(instance.exports["run.state.machine"]()))` + return exec.Command(node, "-e", script, wasm).CombinedOutput() + }, + }, + { + name: "J64", + triple: "wasm64-unknown-emscripten", + run: func(t *testing.T, wasm string) ([]byte, error) { + if nodeErr != nil { + t.Skip("node is not installed") + } + script := `const fs=require("fs");WebAssembly.instantiate(fs.readFileSync(process.argv[1])).then(({instance})=>console.log(instance.exports["run.state.machine"]()))` + return exec.Command(node, "-e", script, wasm).CombinedOutput() + }, + }, + { + name: "P1", + triple: "wasm32-unknown-wasip1", + run: func(t *testing.T, wasm string) ([]byte, error) { + if wasmtimeErr != nil { + t.Skip("wasmtime is not installed") + } + return exec.Command(wasmtime, "run", "--invoke", "run.state.machine", wasm).CombinedOutput() + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + object := buildExecutableWasmResumeObject(t, test.triple) + dir := t.TempDir() + objectPath := filepath.Join(dir, "resume.o") + wasmPath := filepath.Join(dir, "resume.wasm") + if err := os.WriteFile(objectPath, object, 0o600); err != nil { + t.Fatal(err) + } + linkArgs := []string{ + "--no-entry", + "--export=run.state.machine", + "-o", wasmPath, + objectPath, + } + if strings.HasPrefix(test.triple, "wasm64-") { + linkArgs = append([]string{"-mwasm64"}, linkArgs...) + } + if output, err := exec.Command(wasmLD, linkArgs...).CombinedOutput(); err != nil { + t.Fatalf("link %s: %v\n%s", test.name, err, output) + } + output, err := test.run(t, wasmPath) + if err != nil { + if test.name == "J64" && + strings.Contains(string(output), "invalid table elements limits flags") { + version, _ := exec.Command(node, "--version").CombinedOutput() + t.Skipf( + "node %s does not support LLVM wasm64 table limits", + strings.TrimSpace(string(version)), + ) + } + t.Fatalf("execute %s: %v\n%s", test.name, err, output) + } + fields := strings.Fields(string(output)) + if len(fields) == 0 || fields[len(fields)-1] != "14" { + t.Fatalf("%s result = %q, want 14", test.name, output) + } + }) + } +} + +func buildExecutableWasmResumeObject(t *testing.T, triple string) []byte { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule(triple) + defer mod.Dispose() + + target, err := llvm.GetTargetFromTriple(triple) + if err != nil { + t.Fatal(err) + } + machine := target.CreateTargetMachine( + triple, "", "", llvm.CodeGenLevelNone, llvm.RelocDefault, llvm.CodeModelDefault, + ) + defer machine.Dispose() + targetData := machine.CreateTargetData() + defer targetData.Dispose() + mod.SetTarget(triple) + mod.SetDataLayout(targetData.String()) + + i32 := ctx.Int32Type() + sig := llvm.FunctionType(i32, []llvm.Type{i32}, false) + callee := llvm.AddFunction(mod, "callee", sig) + markFunction(ctx, callee) + block := ctx.AddBasicBlock(callee, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + builder.CreateRet(builder.CreateAdd(callee.Param(0), llvm.ConstInt(i32, 1, false), "result")) + + suspend := llvm.AddFunction(mod, SuspendSymbol, llvm.FunctionType(ctx.VoidType(), nil, false)) + markFunction(ctx, suspend) + caller := llvm.AddFunction(mod, "caller", sig) + markFunction(ctx, caller) + block = ctx.AddBasicBlock(caller, "entry") + builder.SetInsertPointAtEnd(block) + call := builder.CreateCall(sig, callee, []llvm.Value{caller.Param(0)}, "called") + markCall(ctx, call) + suspendCall := builder.CreateCall(suspend.GlobalValueType(), suspend, nil, "") + markCall(ctx, suspendCall) + builder.CreateRet(builder.CreateMul(call, llvm.ConstInt(i32, 2, false), "result")) + + lowered, err := lowerPrototype(mod, targetData) + if err != nil { + t.Fatal(err) + } + var root loweredState + for _, state := range lowered { + if state.layout.plan.function == caller { + root = state + break + } + } + if root.entry.IsNil() { + t.Fatal("caller state machine was not emitted") + } + defineStateMachineHarness(mod, targetData, root, []llvm.Value{ + llvm.ConstInt(i32, 6, false), + }) + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify %s executable: %v\n%s", triple, err, mod.String()) + } + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + if err := mod.RunPasses("default", machine, options); err != nil { + t.Fatalf("optimize %s executable: %v\n%s", triple, err, mod.String()) + } + buffer, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit %s executable: %v\n%s", triple, err, mod.String()) + } + defer buffer.Dispose() + return append([]byte(nil), buffer.Bytes()...) +} diff --git a/internal/wasmresume/frameplan.go b/internal/wasmresume/frameplan.go new file mode 100644 index 0000000000..63b58a915f --- /dev/null +++ b/internal/wasmresume/frameplan.go @@ -0,0 +1,441 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import ( + "fmt" + "sort" + + "github.com/xgo-dev/llvm" +) + +type slotKind uint8 + +const ( + slotParameter slotKind = iota + slotFunctionResult + slotAlloca + slotValue + slotUnwind +) + +type frameSlot struct { + id uint32 + kind slotKind + typ llvm.Type + value llvm.Value + dynamic bool +} + +type callSite struct { + id uint32 + call llvm.Value + live []uint32 + resultSlot uint32 +} + +type framePlan struct { + function llvm.Value + slots []frameSlot + resultSlot uint32 + calls []callSite + unwindSlot uint32 + unwindPC uint32 + unwindBlock llvm.BasicBlock +} + +type blockLiveness struct { + def valueSet + use valueSet + liveIn valueSet + liveOut valueSet +} + +type valueSet map[llvm.Value]struct{} + +// planFrames computes the persistent values needed by each generated frame. +// Inventory runs first so every resumable call has a stable in-function ID. +func planFrames(mod llvm.Module) ([]framePlan, error) { + if _, err := Inventory(mod); err != nil { + return nil, err + } + + kind := mod.Context().MDKindID(CallMetadata) + var plans []framePlan + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + if !hasFunctionMarker(fn) { + continue + } + if err := llvm.VerifyFunction(fn, llvm.ReturnStatusAction); err != nil { + return nil, fmt.Errorf("%s: invalid resumable function: %w", fn.Name(), err) + } + plan, err := planFunctionFrame(fn, kind) + if err != nil { + return nil, fmt.Errorf("%s: %w", fn.Name(), err) + } + plans = append(plans, plan) + } + return plans, nil +} + +func planFunctionFrame(fn llvm.Value, metadataKind int) (framePlan, error) { + unwind, err := findUnwindPlan(fn) + if err != nil { + return framePlan{}, err + } + values, candidates, kinds := frameCandidates(fn) + blocks, liveness := analyzeLiveness(fn, candidates) + + var rawCalls []struct { + id uint32 + call llvm.Value + live valueSet + result llvm.Value + } + needed := make(valueSet) + if !unwind.block.IsNil() { + unionInto(needed, liveness[unwind.block].liveIn) + } + for _, block := range blocks { + live := cloneSet(liveness[block].liveOut) + for instr := block.LastInstruction(); !instr.IsNil(); instr = llvm.PrevInstruction(instr) { + if hasMetadata(instr, metadataKind) { + id, err := resumeID(instr.Metadata(metadataKind)) + if err != nil { + return framePlan{}, err + } + across := cloneSet(live) + var result llvm.Value + if _, ok := across[instr]; ok { + result = instr + delete(across, instr) + needed[instr] = struct{}{} + } + unionInto(across, referencedAllocas(instr, candidates)) + for value := range across { + needed[value] = struct{}{} + } + rawCalls = append(rawCalls, struct { + id uint32 + call llvm.Value + live valueSet + result llvm.Value + }{id: id, call: instr, live: across, result: result}) + } + + delete(live, instr) + if instr.InstructionOpcode() != llvm.PHI { + addLocalOperands(live, instr, candidates) + } + } + } + + sort.Slice(rawCalls, func(i, j int) bool { + return rawCalls[i].id < rawCalls[j].id + }) + + plan := framePlan{function: fn} + slots := make(map[llvm.Value]uint32) + addSlot := func(kind slotKind, typ llvm.Type, value llvm.Value, dynamic bool) uint32 { + id := uint32(len(plan.slots) + 1) + plan.slots = append(plan.slots, frameSlot{ + id: id, kind: kind, typ: typ, value: value, dynamic: dynamic, + }) + if !value.IsNil() { + slots[value] = id + } + return id + } + for _, value := range values { + if kinds[value] == slotParameter { + addSlot(slotParameter, value.Type(), value, false) + } + } + if typ := fn.GlobalValueType().ReturnType(); typ.TypeKind() != llvm.VoidTypeKind { + plan.resultSlot = addSlot(slotFunctionResult, typ, llvm.Value{}, false) + } + for _, value := range values { + if kinds[value] == slotParameter { + continue + } + if _, ok := needed[value]; !ok { + continue + } + typ, dynamic := persistentSlotType(value, kinds[value]) + addSlot(kinds[value], typ, value, dynamic) + } + if !unwind.block.IsNil() { + plan.unwindSlot = addSlot(slotUnwind, unwind.typ, llvm.Value{}, false) + plan.unwindPC = 1 + if len(rawCalls) != 0 { + plan.unwindPC = rawCalls[len(rawCalls)-1].id + 1 + } + if plan.unwindPC > maxResumeID { + return framePlan{}, fmt.Errorf("unwind state exceeds maximum resume ID") + } + plan.unwindBlock = unwind.block + } + + for _, raw := range rawCalls { + site := callSite{id: raw.id, call: raw.call} + for _, value := range values { + if _, ok := raw.live[value]; ok { + site.live = append(site.live, slots[value]) + } + } + if !raw.result.IsNil() { + site.resultSlot = slots[raw.result] + } + plan.calls = append(plan.calls, site) + } + return plan, nil +} + +type unwindPlan struct { + block llvm.BasicBlock + typ llvm.Type +} + +func findUnwindPlan(fn llvm.Value) (unwindPlan, error) { + var plan unwindPlan + for block := fn.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instr := block.FirstInstruction(); !instr.IsNil(); instr = llvm.NextInstruction(instr) { + if instr.InstructionOpcode() != llvm.Call || + instr.CalledValue().Name() != RegisterUnwindSymbol { + continue + } + if !plan.block.IsNil() { + return unwindPlan{}, fmt.Errorf("multiple unwind registrations") + } + if instr.OperandsCount() < 3 { + return unwindPlan{}, fmt.Errorf("invalid unwind registration") + } + address := instr.Operand(1) + if address.OperandsCount() != 2 || + address.Operand(0) != fn || + !address.Operand(1).IsBasicBlock() { + return unwindPlan{}, fmt.Errorf("invalid unwind handler") + } + plan.block = address.Operand(1).AsBasicBlock() + plan.typ = instr.Operand(0).Type() + } + } + return plan, nil +} + +func persistentSlotType(value llvm.Value, kind slotKind) (llvm.Type, bool) { + if kind != slotAlloca { + return value.Type(), false + } + elem := value.AllocatedType() + if value.OperandsCount() == 0 { + return elem, false + } + count := value.Operand(0).IsAConstantInt() + if count.IsNil() { + return value.Type(), true + } + n := count.ZExtValue() + if n == 1 { + return elem, false + } + return llvm.ArrayType(elem, int(n)), false +} + +func frameCandidates(fn llvm.Value) ([]llvm.Value, valueSet, map[llvm.Value]slotKind) { + var values []llvm.Value + candidates := make(valueSet) + kinds := make(map[llvm.Value]slotKind) + add := func(value llvm.Value, kind slotKind) { + values = append(values, value) + candidates[value] = struct{}{} + kinds[value] = kind + } + for param := fn.FirstParam(); !param.IsNil(); param = llvm.NextParam(param) { + add(param, slotParameter) + } + for block := fn.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instr := block.FirstInstruction(); !instr.IsNil(); instr = llvm.NextInstruction(instr) { + if instr.Type().TypeKind() == llvm.VoidTypeKind { + continue + } + kind := slotValue + if !instr.IsAAllocaInst().IsNil() { + kind = slotAlloca + } + add(instr, kind) + } + } + return values, candidates, kinds +} + +func analyzeLiveness(fn llvm.Value, candidates valueSet) ([]llvm.BasicBlock, map[llvm.BasicBlock]*blockLiveness) { + var blocks []llvm.BasicBlock + info := make(map[llvm.BasicBlock]*blockLiveness) + for block := fn.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + blocks = append(blocks, block) + state := &blockLiveness{ + def: make(valueSet), + use: make(valueSet), + liveIn: make(valueSet), + liveOut: make(valueSet), + } + info[block] = state + for instr := block.FirstInstruction(); !instr.IsNil(); instr = llvm.NextInstruction(instr) { + if instr.InstructionOpcode() != llvm.PHI { + for operand := range localOperands(instr, candidates) { + if _, defined := state.def[operand]; !defined { + state.use[operand] = struct{}{} + } + } + } + if _, ok := candidates[instr]; ok { + state.def[instr] = struct{}{} + } + } + } + + changed := true + for changed { + changed = false + for i := len(blocks) - 1; i >= 0; i-- { + block := blocks[i] + state := info[block] + out := make(valueSet) + terminator := block.LastInstruction() + for successorIndex := 0; successorIndex < terminator.SuccessorsCount(); successorIndex++ { + successor := terminator.Successor(successorIndex) + unionInto(out, info[successor].liveIn) + addPhiEdgeUses(out, successor, block, candidates) + } + in := cloneSet(state.use) + for value := range out { + if _, defined := state.def[value]; !defined { + in[value] = struct{}{} + } + } + if !equalSet(out, state.liveOut) || !equalSet(in, state.liveIn) { + state.liveOut = out + state.liveIn = in + changed = true + } + } + } + return blocks, info +} + +func addPhiEdgeUses(dst valueSet, successor, predecessor llvm.BasicBlock, candidates valueSet) { + for phi := successor.FirstInstruction(); !phi.IsNil() && phi.InstructionOpcode() == llvm.PHI; phi = llvm.NextInstruction(phi) { + for i := 0; i < phi.IncomingCount(); i++ { + value := phi.IncomingValue(i) + if phi.IncomingBlock(i) == predecessor { + if _, ok := candidates[value]; ok { + dst[value] = struct{}{} + } + } + } + } +} + +func localOperands(instr llvm.Value, candidates valueSet) valueSet { + operands := make(valueSet) + addLocalOperands(operands, instr, candidates) + return operands +} + +func addLocalOperands(dst valueSet, instr llvm.Value, candidates valueSet) { + for i := 0; i < instr.OperandsCount(); i++ { + operand := instr.Operand(i) + if _, ok := candidates[operand]; ok { + dst[operand] = struct{}{} + } + } +} + +func referencedAllocas(call llvm.Value, candidates valueSet) valueSet { + allocas := make(valueSet) + visited := make(valueSet) + var visit func(llvm.Value) + visit = func(value llvm.Value) { + if _, ok := visited[value]; ok { + return + } + visited[value] = struct{}{} + if !value.IsAAllocaInst().IsNil() { + allocas[value] = struct{}{} + return + } + if _, ok := candidates[value]; !ok { + return + } + if value.IsAInstruction().IsNil() { + return + } + switch value.InstructionOpcode() { + case llvm.Call, llvm.Load: + return + } + for i := 0; i < value.OperandsCount(); i++ { + visit(value.Operand(i)) + } + } + + callee := call.CalledValue() + for i := 0; i < call.OperandsCount(); i++ { + operand := call.Operand(i) + if operand != callee { + visit(operand) + } + } + return allocas +} + +func resumeID(marker llvm.Value) (uint32, error) { + fields := marker.MDNodeOperands() + if len(fields) != 2 || fields[1].IsAConstantInt().IsNil() { + return 0, fmt.Errorf("resumable call marker has no resume ID") + } + id := fields[1].ZExtValue() + if id == 0 || id > maxResumeID { + return 0, fmt.Errorf("invalid resume ID %d", id) + } + return uint32(id), nil +} + +func cloneSet(src valueSet) valueSet { + dst := make(valueSet, len(src)) + unionInto(dst, src) + return dst +} + +func unionInto(dst, src valueSet) { + for value := range src { + dst[value] = struct{}{} + } +} + +func equalSet(a, b valueSet) bool { + if len(a) != len(b) { + return false + } + for value := range a { + if _, ok := b[value]; !ok { + return false + } + } + return true +} diff --git a/internal/wasmresume/frameplan_test.go b/internal/wasmresume/frameplan_test.go new file mode 100644 index 0000000000..64f0467f90 --- /dev/null +++ b/internal/wasmresume/frameplan_test.go @@ -0,0 +1,373 @@ +package wasmresume + +import ( + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestPlanFramesStraightLineValues(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("straight") + defer mod.Dispose() + + i32 := ctx.Int32Type() + callee := llvm.AddFunction(mod, "callee", llvm.FunctionType(i32, []llvm.Type{i32}, false)) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(i32, []llvm.Type{i32}, false)) + markFunction(ctx, fn) + fn.Param(0).SetName("input") + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + before := builder.CreateAdd(fn.Param(0), llvm.ConstInt(i32, 1, false), "before") + call := builder.CreateCall(callee.GlobalValueType(), callee, []llvm.Value{before}, "result") + markCall(ctx, call) + after := builder.CreateAdd(before, call, "after") + builder.CreateRet(after) + + plans, err := planFrames(mod) + if err != nil { + t.Fatal(err) + } + plan := onlyFramePlan(t, plans) + assertSlots(t, plan, []slotWant{ + {name: "input", kind: slotParameter}, + {kind: slotFunctionResult}, + {name: "before", kind: slotValue}, + {name: "result", kind: slotValue}, + }) + if len(plan.calls) != 1 { + t.Fatalf("calls = %+v", plan.calls) + } + if got, want := plan.calls[0].live, []uint32{3}; !equalIDs(got, want) { + t.Fatalf("live slots = %v, want %v", got, want) + } + if plan.resultSlot != 2 { + t.Fatalf("function result slot = %d, want 2", plan.resultSlot) + } + if plan.calls[0].resultSlot != 4 { + t.Fatalf("call result slot = %d, want 4", plan.calls[0].resultSlot) + } +} + +func TestPlanFramesKeepsParameterAndAllocaAcrossCall(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("alloca") + defer mod.Dispose() + + i32 := ctx.Int32Type() + voidFn := llvm.FunctionType(ctx.VoidType(), nil, false) + callee := llvm.AddFunction(mod, "callee", voidFn) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(i32, []llvm.Type{i32}, false)) + markFunction(ctx, fn) + fn.Param(0).SetName("input") + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + local := builder.CreateAlloca(i32, "local") + builder.CreateStore(fn.Param(0), local) + call := builder.CreateCall(voidFn, callee, nil, "") + markCall(ctx, call) + loaded := builder.CreateLoad(i32, local, "loaded") + after := builder.CreateAdd(fn.Param(0), loaded, "after") + builder.CreateRet(after) + + plans, err := planFrames(mod) + if err != nil { + t.Fatal(err) + } + plan := onlyFramePlan(t, plans) + assertSlots(t, plan, []slotWant{ + {name: "input", kind: slotParameter}, + {kind: slotFunctionResult}, + {name: "local", kind: slotAlloca}, + }) + if got, want := plan.calls[0].live, []uint32{1, 3}; !equalIDs(got, want) { + t.Fatalf("live slots = %v, want %v", got, want) + } + if plan.calls[0].resultSlot != 0 { + t.Fatalf("void call result slot = %d, want 0", plan.calls[0].resultSlot) + } + if plan.slots[2].typ != i32 || plan.slots[2].dynamic { + t.Fatalf("static alloca slot = %+v, want embedded i32", plan.slots[2]) + } +} + +func TestPlanFramesKeepsAllocaReferencedOnlyByCallArgument(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("alloca-argument") + defer mod.Dispose() + + i32 := ctx.Int32Type() + ptr := llvm.PointerType(i32, 0) + calleeType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr}, false) + callee := llvm.AddFunction(mod, "callee", calleeType) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), nil, false)) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + local := builder.CreateAlloca(i32, "local") + derived := builder.CreateGEP(i32, local, []llvm.Value{llvm.ConstInt(ctx.Int32Type(), 0, false)}, "derived") + call := builder.CreateCall(calleeType, callee, []llvm.Value{derived}, "") + markCall(ctx, call) + builder.CreateRetVoid() + + plans, err := planFrames(mod) + if err != nil { + t.Fatal(err) + } + plan := onlyFramePlan(t, plans) + assertSlots(t, plan, []slotWant{{name: "local", kind: slotAlloca}}) + if got, want := plan.calls[0].live, []uint32{1}; !equalIDs(got, want) { + t.Fatalf("live slots = %v, want %v", got, want) + } +} + +func TestPlanFramesMarksDynamicAllocaStorage(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("dynamic-alloca") + defer mod.Dispose() + + i32 := ctx.Int32Type() + ptr := llvm.PointerType(i32, 0) + calleeType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr}, false) + callee := llvm.AddFunction(mod, "callee", calleeType) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), []llvm.Type{i32}, false)) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + local := builder.CreateArrayAlloca(i32, fn.Param(0), "local") + call := builder.CreateCall(calleeType, callee, []llvm.Value{local}, "") + markCall(ctx, call) + builder.CreateRetVoid() + + plans, err := planFrames(mod) + if err != nil { + t.Fatal(err) + } + plan := onlyFramePlan(t, plans) + assertSlots(t, plan, []slotWant{ + {kind: slotParameter}, + {name: "local", kind: slotAlloca}, + }) + if plan.slots[1].typ != ptr || !plan.slots[1].dynamic { + t.Fatalf("dynamic alloca slot = %+v, want pointer storage", plan.slots[1]) + } +} + +func TestPlanFramesDoesNotPersistCopiedCallArguments(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("copied-arguments") + defer mod.Dispose() + + i32 := ctx.Int32Type() + calleeType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{i32, i32}, false) + callee := llvm.AddFunction(mod, "callee", calleeType) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), []llvm.Type{i32}, false)) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + local := builder.CreateAlloca(i32, "local") + builder.CreateStore(llvm.ConstInt(i32, 7, false), local) + loaded := builder.CreateLoad(i32, local, "loaded") + call := builder.CreateCall(calleeType, callee, []llvm.Value{fn.Param(0), loaded}, "") + markCall(ctx, call) + builder.CreateRetVoid() + + plans, err := planFrames(mod) + if err != nil { + t.Fatal(err) + } + plan := onlyFramePlan(t, plans) + assertSlots(t, plan, []slotWant{{kind: slotParameter}}) + if len(plan.calls[0].live) != 0 { + t.Fatalf("copied arguments created persistent slots: %+v", plan) + } +} + +func TestPlanFramesTracksPhiUseOnPredecessorEdge(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("phi") + defer mod.Dispose() + + i1 := ctx.Int1Type() + i32 := ctx.Int32Type() + voidFn := llvm.FunctionType(ctx.VoidType(), nil, false) + callee := llvm.AddFunction(mod, "callee", voidFn) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(i32, []llvm.Type{i1, i32}, false)) + markFunction(ctx, fn) + fn.Param(1).SetName("input") + entry := ctx.AddBasicBlock(fn, "entry") + left := ctx.AddBasicBlock(fn, "left") + right := ctx.AddBasicBlock(fn, "right") + merge := ctx.AddBasicBlock(fn, "merge") + builder := ctx.NewBuilder() + defer builder.Dispose() + + builder.SetInsertPointAtEnd(entry) + builder.CreateCondBr(fn.Param(0), left, right) + builder.SetInsertPointAtEnd(left) + call := builder.CreateCall(voidFn, callee, nil, "") + markCall(ctx, call) + builder.CreateBr(merge) + builder.SetInsertPointAtEnd(right) + builder.CreateBr(merge) + builder.SetInsertPointAtEnd(merge) + phi := builder.CreatePHI(i32, "selected") + phi.AddIncoming( + []llvm.Value{fn.Param(1), llvm.ConstInt(i32, 0, false)}, + []llvm.BasicBlock{left, right}, + ) + builder.CreateRet(phi) + + plans, err := planFrames(mod) + if err != nil { + t.Fatal(err) + } + plan := onlyFramePlan(t, plans) + assertSlots(t, plan, []slotWant{ + {kind: slotParameter}, + {name: "input", kind: slotParameter}, + {kind: slotFunctionResult}, + }) + if got, want := plan.calls[0].live, []uint32{2}; !equalIDs(got, want) { + t.Fatalf("live slots = %v, want %v", got, want) + } +} + +func TestPlanFramesIncludesMarkedLeaf(t *testing.T) { + ctx, mod, _, builder := newInventoryTestFunction(t, true) + defer ctx.Dispose() + defer mod.Dispose() + defer builder.Dispose() + builder.CreateRetVoid() + + plans, err := planFrames(mod) + if err != nil { + t.Fatal(err) + } + plan := onlyFramePlan(t, plans) + if len(plan.slots) != 0 || len(plan.calls) != 0 { + t.Fatalf("leaf plan = %+v", plan) + } +} + +func TestPlanFramesReservesLeafParametersAndResult(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("leaf-abi") + defer mod.Dispose() + + i32 := ctx.Int32Type() + fn := llvm.AddFunction(mod, "leaf", llvm.FunctionType(i32, []llvm.Type{i32}, false)) + markFunction(ctx, fn) + fn.Param(0).SetName("input") + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + builder.CreateRet(fn.Param(0)) + + plans, err := planFrames(mod) + if err != nil { + t.Fatal(err) + } + plan := onlyFramePlan(t, plans) + assertSlots(t, plan, []slotWant{ + {name: "input", kind: slotParameter}, + {kind: slotFunctionResult}, + }) + if plan.resultSlot != 2 { + t.Fatalf("result slot = %d, want 2", plan.resultSlot) + } +} + +func TestPlanFramesOrdersCallsByResumeID(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("order") + defer mod.Dispose() + + voidFn := llvm.FunctionType(ctx.VoidType(), nil, false) + callee := llvm.AddFunction(mod, "callee", voidFn) + fn := llvm.AddFunction(mod, "caller", voidFn) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + first := builder.CreateCall(voidFn, callee, nil, "") + markCall(ctx, first) + second := builder.CreateCall(voidFn, callee, nil, "") + markCall(ctx, second) + builder.CreateRetVoid() + + plans, err := planFrames(mod) + if err != nil { + t.Fatal(err) + } + calls := onlyFramePlan(t, plans).calls + if len(calls) != 2 || calls[0].id != 1 || calls[1].id != 2 { + t.Fatalf("calls = %+v", calls) + } +} + +type slotWant struct { + name string + kind slotKind +} + +func onlyFramePlan(t *testing.T, plans []framePlan) framePlan { + t.Helper() + if len(plans) != 1 { + t.Fatalf("plans = %+v", plans) + } + return plans[0] +} + +func assertSlots(t *testing.T, plan framePlan, want []slotWant) { + t.Helper() + if len(plan.slots) != len(want) { + t.Fatalf("slots = %+v, want %+v", plan.slots, want) + } + for i, slot := range plan.slots { + name := "" + if !slot.value.IsNil() { + name = slot.value.Name() + } + if slot.id != uint32(i+1) || name != want[i].name || slot.kind != want[i].kind { + t.Fatalf("slot %d = {id:%d name:%q kind:%d}, want {id:%d name:%q kind:%d}", + i, slot.id, name, slot.kind, i+1, want[i].name, want[i].kind) + } + } +} + +func equalIDs(a, b []uint32) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func markFunction(ctx llvm.Context, fn llvm.Value) { + fn.AddFunctionAttr(ctx.CreateStringAttribute(FunctionAttribute, "1")) +} diff --git a/internal/wasmresume/inventory.go b/internal/wasmresume/inventory.go new file mode 100644 index 0000000000..9c5889cc34 --- /dev/null +++ b/internal/wasmresume/inventory.go @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package wasmresume plans the compiler half of LLGo's experimental +// WebAssembly resumable call ABI. +package wasmresume + +import ( + "fmt" + + "github.com/xgo-dev/llvm" +) + +const ( + FunctionAttribute = "llgo.wasm.resume" + CallMetadata = "llgo.wasm.resume.call" + SuspendSymbol = "github.com/goplus/llgo/runtime/internal/wasmresume.SuspendCurrent" + RegisterUnwindSymbol = "github.com/goplus/llgo/runtime/internal/wasmresume.RegisterUnwind" + ClearUnwindSymbol = "github.com/goplus/llgo/runtime/internal/wasmresume.ClearUnwind" + MarkerVersion = 1 + maxResumeID = 1<<16 - 1 +) + +// Function describes the resumable calls in one generated Go function. +type Function struct { + Name string + Calls []Call +} + +// Call describes one generated Go call and its in-function resume ID. +type Call struct { + ID uint32 + Callee string + Indirect bool +} + +// Inventory scans the actual LLVM calls produced by the frontend and assigns +// deterministic, function-local resume IDs. +func Inventory(mod llvm.Module) ([]Function, error) { + ctx := mod.Context() + kind := ctx.MDKindID(CallMetadata) + var functions []Function + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + markedFunction := hasFunctionMarker(fn) + var calls []Call + for block := fn.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instr := block.FirstInstruction(); !instr.IsNil(); instr = llvm.NextInstruction(instr) { + if !hasMetadata(instr, kind) { + continue + } + if instr.InstructionOpcode() != llvm.Call { + return nil, fmt.Errorf("%s: resumable marker is attached to a non-call instruction", fn.Name()) + } + if !markedFunction { + return nil, fmt.Errorf("%s: resumable call is in an unmarked function", fn.Name()) + } + if err := validateCallMarker(instr.Metadata(kind)); err != nil { + return nil, fmt.Errorf("%s: %w", fn.Name(), err) + } + if len(calls) == maxResumeID { + return nil, fmt.Errorf("%s: too many resumable calls", fn.Name()) + } + target := instr.CalledValue() + callee := "" + if !target.IsAFunction().IsNil() { + callee = target.Name() + } + call := Call{ + ID: uint32(len(calls) + 1), + Callee: callee, + Indirect: callee == "", + } + calls = append(calls, call) + setCallMarker(ctx, instr, kind, call.ID) + } + } + if markedFunction { + functions = append(functions, Function{Name: fn.Name(), Calls: calls}) + } + } + return functions, nil +} + +func hasFunctionMarker(fn llvm.Value) bool { + for _, attr := range fn.GetFunctionAttributes() { + if attr.IsString() && attr.GetStringKind() == FunctionAttribute { + return attr.GetStringValue() == "1" + } + } + return false +} + +func hasMetadata(instr llvm.Value, kind int) bool { + return instr.HasMetadata() && !instr.Metadata(kind).IsNil() +} + +func validateCallMarker(marker llvm.Value) error { + fields := marker.MDNodeOperands() + if len(fields) < 1 || len(fields) > 2 || fields[0].IsAConstantInt().IsNil() || + fields[0].ZExtValue() != MarkerVersion { + return fmt.Errorf("invalid resumable call marker") + } + return nil +} + +func setCallMarker(ctx llvm.Context, call llvm.Value, kind int, id uint32) { + i32 := ctx.Int32Type() + fields := []llvm.Metadata{ + llvm.ConstInt(i32, MarkerVersion, false).ConstantAsMetadata(), + llvm.ConstInt(i32, uint64(id), false).ConstantAsMetadata(), + } + call.SetMetadata(kind, ctx.MDNode(fields)) +} diff --git a/internal/wasmresume/inventory_test.go b/internal/wasmresume/inventory_test.go new file mode 100644 index 0000000000..30aebe7775 --- /dev/null +++ b/internal/wasmresume/inventory_test.go @@ -0,0 +1,152 @@ +package wasmresume + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestInventoryNumbersDirectAndIndirectCalls(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("resume") + defer mod.Dispose() + + voidFn := llvm.FunctionType(ctx.VoidType(), nil, false) + callee := llvm.AddFunction(mod, "callee", voidFn) + callerType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{callee.Type()}, false) + fn := llvm.AddFunction(mod, "caller", callerType) + fn.AddFunctionAttr(ctx.CreateStringAttribute(FunctionAttribute, "1")) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + + direct := builder.CreateCall(voidFn, callee, nil, "") + markCall(ctx, direct) + target := fn.Param(0) + indirect := builder.CreateCall(voidFn, target, nil, "") + markCall(ctx, indirect) + builder.CreateRetVoid() + + functions, err := Inventory(mod) + if err != nil { + t.Fatal(err) + } + if len(functions) != 1 || functions[0].Name != "caller" { + t.Fatalf("functions = %+v", functions) + } + calls := functions[0].Calls + if len(calls) != 2 { + t.Fatalf("calls = %+v", calls) + } + if calls[0].ID != 1 || calls[0].Callee != "callee" || calls[0].Indirect { + t.Fatalf("direct call = %+v", calls[0]) + } + if calls[1].ID != 2 || calls[1].Callee != "" || !calls[1].Indirect { + t.Fatalf("indirect call = %+v", calls[1]) + } + + ir := mod.String() + if !strings.Contains(ir, "!"+CallMetadata+" !0") || + !strings.Contains(ir, "!"+CallMetadata+" !1") || + !strings.Contains(ir, "!0 = !{i32 1, i32 1}") || + !strings.Contains(ir, "!1 = !{i32 1, i32 2}") { + t.Fatalf("resume IDs were not written to call metadata:\n%s", ir) + } +} + +func TestInventoryRejectsMarkerInUnmarkedFunction(t *testing.T) { + ctx, mod, fn, builder := newInventoryTestFunction(t, false) + defer ctx.Dispose() + defer mod.Dispose() + defer builder.Dispose() + call := builder.CreateCall(llvm.FunctionType(ctx.VoidType(), nil, false), fn, nil, "") + markCall(ctx, call) + builder.CreateRetVoid() + + if _, err := Inventory(mod); err == nil || !strings.Contains(err.Error(), "unmarked function") { + t.Fatalf("Inventory error = %v", err) + } +} + +func TestInventoryRejectsMarkerOnNonCall(t *testing.T) { + ctx, mod, _, builder := newInventoryTestFunction(t, true) + defer ctx.Dispose() + defer mod.Dispose() + defer builder.Dispose() + ret := builder.CreateRetVoid() + markCall(ctx, ret) + + if _, err := Inventory(mod); err == nil || !strings.Contains(err.Error(), "non-call") { + t.Fatalf("Inventory error = %v", err) + } +} + +func TestInventoryIgnoresUnmarkedDeclarations(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("empty") + defer mod.Dispose() + llvm.AddFunction(mod, "declaration", llvm.FunctionType(ctx.VoidType(), nil, false)) + functions, err := Inventory(mod) + if err != nil { + t.Fatal(err) + } + if len(functions) != 0 { + t.Fatalf("functions = %+v, want empty", functions) + } +} + +func TestInventoryIncludesMarkedLeafFunction(t *testing.T) { + ctx, mod, _, builder := newInventoryTestFunction(t, true) + defer ctx.Dispose() + defer mod.Dispose() + defer builder.Dispose() + builder.CreateRetVoid() + + functions, err := Inventory(mod) + if err != nil { + t.Fatal(err) + } + if len(functions) != 1 || functions[0].Name != "function" || len(functions[0].Calls) != 0 { + t.Fatalf("functions = %+v", functions) + } +} + +func TestInventoryRejectsInvalidMarker(t *testing.T) { + ctx, mod, fn, builder := newInventoryTestFunction(t, true) + defer ctx.Dispose() + defer mod.Dispose() + defer builder.Dispose() + call := builder.CreateCall(llvm.FunctionType(ctx.VoidType(), nil, false), fn, nil, "") + kind := ctx.MDKindID(CallMetadata) + version := llvm.ConstInt(ctx.Int32Type(), MarkerVersion+1, false).ConstantAsMetadata() + call.SetMetadata(kind, ctx.MDNode([]llvm.Metadata{version})) + builder.CreateRetVoid() + + if _, err := Inventory(mod); err == nil || !strings.Contains(err.Error(), "invalid resumable call marker") { + t.Fatalf("Inventory error = %v", err) + } +} + +func newInventoryTestFunction(t *testing.T, marked bool) (llvm.Context, llvm.Module, llvm.Value, llvm.Builder) { + t.Helper() + ctx := llvm.NewContext() + mod := ctx.NewModule("invalid") + fn := llvm.AddFunction(mod, "function", llvm.FunctionType(ctx.VoidType(), nil, false)) + if marked { + fn.AddFunctionAttr(ctx.CreateStringAttribute(FunctionAttribute, "1")) + } + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + builder.SetInsertPointAtEnd(block) + return ctx, mod, fn, builder +} + +func markCall(ctx llvm.Context, instr llvm.Value) { + kind := ctx.MDKindID(CallMetadata) + version := llvm.ConstInt(ctx.Int32Type(), MarkerVersion, false).ConstantAsMetadata() + instr.SetMetadata(kind, ctx.MDNode([]llvm.Metadata{version})) +} diff --git a/internal/wasmresume/ir.go b/internal/wasmresume/ir.go new file mode 100644 index 0000000000..917e912917 --- /dev/null +++ b/internal/wasmresume/ir.go @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import ( + "fmt" + + "github.com/xgo-dev/llvm" +) + +func splitBlockAfter(ctx llvm.Context, call llvm.Value, name string) (llvm.BasicBlock, error) { + block := call.InstructionParent() + if block.IsNil() || call.InstructionOpcode() != llvm.Call { + return llvm.BasicBlock{}, fmt.Errorf("split point is not a call instruction") + } + firstMoved := llvm.NextInstruction(call) + if firstMoved.IsNil() { + return llvm.BasicBlock{}, fmt.Errorf("call has no continuation") + } + + terminator := block.LastInstruction() + successors := make([]llvm.BasicBlock, terminator.SuccessorsCount()) + for i := range successors { + successors[i] = terminator.Successor(i) + } + + fn := block.Parent() + continuation := ctx.AddBasicBlock(fn, name) + continuation.MoveAfter(block) + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(continuation) + for instr := firstMoved; !instr.IsNil(); { + next := llvm.NextInstruction(instr) + instrName := instr.Name() + instr.RemoveFromParentAsInstruction() + if instrName == "" { + builder.Insert(instr) + } else { + builder.InsertWithName(instr, instrName) + } + instr = next + } + builder.SetInsertPointAtEnd(block) + builder.CreateBr(continuation) + + for _, successor := range successors { + replacePhiPredecessor(ctx, successor, block, continuation) + } + return continuation, nil +} + +func replacePhiPredecessor(ctx llvm.Context, block, old, replacement llvm.BasicBlock) { + var phis []llvm.Value + for phi := block.FirstInstruction(); !phi.IsNil() && phi.InstructionOpcode() == llvm.PHI; phi = llvm.NextInstruction(phi) { + phis = append(phis, phi) + } + builder := ctx.NewBuilder() + defer builder.Dispose() + for _, phi := range phis { + values := make([]llvm.Value, phi.IncomingCount()) + blocks := make([]llvm.BasicBlock, len(values)) + changed := false + for i := range values { + values[i] = phi.IncomingValue(i) + blocks[i] = phi.IncomingBlock(i) + if blocks[i] == old { + blocks[i] = replacement + changed = true + } + } + if !changed { + continue + } + builder.SetInsertPointBefore(phi) + next := builder.CreatePHI(phi.Type(), "") + next.AddIncoming(values, blocks) + next.InstructionSetDebugLoc(phi.InstructionDebugLoc()) + name := phi.Name() + phi.SetName("") + next.SetName(name) + phi.ReplaceAllUsesWith(next) + phi.EraseFromParentAsInstruction() + } +} diff --git a/internal/wasmresume/ir_test.go b/internal/wasmresume/ir_test.go new file mode 100644 index 0000000000..d59c549639 --- /dev/null +++ b/internal/wasmresume/ir_test.go @@ -0,0 +1,91 @@ +package wasmresume + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestSplitBlockAfterMovesContinuationAndRewritesPhi(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("split") + defer mod.Dispose() + + i1 := ctx.Int1Type() + i32 := ctx.Int32Type() + voidFn := llvm.FunctionType(ctx.VoidType(), nil, false) + callee := llvm.AddFunction(mod, "callee", voidFn) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(i32, []llvm.Type{i1, i32}, false)) + entry := ctx.AddBasicBlock(fn, "entry") + other := ctx.AddBasicBlock(fn, "other") + merge := ctx.AddBasicBlock(fn, "merge") + builder := ctx.NewBuilder() + defer builder.Dispose() + + builder.SetInsertPointAtEnd(entry) + call := builder.CreateCall(voidFn, callee, nil, "") + value := builder.CreateAdd(fn.Param(1), llvm.ConstInt(i32, 2, false), "value") + builder.CreateBr(merge) + builder.SetInsertPointAtEnd(other) + builder.CreateBr(merge) + builder.SetInsertPointAtEnd(merge) + phi := builder.CreatePHI(i32, "selected") + phi.AddIncoming( + []llvm.Value{value, llvm.ConstInt(i32, 0, false)}, + []llvm.BasicBlock{entry, other}, + ) + builder.CreateRet(phi) + + continuation, err := splitBlockAfter(ctx, call, "resume.1") + if err != nil { + t.Fatal(err) + } + if continuation != llvm.NextBasicBlock(entry) { + t.Fatal("continuation was not placed after the split block") + } + if got := llvm.NextInstruction(call).InstructionOpcode(); got != llvm.Br { + t.Fatalf("split block terminator = %v, want br", got) + } + if got := continuation.FirstInstruction().Name(); got != "value" { + t.Fatalf("first continuation instruction = %q, want value:\n%s", got, mod.String()) + } + nextPhi := merge.FirstInstruction() + if nextPhi.InstructionOpcode() != llvm.PHI || nextPhi.Name() != "selected" { + t.Fatalf("replacement phi = %v %q", nextPhi.InstructionOpcode(), nextPhi.Name()) + } + if nextPhi.IncomingBlock(0) != continuation || nextPhi.IncomingBlock(1) != other { + t.Fatal("replacement phi has incorrect predecessors") + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify split module: %v\n%s", err, mod.String()) + } +} + +func TestSplitBlockAfterRejectsInvalidPoints(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("invalid-split") + defer mod.Dispose() + fn := llvm.AddFunction(mod, "function", llvm.FunctionType(ctx.VoidType(), nil, false)) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + ret := builder.CreateRetVoid() + + if _, err := splitBlockAfter(ctx, ret, "resume"); err == nil || + !strings.Contains(err.Error(), "not a call") { + t.Fatalf("non-call split error = %v", err) + } + + callBlock := ctx.AddBasicBlock(fn, "unterminated") + builder.SetInsertPointAtEnd(callBlock) + callee := llvm.AddFunction(mod, "callee", llvm.FunctionType(ctx.VoidType(), nil, false)) + call := builder.CreateCall(callee.GlobalValueType(), callee, nil, "") + if _, err := splitBlockAfter(ctx, call, "resume"); err == nil || + !strings.Contains(err.Error(), "no continuation") { + t.Fatalf("terminal call split error = %v", err) + } +} diff --git a/internal/wasmresume/layout.go b/internal/wasmresume/layout.go new file mode 100644 index 0000000000..6e81d3550c --- /dev/null +++ b/internal/wasmresume/layout.go @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import "github.com/xgo-dev/llvm" + +const frameHeaderFields = 3 + +type frameLayout struct { + plan framePlan + typ llvm.Type + size uint64 + alignment int + fields []int + unwindOffset uint64 +} + +func layoutFrames(mod llvm.Module, targetData llvm.TargetData) ([]frameLayout, error) { + plans, err := planFrames(mod) + if err != nil { + return nil, err + } + ctx := mod.Context() + ptr := llvm.PointerType(ctx.Int8Type(), 0) + header := []llvm.Type{ptr, ptr, ctx.Int32Type()} + + layouts := make([]frameLayout, len(plans)) + for i, plan := range plans { + fields := append([]llvm.Type(nil), header...) + fieldIndices := make([]int, len(plan.slots)+1) + headerType := ctx.StructType(header, false) + frameAlign := targetData.ABITypeAlignment(headerType) + for _, slot := range plan.slots { + align := targetData.ABITypeAlignment(slot.typ) + if slot.kind == slotAlloca && slot.value.Alignment() > align { + align = slot.value.Alignment() + } + if align > frameAlign { + frameAlign = align + } + withSlot := append(append([]llvm.Type(nil), fields...), slot.typ) + naturalOffset := targetData.ElementOffset( + ctx.StructType(withSlot, false), len(withSlot)-1, + ) + if padding := alignmentPadding(naturalOffset, uint64(align)); padding != 0 { + fields = append(fields, llvm.ArrayType(ctx.Int8Type(), int(padding))) + } + fieldIndices[slot.id] = len(fields) + fields = append(fields, slot.typ) + } + typ := ctx.StructType(fields, false) + var unwindOffset uint64 + if plan.unwindSlot != 0 { + unwindOffset = targetData.ElementOffset(typ, fieldIndices[plan.unwindSlot]) + } + layouts[i] = frameLayout{ + plan: plan, + typ: typ, + size: targetData.TypeAllocSize(typ), + alignment: frameAlign, + fields: fieldIndices, + unwindOffset: unwindOffset, + } + } + return layouts, nil +} + +func (l frameLayout) fieldIndex(slotID uint32) int { + if slotID == 0 || int(slotID) >= len(l.fields) { + return -1 + } + return l.fields[slotID] +} + +func alignmentPadding(offset, align uint64) uint64 { + return -offset & (align - 1) +} diff --git a/internal/wasmresume/layout_test.go b/internal/wasmresume/layout_test.go new file mode 100644 index 0000000000..b710c520cb --- /dev/null +++ b/internal/wasmresume/layout_test.go @@ -0,0 +1,148 @@ +package wasmresume + +import ( + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestLayoutFramesUsesRuntimeHeaderAndStableSlots(t *testing.T) { + for _, test := range []struct { + name string + dataLayout string + wantSize uint64 + wantAlign int + wantOffset []uint64 + }{ + { + name: "wasm32", + dataLayout: "e-m:e-p:32:32-i64:64-n32:64-S128", + wantSize: 32, + wantAlign: 8, + wantOffset: []uint64{0, 4, 8, 16, 24}, + }, + { + name: "wasm64", + dataLayout: "e-m:e-p:64:64-i64:64-n32:64-S128", + wantSize: 40, + wantAlign: 8, + wantOffset: []uint64{0, 8, 16, 24, 32}, + }, + } { + t.Run(test.name, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule(test.name) + defer mod.Dispose() + targetData := llvm.NewTargetData(test.dataLayout) + defer targetData.Dispose() + + i64 := ctx.Int64Type() + fn := llvm.AddFunction(mod, "leaf", llvm.FunctionType(i64, []llvm.Type{i64}, false)) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + builder.CreateRet(fn.Param(0)) + + layouts, err := layoutFrames(mod, targetData) + if err != nil { + t.Fatal(err) + } + if len(layouts) != 1 { + t.Fatalf("layouts = %+v", layouts) + } + layout := layouts[0] + if layout.size != test.wantSize || layout.alignment != test.wantAlign { + t.Fatalf("layout size/alignment = %d/%d, want %d/%d", + layout.size, layout.alignment, test.wantSize, test.wantAlign) + } + for field, want := range test.wantOffset { + if got := targetData.ElementOffset(layout.typ, field); got != want { + t.Errorf("field %d offset = %d, want %d", field, got, want) + } + } + if layout.fieldIndex(1) != 3 || layout.fieldIndex(2) != 4 { + t.Fatalf("slot fields = %d/%d, want 3/4", layout.fieldIndex(1), layout.fieldIndex(2)) + } + if layout.fieldIndex(0) != -1 || layout.fieldIndex(3) != -1 { + t.Fatalf("invalid slot fields = %d/%d, want -1/-1", + layout.fieldIndex(0), layout.fieldIndex(3)) + } + }) + } +} + +func TestLayoutFramesKeepsDynamicAllocaAsPointer(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("dynamic") + defer mod.Dispose() + targetData := llvm.NewTargetData("e-m:e-p:32:32-i64:64-n32:64-S128") + defer targetData.Dispose() + + i32 := ctx.Int32Type() + ptr := llvm.PointerType(i32, 0) + calleeType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr}, false) + callee := llvm.AddFunction(mod, "callee", calleeType) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), []llvm.Type{i32}, false)) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + local := builder.CreateArrayAlloca(i32, fn.Param(0), "local") + call := builder.CreateCall(calleeType, callee, []llvm.Value{local}, "") + markCall(ctx, call) + builder.CreateRetVoid() + + layouts, err := layoutFrames(mod, targetData) + if err != nil { + t.Fatal(err) + } + layout := layouts[0] + fields := layout.typ.StructElementTypes() + if len(fields) != 5 || fields[4].TypeKind() != llvm.PointerTypeKind { + t.Fatalf("frame fields = %v, want dynamic alloca pointer at field 4", fields) + } +} + +func TestLayoutFramesPreservesAllocaAlignment(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("aligned") + defer mod.Dispose() + targetData := llvm.NewTargetData("e-m:e-p:64:64-i64:64-n32:64-S128") + defer targetData.Dispose() + + i32 := ctx.Int32Type() + ptr := llvm.PointerType(i32, 0) + calleeType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr}, false) + callee := llvm.AddFunction(mod, "callee", calleeType) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), nil, false)) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + local := builder.CreateAlloca(i32, "local") + local.SetAlignment(32) + call := builder.CreateCall(calleeType, callee, []llvm.Value{local}, "") + markCall(ctx, call) + builder.CreateRetVoid() + + layouts, err := layoutFrames(mod, targetData) + if err != nil { + t.Fatal(err) + } + layout := layouts[0] + if layout.alignment != 32 { + t.Fatalf("frame alignment = %d, want 32", layout.alignment) + } + slot := layout.plan.slots[0] + offset := targetData.ElementOffset(layout.typ, layout.fieldIndex(slot.id)) + if offset%32 != 0 { + t.Fatalf("aligned alloca offset = %d, want a multiple of 32", offset) + } +} diff --git a/internal/wasmresume/leaf.go b/internal/wasmresume/leaf.go new file mode 100644 index 0000000000..7e9ed02b3e --- /dev/null +++ b/internal/wasmresume/leaf.go @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import "github.com/xgo-dev/llvm" + +type loweredLeaf struct { + layout frameLayout + entry llvm.Value + descriptor llvm.Value +} + +// emitLeafEntries emits the descriptor ABI for functions which cannot suspend +// below their own frame. Non-leaf state-machine lowering is a later stage. +func emitLeafEntries(mod llvm.Module, targetData llvm.TargetData) ([]loweredLeaf, error) { + layouts, err := layoutFrames(mod, targetData) + if err != nil { + return nil, err + } + return emitLeafEntriesForLayouts(mod, newResumeABI(mod.Context(), targetData), layouts) +} + +func emitLeafEntriesForLayouts( + mod llvm.Module, abi resumeABI, layouts []frameLayout, +) ([]loweredLeaf, error) { + ctx := mod.Context() + var lowered []loweredLeaf + for _, layout := range layouts { + fn := layout.plan.function + if fn.IsDeclaration() || needsStateMachine(layout) { + continue + } + entry, descriptor, err := abi.defineEntryAndDescriptor(mod, layout) + if err != nil { + return nil, err + } + block := ctx.AddBasicBlock(entry, "entry") + builder := ctx.NewBuilder() + builder.SetInsertPointAtEnd(block) + + rawFrame := entry.Param(1) + params := make([]llvm.Value, 0, fn.ParamsCount()) + for _, slot := range layout.plan.slots { + if slot.kind != slotParameter { + continue + } + field := builder.CreateStructGEP(layout.typ, rawFrame, layout.fieldIndex(slot.id), "") + params = append(params, builder.CreateLoad(slot.typ, field, slot.value.Name())) + } + call := builder.CreateCall(fn.GlobalValueType(), fn, params, "") + call.SetInstructionCallConv(fn.FunctionCallConv()) + if layout.plan.resultSlot != 0 { + field := builder.CreateStructGEP( + layout.typ, rawFrame, layout.fieldIndex(layout.plan.resultSlot), "", + ) + builder.CreateStore(call, field) + } + builder.CreateRet(llvm.ConstInt(ctx.Int8Type(), actionReturn, false)) + builder.Dispose() + + lowered = append(lowered, loweredLeaf{ + layout: layout, entry: entry, descriptor: descriptor, + }) + } + return lowered, nil +} diff --git a/internal/wasmresume/leaf_test.go b/internal/wasmresume/leaf_test.go new file mode 100644 index 0000000000..1b571b365c --- /dev/null +++ b/internal/wasmresume/leaf_test.go @@ -0,0 +1,109 @@ +package wasmresume + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestEmitLeafEntriesLoadsParametersAndStoresResult(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("leaf") + defer mod.Dispose() + mod.SetDataLayout("e-m:e-p:32:32-i64:64-n32:64-S128") + targetData := llvm.NewTargetData(mod.DataLayout()) + defer targetData.Dispose() + + i32 := ctx.Int32Type() + fn := llvm.AddFunction(mod, "leaf", llvm.FunctionType(i32, []llvm.Type{i32}, false)) + markFunction(ctx, fn) + fn.Param(0).SetName("input") + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + builder.CreateRet(builder.CreateAdd(fn.Param(0), llvm.ConstInt(i32, 1, false), "value")) + + lowered, err := emitLeafEntries(mod, targetData) + if err != nil { + t.Fatal(err) + } + if len(lowered) != 1 || lowered[0].layout.size != 20 || lowered[0].layout.alignment != 4 { + t.Fatalf("lowered leaves = %+v", lowered) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify lowered leaf: %v\n%s", err, mod.String()) + } + + ir := mod.String() + for _, want := range []string{ + `@__llgo_wasm_resume_desc.leaf = constant { ptr, i32, i32, i32, i32 }`, + `{ ptr @__llgo_wasm_resume.leaf, i32 20, i32 4, i32 0, i32 0 }`, + `define internal i8 @__llgo_wasm_resume.leaf(ptr %0, ptr %1)`, + `load i32, ptr %2`, + `call i32 @leaf(i32 %input)`, + `store i32 %3, ptr %4`, + `ret i8 1`, + } { + if !strings.Contains(ir, want) { + t.Errorf("lowered leaf is missing %q:\n%s", want, ir) + } + } +} + +func TestEmitLeafEntriesSkipsNonLeafAndDeclarations(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("skip") + defer mod.Dispose() + targetData := llvm.NewTargetData("e-m:e-p:32:32-i64:64-n32:64-S128") + defer targetData.Dispose() + + voidFn := llvm.FunctionType(ctx.VoidType(), nil, false) + declaration := llvm.AddFunction(mod, "declaration", voidFn) + markFunction(ctx, declaration) + callee := llvm.AddFunction(mod, "callee", voidFn) + nonLeaf := llvm.AddFunction(mod, "nonleaf", voidFn) + markFunction(ctx, nonLeaf) + block := ctx.AddBasicBlock(nonLeaf, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + call := builder.CreateCall(voidFn, callee, nil, "") + markCall(ctx, call) + builder.CreateRetVoid() + + lowered, err := emitLeafEntries(mod, targetData) + if err != nil { + t.Fatal(err) + } + if len(lowered) != 0 { + t.Fatalf("lowered leaves = %+v, want empty", lowered) + } +} + +func TestEmitLeafEntriesRejectsDuplicateSymbols(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("duplicate") + defer mod.Dispose() + targetData := llvm.NewTargetData("e-m:e-p:32:32-i64:64-n32:64-S128") + defer targetData.Dispose() + + voidFn := llvm.FunctionType(ctx.VoidType(), nil, false) + fn := llvm.AddFunction(mod, "leaf", voidFn) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + builder.CreateRetVoid() + llvm.AddFunction(mod, resumeEntryPrefix+"leaf", llvm.FunctionType(ctx.Int8Type(), nil, false)) + + if _, err := emitLeafEntries(mod, targetData); err == nil || + !strings.Contains(err.Error(), "duplicate resumable descriptor") { + t.Fatalf("emitLeafEntries error = %v", err) + } +} diff --git a/internal/wasmresume/spill.go b/internal/wasmresume/spill.go new file mode 100644 index 0000000000..7e5a57c999 --- /dev/null +++ b/internal/wasmresume/spill.go @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import ( + "fmt" + + "github.com/xgo-dev/llvm" +) + +func spillValue(ctx llvm.Context, value, field llvm.Value) error { + if value.IsAInstruction().IsNil() { + replaceValueUsesWithLoads(ctx, value, field, llvm.Value{}) + return nil + } + if !value.IsAAllocaInst().IsNil() { + if _, dynamic := persistentSlotType(value, slotAlloca); dynamic { + return fmt.Errorf("dynamic alloca %q requires separate frame storage", value.Name()) + } + value.ReplaceAllUsesWith(field) + value.EraseFromParentAsInstruction() + return nil + } + builder := ctx.NewBuilder() + defer builder.Dispose() + if value.InstructionOpcode() == llvm.PHI { + next := value + for !next.IsNil() && next.InstructionOpcode() == llvm.PHI { + next = llvm.NextInstruction(next) + } + if next.IsNil() { + return fmt.Errorf("phi %q has no insertion point", value.Name()) + } + builder.SetInsertPointBefore(next) + } else { + next := llvm.NextInstruction(value) + if next.IsNil() { + return fmt.Errorf("value %q has no insertion point", value.Name()) + } + builder.SetInsertPointBefore(next) + } + store := builder.CreateStore(value, field) + store.InstructionSetDebugLoc(value.InstructionDebugLoc()) + replaceValueUsesWithLoads(ctx, value, field, store) + return nil +} + +func replaceValueUsesWithLoads(ctx llvm.Context, value, field, skip llvm.Value) { + var users []llvm.Value + seen := make(map[llvm.Value]struct{}) + for use := value.FirstUse(); !use.IsNil(); use = use.NextUse() { + user := use.User() + if user == skip { + continue + } + if _, ok := seen[user]; !ok { + seen[user] = struct{}{} + users = append(users, user) + } + } + + builder := ctx.NewBuilder() + defer builder.Dispose() + for _, user := range users { + if user.InstructionOpcode() == llvm.PHI { + for i := 0; i < user.IncomingCount(); i++ { + if user.IncomingValue(i) != value { + continue + } + terminator := user.IncomingBlock(i).LastInstruction() + builder.SetInsertPointBefore(terminator) + load := builder.CreateLoad(value.Type(), field, value.Name()+".reload") + load.InstructionSetDebugLoc(user.InstructionDebugLoc()) + user.SetOperand(i, load) + } + continue + } + builder.SetInsertPointBefore(user) + load := builder.CreateLoad(value.Type(), field, value.Name()+".reload") + load.InstructionSetDebugLoc(user.InstructionDebugLoc()) + for i := 0; i < user.OperandsCount(); i++ { + if user.Operand(i) == value { + user.SetOperand(i, load) + } + } + } +} diff --git a/internal/wasmresume/spill_test.go b/internal/wasmresume/spill_test.go new file mode 100644 index 0000000000..126ff9a098 --- /dev/null +++ b/internal/wasmresume/spill_test.go @@ -0,0 +1,275 @@ +package wasmresume + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestSpillValueStoresDefinitionAndReloadsUses(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("spill") + defer mod.Dispose() + i32 := ctx.Int32Type() + frameType := ctx.StructType([]llvm.Type{i32}, false) + fn := llvm.AddFunction(mod, "function", llvm.FunctionType(i32, []llvm.Type{ + llvm.PointerType(frameType, 0), i32, + }, false)) + fn.Param(1).SetName("input") + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + field := builder.CreateStructGEP(frameType, fn.Param(0), 0, "field") + value := builder.CreateAdd(fn.Param(1), llvm.ConstInt(i32, 1, false), "value") + result := builder.CreateMul(value, value, "result") + builder.CreateRet(result) + + if err := spillValue(ctx, value, field); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify spilled module: %v\n%s", err, mod.String()) + } + ir := mod.String() + if !strings.Contains(ir, "store i32 %value, ptr %field") || + strings.Count(ir, "load i32, ptr %field") != 1 || + !strings.Contains(ir, "mul i32 %value.reload, %value.reload") { + t.Fatalf("value was not canonicalized through the frame:\n%s", ir) + } +} + +func TestSpillValueReloadsParameter(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("spill-parameter") + defer mod.Dispose() + i32 := ctx.Int32Type() + frameType := ctx.StructType([]llvm.Type{i32}, false) + fn := llvm.AddFunction(mod, "function", llvm.FunctionType(i32, []llvm.Type{ + llvm.PointerType(frameType, 0), i32, + }, false)) + fn.Param(1).SetName("input") + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + field := builder.CreateStructGEP(frameType, fn.Param(0), 0, "field") + builder.CreateRet(fn.Param(1)) + + if err := spillValue(ctx, fn.Param(1), field); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify parameter reload: %v\n%s", err, mod.String()) + } + if !strings.Contains(mod.String(), "ret i32 %input.reload") { + t.Fatalf("parameter use was not loaded from the frame:\n%s", mod.String()) + } +} + +func TestSpillValueStoresPhiAfterPhiGroup(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("spill-phi-definition") + defer mod.Dispose() + i1 := ctx.Int1Type() + i32 := ctx.Int32Type() + frameType := ctx.StructType([]llvm.Type{i32}, false) + fn := llvm.AddFunction(mod, "function", llvm.FunctionType(i32, []llvm.Type{ + llvm.PointerType(frameType, 0), i1, i32, + }, false)) + entry := ctx.AddBasicBlock(fn, "entry") + left := ctx.AddBasicBlock(fn, "left") + right := ctx.AddBasicBlock(fn, "right") + merge := ctx.AddBasicBlock(fn, "merge") + builder := ctx.NewBuilder() + defer builder.Dispose() + + builder.SetInsertPointAtEnd(entry) + field := builder.CreateStructGEP(frameType, fn.Param(0), 0, "field") + builder.CreateCondBr(fn.Param(1), left, right) + builder.SetInsertPointAtEnd(left) + builder.CreateBr(merge) + builder.SetInsertPointAtEnd(right) + builder.CreateBr(merge) + builder.SetInsertPointAtEnd(merge) + phi := builder.CreatePHI(i32, "selected") + phi.AddIncoming([]llvm.Value{ + fn.Param(2), llvm.ConstInt(i32, 0, false), + }, []llvm.BasicBlock{left, right}) + result := builder.CreateAdd(phi, llvm.ConstInt(i32, 1, false), "result") + builder.CreateRet(result) + + if err := spillValue(ctx, phi, field); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify phi spill: %v\n%s", err, mod.String()) + } + ir := mod.String() + if !strings.Contains(ir, "store i32 %selected, ptr %field") || + !strings.Contains(ir, "add i32 %selected.reload, 1") { + t.Fatalf("phi was not canonicalized through the frame:\n%s", ir) + } +} + +func TestSpillValueReloadsPhiOnIncomingEdge(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("spill-phi") + defer mod.Dispose() + + i1 := ctx.Int1Type() + i32 := ctx.Int32Type() + frameType := ctx.StructType([]llvm.Type{i32}, false) + fn := llvm.AddFunction(mod, "function", llvm.FunctionType(i32, []llvm.Type{ + llvm.PointerType(frameType, 0), i1, i32, + }, false)) + entry := ctx.AddBasicBlock(fn, "entry") + left := ctx.AddBasicBlock(fn, "left") + right := ctx.AddBasicBlock(fn, "right") + merge := ctx.AddBasicBlock(fn, "merge") + builder := ctx.NewBuilder() + defer builder.Dispose() + + builder.SetInsertPointAtEnd(entry) + field := builder.CreateStructGEP(frameType, fn.Param(0), 0, "field") + store := builder.CreateStore(fn.Param(2), field) + builder.CreateCondBr(fn.Param(1), left, right) + builder.SetInsertPointAtEnd(left) + builder.CreateBr(merge) + builder.SetInsertPointAtEnd(right) + builder.CreateBr(merge) + builder.SetInsertPointAtEnd(merge) + phi := builder.CreatePHI(i32, "selected") + phi.AddIncoming( + []llvm.Value{fn.Param(2), llvm.ConstInt(i32, 0, false)}, + []llvm.BasicBlock{left, right}, + ) + builder.CreateRet(phi) + + replaceValueUsesWithLoads(ctx, fn.Param(2), field, store) + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify phi reload: %v\n%s", err, mod.String()) + } + if incoming := merge.FirstInstruction().IncomingValue(0); incoming.InstructionParent() != left { + t.Fatalf("phi reload is not on the incoming edge:\n%s", mod.String()) + } +} + +func TestSpillValueReplacesAllocaWithFrameAddress(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("spill-alloca") + defer mod.Dispose() + i32 := ctx.Int32Type() + frameType := ctx.StructType([]llvm.Type{i32}, false) + fn := llvm.AddFunction(mod, "function", llvm.FunctionType(i32, []llvm.Type{ + llvm.PointerType(frameType, 0), + }, false)) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + field := builder.CreateStructGEP(frameType, fn.Param(0), 0, "field") + local := builder.CreateAlloca(i32, "local") + builder.CreateStore(llvm.ConstInt(i32, 9, false), local) + builder.CreateRet(builder.CreateLoad(i32, local, "result")) + + if err := spillValue(ctx, local, field); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify alloca frame address: %v\n%s", err, mod.String()) + } + if strings.Contains(mod.String(), " = alloca ") { + t.Fatalf("alloca remains after frame replacement:\n%s", mod.String()) + } +} + +func TestSpillValueRejectsUnsupportedDefinitions(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("spill-errors") + defer mod.Dispose() + i32 := ctx.Int32Type() + frameType := ctx.StructType([]llvm.Type{i32}, false) + fn := llvm.AddFunction(mod, "function", llvm.FunctionType(ctx.VoidType(), []llvm.Type{i32}, false)) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + field := builder.CreateStructGEP(frameType, builder.CreateAlloca(frameType, "frame"), 0, "field") + dynamic := builder.CreateArrayAlloca(i32, fn.Param(0), "dynamic") + builder.CreateRetVoid() + + if err := spillValue(ctx, dynamic, field); err == nil || !strings.Contains(err.Error(), "separate frame storage") { + t.Fatalf("dynamic alloca spill error = %v", err) + } +} + +func TestSpillValueStoresOrdinaryCallResult(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("spill-call") + defer mod.Dispose() + i32 := ctx.Int32Type() + frameType := ctx.StructType([]llvm.Type{i32}, false) + callee := llvm.AddFunction(mod, "callee", llvm.FunctionType(i32, nil, false)) + fn := llvm.AddFunction(mod, "function", llvm.FunctionType(i32, []llvm.Type{ + llvm.PointerType(frameType, 0), + }, false)) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + field := builder.CreateStructGEP(frameType, fn.Param(0), 0, "field") + call := builder.CreateCall(callee.GlobalValueType(), callee, nil, "call") + builder.CreateRet(call) + + if err := spillValue(ctx, call, field); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify spilled call result: %v\n%s", err, mod.String()) + } + ir := mod.String() + if !strings.Contains(ir, "store i32 %call, ptr %field") || + !strings.Contains(ir, "ret i32 %call.reload") { + t.Fatalf("ordinary call result was not stored in the frame:\n%s", ir) + } +} + +func TestSpillValueReplacesOverAlignedAlloca(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("spill-aligned-alloca") + defer mod.Dispose() + i32 := ctx.Int32Type() + frameType := ctx.StructType([]llvm.Type{i32}, false) + fn := llvm.AddFunction(mod, "function", llvm.FunctionType(ctx.VoidType(), nil, false)) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + frame := builder.CreateAlloca(frameType, "frame") + frame.SetAlignment(32) + field := builder.CreateStructGEP(frameType, frame, 0, "field") + local := builder.CreateAlloca(i32, "local") + local.SetAlignment(32) + builder.CreateStore(llvm.ConstInt(i32, 9, false), local) + builder.CreateRetVoid() + + if err := spillValue(ctx, local, field); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify aligned alloca frame address: %v\n%s", err, mod.String()) + } + if strings.Contains(mod.String(), "%local = alloca") { + t.Fatalf("over-aligned alloca remains after frame replacement:\n%s", mod.String()) + } +} diff --git a/internal/wasmresume/start.go b/internal/wasmresume/start.go new file mode 100644 index 0000000000..c9c4f71a5d --- /dev/null +++ b/internal/wasmresume/start.go @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import ( + "fmt" + + "github.com/xgo-dev/llvm" +) + +func emitStartEntriesForLayouts( + mod llvm.Module, abi resumeABI, layouts []frameLayout, +) error { + ctx := mod.Context() + var alloc llvm.Value + for _, layout := range layouts { + fn := layout.plan.function + if fn.IsDeclaration() || fn.GlobalValueType().IsFunctionVarArg() { + continue + } + descriptor := mod.NamedGlobal(descriptorPrefix + fn.Name()) + if descriptor.IsNil() || descriptor.Initializer().IsNil() { + return fmt.Errorf("%s: resumable descriptor is not defined", fn.Name()) + } + + params := append([]llvm.Type{abi.ptr}, fn.GlobalValueType().ParamTypes()...) + startType := llvm.FunctionType(abi.ptr, params, false) + startName := StartSymbol(fn.Name()) + start := mod.NamedFunction(startName) + if start.IsNil() { + start = llvm.AddFunction(mod, startName, startType) + } else if !start.IsDeclaration() || start.GlobalValueType() != startType { + return fmt.Errorf("%s: incompatible resumable start entry", fn.Name()) + } + start.SetLinkage(fn.Linkage()) + if alloc.IsNil() { + alloc = declareFrameAllocator(mod, abi) + } + + block := ctx.AddBasicBlock(start, "entry") + builder := ctx.NewBuilder() + builder.SetInsertPointAtEnd(block) + child := builder.CreateCall(alloc.GlobalValueType(), alloc, []llvm.Value{ + start.Param(0), + llvm.ConstInt(abi.uintptrType, layout.size, false), + llvm.ConstInt(abi.uintptrType, uint64(layout.alignment), false), + }, "child") + builder.CreateIntrinsic(ctx.VoidType(), llvm.LookupIntrinsicID("llvm.memset"), []llvm.Value{ + child, + llvm.ConstInt(ctx.Int8Type(), 0, false), + llvm.ConstInt(abi.uintptrType, layout.size, false), + llvm.ConstInt(ctx.Int1Type(), 0, false), + }, "") + + contextTop := builder.CreateStructGEP(abi.contextType, start.Param(0), 0, "") + parent := builder.CreateLoad(abi.ptr, contextTop, "parent") + builder.CreateStore(parent, builder.CreateStructGEP(layout.typ, child, 0, "")) + builder.CreateStore(descriptor, builder.CreateStructGEP(layout.typ, child, 1, "")) + builder.CreateStore( + llvm.ConstInt(ctx.Int32Type(), 0, false), + builder.CreateStructGEP(layout.typ, child, 2, ""), + ) + param := 1 + for _, slot := range layout.plan.slots { + if slot.kind != slotParameter { + continue + } + builder.CreateStore( + start.Param(param), + builder.CreateStructGEP(layout.typ, child, layout.fieldIndex(slot.id), ""), + ) + param++ + } + builder.CreateRet(child) + builder.Dispose() + } + return nil +} diff --git a/internal/wasmresume/start_test.go b/internal/wasmresume/start_test.go new file mode 100644 index 0000000000..a205149628 --- /dev/null +++ b/internal/wasmresume/start_test.go @@ -0,0 +1,32 @@ +package wasmresume + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestStartEntryRejectsIncompatibleDeclaration(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("incompatible-start") + defer mod.Dispose() + targetData := llvm.NewTargetData("e-m:e-p:32:32-i64:64-n32:64-S128") + defer targetData.Dispose() + + voidFn := llvm.FunctionType(ctx.VoidType(), nil, false) + fn := llvm.AddFunction(mod, "leaf", voidFn) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + builder.CreateRetVoid() + llvm.AddFunction(mod, startEntryPrefix+fn.Name(), voidFn) + + if _, err := lowerPrototype(mod, targetData); err == nil || + !strings.Contains(err.Error(), "incompatible resumable start entry") { + t.Fatalf("lowerPrototype error = %v", err) + } +} diff --git a/internal/wasmresume/state.go b/internal/wasmresume/state.go new file mode 100644 index 0000000000..81e14abc5d --- /dev/null +++ b/internal/wasmresume/state.go @@ -0,0 +1,401 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import ( + "fmt" + "strings" + + "github.com/xgo-dev/llvm" +) + +const ( + frameAllocName = "__llgo_wasm_resume_alloc" + frameDynamicAllocName = "__llgo_wasm_resume_alloc_dynamic" + frameFreeName = "__llgo_wasm_resume_free" +) + +type loweredState struct { + layout frameLayout + entry llvm.Value + descriptor llvm.Value +} + +// Lower replaces marked Go functions and calls with the experimental +// WebAssembly resumable ABI. +func Lower(mod llvm.Module, targetData llvm.TargetData) error { + triple := mod.Target() + if !strings.HasPrefix(triple, "wasm32-") && !strings.HasPrefix(triple, "wasm64-") { + return fmt.Errorf("wasmresume: target %q is not WebAssembly", triple) + } + _, err := lowerPrototype(mod, targetData) + return err +} + +func lowerPrototype(mod llvm.Module, targetData llvm.TargetData) ([]loweredState, error) { + layouts, err := layoutFrames(mod, targetData) + if err != nil { + return nil, err + } + for _, layout := range layouts { + if layout.plan.function.IsDeclaration() || !needsStateMachine(layout) { + continue + } + if err := validateStateLayout(layout); err != nil { + return nil, fmt.Errorf("%s: %w", layout.plan.function.Name(), err) + } + } + abi := newResumeABI(mod.Context(), targetData) + if _, err := emitLeafEntriesForLayouts(mod, abi, layouts); err != nil { + return nil, err + } + var lowered []loweredState + for _, layout := range layouts { + if layout.plan.function.IsDeclaration() || !needsStateMachine(layout) { + continue + } + entry, descriptor, err := abi.defineEntryAndDescriptor(mod, layout) + if err != nil { + return nil, err + } + lowered = append(lowered, loweredState{ + layout: layout, entry: entry, descriptor: descriptor, + }) + } + if err := emitStartEntriesForLayouts(mod, abi, layouts); err != nil { + return nil, err + } + for i := range lowered { + if err := lowerStateMachine(mod, targetData, abi, &lowered[i]); err != nil { + return nil, err + } + if err := emitCompatibilityWrapper(mod, targetData, abi, &lowered[i]); err != nil { + return nil, err + } + } + return lowered, nil +} + +func needsStateMachine(layout frameLayout) bool { + return len(layout.plan.calls) != 0 || layout.plan.unwindSlot != 0 +} + +func validateStateLayout(layout frameLayout) error { + for _, site := range layout.plan.calls { + call := site.call + if call.CalledFunctionType().IsFunctionVarArg() { + return fmt.Errorf("resume call %d is variadic", site.id) + } + if llvm.NextInstruction(call).IsNil() { + return fmt.Errorf("resume call %d has no continuation", site.id) + } + } + return nil +} + +func lowerStateMachine( + mod llvm.Module, targetData llvm.TargetData, abi resumeABI, lowered *loweredState, +) error { + layout := lowered.layout + fn := layout.plan.function + ctx := mod.Context() + + var blocks []llvm.BasicBlock + var returns []llvm.Value + for block := fn.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + blocks = append(blocks, block) + for instr := block.FirstInstruction(); !instr.IsNil(); instr = llvm.NextInstruction(instr) { + if !instr.IsAReturnInst().IsNil() { + returns = append(returns, instr) + } + } + } + if len(blocks) == 0 { + return fmt.Errorf("%s: resumable definition has no body", fn.Name()) + } + originalEntry := blocks[0] + blockAddresses := collectMovedBlockAddresses(fn, blocks) + + dispatch := ctx.AddBasicBlock(lowered.entry, "dispatch") + for _, block := range blocks { + block.RemoveFromParent() + llvm.AppendExistingBasicBlock(lowered.entry, block) + } + remapMovedBlockAddresses(lowered.entry, blockAddresses) + + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(dispatch) + rawFrame := lowered.entry.Param(1) + fields := make(map[uint32]llvm.Value, len(layout.plan.slots)) + for _, slot := range layout.plan.slots { + fields[slot.id] = builder.CreateStructGEP( + layout.typ, rawFrame, layout.fieldIndex(slot.id), "", + ) + } + + for _, slot := range layout.plan.slots { + if slot.kind == slotUnwind { + continue + } + if isStackSave(slot.value) { + if err := lowerPersistentStackSave(slot.value); err != nil { + return fmt.Errorf("%s: %w", fn.Name(), err) + } + continue + } + if slot.kind == slotAlloca && slot.dynamic { + if err := lowerDynamicAlloca( + mod, targetData, abi, lowered.entry, slot.value, fields[slot.id], + ); err != nil { + return fmt.Errorf("%s: %w", fn.Name(), err) + } + continue + } + switch slot.kind { + case slotFunctionResult: + continue + case slotValue: + if slot.value.InstructionOpcode() == llvm.Call && + isResumeCallResult(layout.plan, slot.id) { + continue + } + } + if err := spillValue(ctx, slot.value, fields[slot.id]); err != nil { + return fmt.Errorf("%s: %w", fn.Name(), err) + } + } + if err := lowerUnwindMarkers( + ctx, lowered.entry, layout.plan, fields[layout.plan.unwindSlot], + ); err != nil { + return fmt.Errorf("%s: %w", fn.Name(), err) + } + + continuations := make(map[uint32]llvm.BasicBlock, len(layout.plan.calls)) + for _, site := range layout.plan.calls { + continuation, err := splitBlockAfter(ctx, site.call, fmt.Sprintf("resume.%d", site.id)) + if err != nil { + return fmt.Errorf("%s: call %d: %w", fn.Name(), site.id, err) + } + continuations[site.id] = continuation + if site.call.CalledValue().Name() == SuspendSymbol { + lowerSuspendCall(ctx, layout, lowered.entry, site) + continue + } + if err := lowerResumeCall( + mod, abi, layout, fields, lowered.entry, site, continuation, + ); err != nil { + return fmt.Errorf("%s: call %d: %w", fn.Name(), site.id, err) + } + } + + for _, ret := range returns { + builder.SetInsertPointBefore(ret) + if layout.plan.resultSlot != 0 { + builder.CreateStore(ret.Operand(0), fields[layout.plan.resultSlot]) + } + next := builder.CreateRet(llvm.ConstInt(ctx.Int8Type(), actionReturn, false)) + next.InstructionSetDebugLoc(ret.InstructionDebugLoc()) + ret.EraseFromParentAsInstruction() + } + + invalid := ctx.AddBasicBlock(lowered.entry, "invalid-pc") + builder.SetInsertPointAtEnd(invalid) + builder.CreateUnreachable() + builder.SetInsertPointAtEnd(dispatch) + pcField := builder.CreateStructGEP(layout.typ, rawFrame, 2, "") + pc := builder.CreateLoad(ctx.Int32Type(), pcField, "pc") + switchPC := builder.CreateSwitch(pc, invalid, len(continuations)+1) + switchPC.AddCase(llvm.ConstInt(ctx.Int32Type(), 0, false), originalEntry) + for _, site := range layout.plan.calls { + switchPC.AddCase( + llvm.ConstInt(ctx.Int32Type(), uint64(site.id), false), + continuations[site.id], + ) + } + if layout.plan.unwindPC != 0 { + switchPC.AddCase( + llvm.ConstInt(ctx.Int32Type(), uint64(layout.plan.unwindPC), false), + layout.plan.unwindBlock, + ) + } + return nil +} + +func isResumeCallResult(plan framePlan, slotID uint32) bool { + for _, site := range plan.calls { + if site.resultSlot == slotID { + return true + } + } + return false +} + +func lowerSuspendCall( + ctx llvm.Context, + parentLayout frameLayout, + entry llvm.Value, + site callSite, +) { + call := site.call + callBlock := call.InstructionParent() + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointBefore(call) + builder.CreateStore( + llvm.ConstInt(ctx.Int32Type(), uint64(site.id), false), + builder.CreateStructGEP(parentLayout.typ, entry.Param(1), 2, ""), + ) + call.EraseFromParentAsInstruction() + terminator := callBlock.LastInstruction() + builder.SetInsertPointBefore(terminator) + builder.CreateRet(llvm.ConstInt(ctx.Int8Type(), actionSuspend, false)) + terminator.EraseFromParentAsInstruction() +} + +func lowerResumeCall( + mod llvm.Module, + abi resumeABI, + parentLayout frameLayout, + parentFields map[uint32]llvm.Value, + entry llvm.Value, + site callSite, + continuation llvm.BasicBlock, +) error { + ctx := mod.Context() + call := site.call + callBlock := call.InstructionParent() + callee := call.CalledValue() + free := declareFrameFree(mod, abi) + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointBefore(call) + + childType := callFramePrefix(ctx, call.CalledFunctionType()) + var child llvm.Value + if callee.IsAFunction().IsNil() || strings.HasPrefix(callee.Name(), startEntryPrefix) { + params := append([]llvm.Type{abi.ptr}, call.CalledFunctionType().ParamTypes()...) + startType := llvm.FunctionType(abi.ptr, params, false) + args := make([]llvm.Value, call.CalledFunctionType().ParamTypesCount()+1) + args[0] = entry.Param(0) + for i := 1; i < len(args); i++ { + args[i] = call.Operand(i - 1) + } + child = builder.CreateCall(startType, callee, args, "child") + } else { + alloc := declareFrameAllocator(mod, abi) + descriptor := mod.NamedGlobal(descriptorPrefix + callee.Name()) + if descriptor.IsNil() { + descriptor = llvm.AddGlobal(mod, abi.descriptorType, descriptorPrefix+callee.Name()) + } + sizeField := builder.CreateStructGEP(abi.descriptorType, descriptor, 1, "") + alignField := builder.CreateStructGEP(abi.descriptorType, descriptor, 2, "") + size := builder.CreateLoad(abi.uintptrType, sizeField, "child.size") + align := builder.CreateLoad(abi.uintptrType, alignField, "child.align") + child = builder.CreateCall( + alloc.GlobalValueType(), alloc, []llvm.Value{entry.Param(0), size, align}, "child", + ) + builder.CreateIntrinsic(ctx.VoidType(), llvm.LookupIntrinsicID("llvm.memset"), []llvm.Value{ + child, + llvm.ConstInt(ctx.Int8Type(), 0, false), + size, + llvm.ConstInt(ctx.Int1Type(), 0, false), + }, "") + + builder.CreateStore(entry.Param(1), builder.CreateStructGEP(childType, child, 0, "")) + builder.CreateStore(descriptor, builder.CreateStructGEP(childType, child, 1, "")) + builder.CreateStore( + llvm.ConstInt(ctx.Int32Type(), 0, false), + builder.CreateStructGEP(childType, child, 2, ""), + ) + for i := 0; i < call.CalledFunctionType().ParamTypesCount(); i++ { + builder.CreateStore( + call.Operand(i), + builder.CreateStructGEP(childType, child, frameHeaderFields+i, ""), + ) + } + } + builder.CreateStore( + llvm.ConstInt(ctx.Int32Type(), uint64(site.id), false), + builder.CreateStructGEP(parentLayout.typ, entry.Param(1), 2, ""), + ) + contextTop := builder.CreateStructGEP(abi.contextType, entry.Param(0), 0, "") + builder.CreateStore(child, contextTop) + + builder.SetInsertPointBefore(continuation.FirstInstruction()) + returnedField := builder.CreateStructGEP(abi.contextType, entry.Param(0), 1, "") + returned := builder.CreateLoad(abi.ptr, returnedField, "returned") + builder.CreateStore(llvm.ConstNull(abi.ptr), returnedField) + if site.resultSlot != 0 { + resultField := frameHeaderFields + call.CalledFunctionType().ParamTypesCount() + result := builder.CreateLoad( + call.Type(), builder.CreateStructGEP(childType, returned, resultField, ""), "call.result", + ) + builder.CreateStore(result, parentFields[site.resultSlot]) + replaceValueUsesWithLoads(ctx, call, parentFields[site.resultSlot], llvm.Value{}) + } + builder.CreateCall( + free.GlobalValueType(), free, []llvm.Value{entry.Param(0), returned}, "", + ) + + call.EraseFromParentAsInstruction() + terminator := callBlock.LastInstruction() + builder.SetInsertPointBefore(terminator) + builder.CreateRet(llvm.ConstInt(ctx.Int8Type(), actionContinue, false)) + terminator.EraseFromParentAsInstruction() + return nil +} + +func callFramePrefix(ctx llvm.Context, typ llvm.Type) llvm.Type { + ptr := llvm.PointerType(ctx.Int8Type(), 0) + fields := []llvm.Type{ptr, ptr, ctx.Int32Type()} + fields = append(fields, typ.ParamTypes()...) + if result := typ.ReturnType(); result.TypeKind() != llvm.VoidTypeKind { + fields = append(fields, result) + } + return ctx.StructType(fields, false) +} + +func declareFrameAllocator(mod llvm.Module, abi resumeABI) llvm.Value { + fn := mod.NamedFunction(frameAllocName) + if fn.IsNil() { + fn = llvm.AddFunction(mod, frameAllocName, llvm.FunctionType( + abi.ptr, []llvm.Type{abi.ptr, abi.uintptrType, abi.uintptrType}, false, + )) + } + return fn +} + +func declareFrameFree(mod llvm.Module, abi resumeABI) llvm.Value { + fn := mod.NamedFunction(frameFreeName) + if fn.IsNil() { + fn = llvm.AddFunction(mod, frameFreeName, llvm.FunctionType( + abi.ctx.VoidType(), []llvm.Type{abi.ptr, abi.ptr}, false, + )) + } + return fn +} + +func declareFrameClose(mod llvm.Module, abi resumeABI) llvm.Value { + fn := mod.NamedFunction(frameCloseName) + if fn.IsNil() { + fn = llvm.AddFunction(mod, frameCloseName, llvm.FunctionType( + abi.ctx.VoidType(), []llvm.Type{abi.ptr}, false, + )) + } + return fn +} diff --git a/internal/wasmresume/state_test.go b/internal/wasmresume/state_test.go new file mode 100644 index 0000000000..6b77f63e96 --- /dev/null +++ b/internal/wasmresume/state_test.go @@ -0,0 +1,547 @@ +package wasmresume + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestLowerPrototypeExecutesDirectCallStateMachine(t *testing.T) { + llvm.LinkInMCJIT() + if err := llvm.InitializeNativeTarget(); err != nil { + t.Fatal(err) + } + if err := llvm.InitializeNativeAsmPrinter(); err != nil { + t.Fatal(err) + } + + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("state-execution") + moduleOwned := true + defer func() { + if moduleOwned { + mod.Dispose() + } + }() + + triple := llvm.DefaultTargetTriple() + target, err := llvm.GetTargetFromTriple(triple) + if err != nil { + t.Fatal(err) + } + machine := target.CreateTargetMachine( + triple, "", "", llvm.CodeGenLevelNone, llvm.RelocDefault, llvm.CodeModelJITDefault, + ) + defer machine.Dispose() + targetData := machine.CreateTargetData() + defer targetData.Dispose() + mod.SetTarget(triple) + mod.SetDataLayout(targetData.String()) + + i32 := ctx.Int32Type() + sig := llvm.FunctionType(i32, []llvm.Type{i32}, false) + callee := llvm.AddFunction(mod, "callee", sig) + markFunction(ctx, callee) + calleeBlock := ctx.AddBasicBlock(callee, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(calleeBlock) + builder.CreateRet(builder.CreateAdd(callee.Param(0), llvm.ConstInt(i32, 1, false), "sum")) + + middle := llvm.AddFunction(mod, "middle", sig) + markFunction(ctx, middle) + middleBlock := ctx.AddBasicBlock(middle, "entry") + builder.SetInsertPointAtEnd(middleBlock) + first := builder.CreateCall(sig, callee, []llvm.Value{middle.Param(0)}, "first") + markCall(ctx, first) + second := builder.CreateCall(sig, callee, []llvm.Value{first}, "second") + markCall(ctx, second) + builder.CreateRet(builder.CreateMul(second, llvm.ConstInt(i32, 2, false), "middle.result")) + + caller := llvm.AddFunction(mod, "caller", sig) + markFunction(ctx, caller) + callerBlock := ctx.AddBasicBlock(caller, "entry") + builder.SetInsertPointAtEnd(callerBlock) + before := builder.CreateAdd(caller.Param(0), llvm.ConstInt(i32, 2, false), "before") + call := builder.CreateCall(sig, middle, []llvm.Value{before}, "called") + markCall(ctx, call) + builder.CreateRet(builder.CreateMul(before, call, "result")) + + lowered, err := lowerPrototype(mod, targetData) + if err != nil { + t.Fatal(err) + } + if len(lowered) != 2 { + t.Fatalf("lowered states = %d, want 2", len(lowered)) + } + var root loweredState + for _, state := range lowered { + if state.layout.plan.function.Name() == "caller" { + root = state + break + } + } + if root.entry.IsNil() { + t.Fatal("caller state machine was not lowered") + } + harness := defineStateMachineHarness( + mod, targetData, root, []llvm.Value{llvm.ConstInt(i32, 5, false)}, + ) + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify executable state machine: %v\n%s", err, mod.String()) + } + + options := llvm.NewMCJITCompilerOptions() + options.SetMCJITOptimizationLevel(0) + engine, err := llvm.NewMCJITCompiler(mod, options) + if err != nil { + t.Fatal(err) + } + moduleOwned = false + defer engine.Dispose() + + result := engine.RunFunction(harness, nil) + defer result.Dispose() + if got := result.Int(true); got != 126 { + t.Fatalf("state machine result = %d, want 126", got) + } + + arg := llvm.NewGenericValueFromInt(i32, 5, true) + defer arg.Dispose() + result = engine.RunFunction(caller, []llvm.GenericValue{arg}) + defer result.Dispose() + if got := result.Int(true); got != 126 { + t.Fatalf("compatibility wrapper result = %d, want 126", got) + } +} + +func TestLowerPrototypeBuildsDirectCallStateMachine(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("state") + defer mod.Dispose() + mod.SetDataLayout("e-m:e-p:32:32-i64:64-n32:64-S128") + targetData := llvm.NewTargetData(mod.DataLayout()) + defer targetData.Dispose() + + i32 := ctx.Int32Type() + sig := llvm.FunctionType(i32, []llvm.Type{i32}, false) + callee := llvm.AddFunction(mod, "callee", sig) + markFunction(ctx, callee) + calleeBlock := ctx.AddBasicBlock(callee, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(calleeBlock) + builder.CreateRet(builder.CreateAdd(callee.Param(0), llvm.ConstInt(i32, 1, false), "sum")) + + caller := llvm.AddFunction(mod, "caller", sig) + markFunction(ctx, caller) + caller.Param(0).SetName("input") + callerBlock := ctx.AddBasicBlock(caller, "entry") + builder.SetInsertPointAtEnd(callerBlock) + before := builder.CreateAdd(caller.Param(0), llvm.ConstInt(i32, 2, false), "before") + call := builder.CreateCall(sig, callee, []llvm.Value{before}, "called") + markCall(ctx, call) + result := builder.CreateMul(before, call, "result") + builder.CreateRet(result) + + lowered, err := lowerPrototype(mod, targetData) + if err != nil { + t.Fatal(err) + } + if len(lowered) != 1 || lowered[0].layout.plan.function != caller { + t.Fatalf("lowered states = %+v", lowered) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify state machine: %v\n%s", err, mod.String()) + } + + ir := mod.String() + for _, want := range []string{ + `@__llgo_wasm_resume_desc.callee = constant`, + `@__llgo_wasm_resume_desc.caller = constant`, + `define internal i8 @__llgo_wasm_resume.caller`, + `switch i32 %pc, label %invalid-pc [`, + `i32 0, label %entry`, + `i32 1, label %resume.1`, + `call ptr @__llgo_wasm_resume_alloc(ptr %0,`, + `call void @llvm.memset`, + `ret i8 0`, + `%returned = load ptr`, + `call void @__llgo_wasm_resume_free(ptr %0, ptr %returned)`, + `ret i8 1`, + } { + if !strings.Contains(ir, want) { + t.Errorf("state machine is missing %q:\n%s", want, ir) + } + } +} + +func TestLowerEmitsWasmObject(t *testing.T) { + llvm.InitializeAllTargetInfos() + llvm.InitializeAllTargets() + llvm.InitializeAllTargetMCs() + llvm.InitializeAllAsmPrinters() + + for _, triple := range []string{"wasm32-unknown-unknown", "wasm64-unknown-unknown"} { + t.Run(triple, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule(triple) + defer mod.Dispose() + + target, err := llvm.GetTargetFromTriple(triple) + if err != nil { + t.Fatal(err) + } + machine := target.CreateTargetMachine( + triple, "", "", llvm.CodeGenLevelNone, llvm.RelocDefault, llvm.CodeModelDefault, + ) + defer machine.Dispose() + targetData := machine.CreateTargetData() + defer targetData.Dispose() + mod.SetTarget(triple) + mod.SetDataLayout(targetData.String()) + + i32 := ctx.Int32Type() + sig := llvm.FunctionType(i32, []llvm.Type{i32}, false) + callee := llvm.AddFunction(mod, "callee", sig) + markFunction(ctx, callee) + calleeBlock := ctx.AddBasicBlock(callee, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(calleeBlock) + builder.CreateRet(callee.Param(0)) + + ptr := llvm.PointerType(ctx.Int8Type(), 0) + callerType := llvm.FunctionType(i32, []llvm.Type{ptr, i32}, false) + caller := llvm.AddFunction(mod, "caller", callerType) + markFunction(ctx, caller) + callerBlock := ctx.AddBasicBlock(caller, "entry") + builder.SetInsertPointAtEnd(callerBlock) + call := builder.CreateCall(sig, caller.Param(0), []llvm.Value{caller.Param(1)}, "called") + markCall(ctx, call) + builder.CreateRet(call) + + if err := Lower(mod, targetData); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify %s state machine: %v\n%s", triple, err, mod.String()) + } + object, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit %s state machine: %v\n%s", triple, err, mod.String()) + } + defer object.Dispose() + if data := object.Bytes(); len(data) < 4 || string(data[:4]) != "\x00asm" { + t.Fatalf("%s object does not have the WebAssembly header", triple) + } + }) + } +} + +func TestLowerRejectsNonWasmTarget(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("native") + defer mod.Dispose() + mod.SetTarget("aarch64-unknown-linux-gnu") + targetData := llvm.NewTargetData("e-m:e-p:64:64-i64:64-n32:64-S128") + defer targetData.Dispose() + + if err := Lower(mod, targetData); err == nil || + !strings.Contains(err.Error(), "is not WebAssembly") { + t.Fatalf("Lower error = %v", err) + } +} + +func TestLowerPrototypeExecutesIndirectStart(t *testing.T) { + llvm.LinkInMCJIT() + if err := llvm.InitializeNativeTarget(); err != nil { + t.Fatal(err) + } + if err := llvm.InitializeNativeAsmPrinter(); err != nil { + t.Fatal(err) + } + + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("indirect-execution") + moduleOwned := true + defer func() { + if moduleOwned { + mod.Dispose() + } + }() + + triple := llvm.DefaultTargetTriple() + target, err := llvm.GetTargetFromTriple(triple) + if err != nil { + t.Fatal(err) + } + machine := target.CreateTargetMachine( + triple, "", "", llvm.CodeGenLevelNone, llvm.RelocDefault, llvm.CodeModelJITDefault, + ) + defer machine.Dispose() + targetData := machine.CreateTargetData() + defer targetData.Dispose() + mod.SetTarget(triple) + mod.SetDataLayout(targetData.String()) + + i32 := ctx.Int32Type() + sig := llvm.FunctionType(i32, []llvm.Type{i32}, false) + callee := llvm.AddFunction(mod, "callee", sig) + markFunction(ctx, callee) + block := ctx.AddBasicBlock(callee, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + builder.CreateRet(builder.CreateAdd(callee.Param(0), llvm.ConstInt(i32, 1, false), "result")) + + ptr := llvm.PointerType(ctx.Int8Type(), 0) + startType := llvm.FunctionType(ptr, []llvm.Type{ptr, i32}, false) + start := llvm.AddFunction(mod, StartSymbol(callee.Name()), startType) + suspend := llvm.AddFunction(mod, SuspendSymbol, llvm.FunctionType(ctx.VoidType(), nil, false)) + markFunction(ctx, suspend) + callerType := llvm.FunctionType(i32, []llvm.Type{ptr, i32}, false) + caller := llvm.AddFunction(mod, "caller", callerType) + markFunction(ctx, caller) + block = ctx.AddBasicBlock(caller, "entry") + builder.SetInsertPointAtEnd(block) + dynamicCall := builder.CreateCall(sig, caller.Param(0), []llvm.Value{caller.Param(1)}, "dynamic") + markCall(ctx, dynamicCall) + constantCall := builder.CreateCall(sig, start, []llvm.Value{dynamicCall}, "constant") + markCall(ctx, constantCall) + suspendCall := builder.CreateCall(suspend.GlobalValueType(), suspend, nil, "") + markCall(ctx, suspendCall) + builder.CreateRet(builder.CreateMul(constantCall, llvm.ConstInt(i32, 2, false), "result")) + + lowered, err := lowerPrototype(mod, targetData) + if err != nil { + t.Fatal(err) + } + var root loweredState + for _, state := range lowered { + if state.layout.plan.function == caller { + root = state + break + } + } + if root.entry.IsNil() || start.IsNil() { + t.Fatal("indirect state machine entries were not emitted") + } + harness := defineStateMachineHarness(mod, targetData, root, []llvm.Value{ + start, llvm.ConstInt(i32, 5, false), + }) + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify indirect state machine: %v\n%s", err, mod.String()) + } + + options := llvm.NewMCJITCompilerOptions() + options.SetMCJITOptimizationLevel(0) + engine, err := llvm.NewMCJITCompiler(mod, options) + if err != nil { + t.Fatal(err) + } + moduleOwned = false + defer engine.Dispose() + + result := engine.RunFunction(harness, nil) + defer result.Dispose() + if got := result.Int(true); got != 14 { + t.Fatalf("indirect state machine result = %d, want 14", got) + } +} + +func defineStateMachineHarness( + mod llvm.Module, targetData llvm.TargetData, lowered loweredState, params []llvm.Value, +) llvm.Value { + ctx := mod.Context() + abi := newResumeABI(ctx, targetData) + i8 := ctx.Int8Type() + i32 := ctx.Int32Type() + + childStorageType := llvm.ArrayType(i8, 4096) + childStorage := llvm.AddGlobal(mod, childStorageType, "child.storage") + childStorage.SetInitializer(llvm.ConstNull(childStorageType)) + childStorage.SetAlignment(16) + childOffset := llvm.AddGlobal(mod, i32, "child.offset") + childOffset.SetInitializer(llvm.ConstInt(i32, 0, false)) + + alloc := mod.NamedFunction(frameAllocName) + block := ctx.AddBasicBlock(alloc, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + offset := builder.CreateLoad(i32, childOffset, "offset") + builder.CreateStore( + builder.CreateAdd(offset, llvm.ConstInt(i32, 256, false), ""), + childOffset, + ) + builder.CreateRet(builder.CreateInBoundsGEP(i8, childStorage, []llvm.Value{offset}, "frame")) + + free := mod.NamedFunction(frameFreeName) + block = ctx.AddBasicBlock(free, "entry") + builder.SetInsertPointAtEnd(block) + builder.CreateRetVoid() + + close := mod.NamedFunction(frameCloseName) + block = ctx.AddBasicBlock(close, "entry") + builder.SetInsertPointAtEnd(block) + builder.CreateRetVoid() + + root := llvm.AddGlobal(mod, lowered.layout.typ, "root.frame") + root.SetInitializer(llvm.ConstNull(lowered.layout.typ)) + root.SetAlignment(lowered.layout.alignment) + context := llvm.AddGlobal(mod, abi.contextType, "resume.context") + context.SetInitializer(llvm.ConstNull(abi.contextType)) + + run := llvm.AddFunction(mod, "run.state.machine", llvm.FunctionType(i32, nil, false)) + entryBlock := ctx.AddBasicBlock(run, "entry") + loopBlock := ctx.AddBasicBlock(run, "loop") + resumeBlock := ctx.AddBasicBlock(run, "resume") + popBlock := ctx.AddBasicBlock(run, "pop") + doneBlock := ctx.AddBasicBlock(run, "done") + failedBlock := ctx.AddBasicBlock(run, "failed") + + builder.SetInsertPointAtEnd(entryBlock) + builder.CreateStore( + llvm.ConstNull(abi.ptr), + builder.CreateStructGEP(lowered.layout.typ, root, 0, ""), + ) + builder.CreateStore( + lowered.descriptor, + builder.CreateStructGEP(lowered.layout.typ, root, 1, ""), + ) + builder.CreateStore( + llvm.ConstInt(i32, 0, false), + builder.CreateStructGEP(lowered.layout.typ, root, 2, ""), + ) + param := 0 + for _, slot := range lowered.layout.plan.slots { + if slot.kind == slotParameter { + builder.CreateStore( + params[param], + builder.CreateStructGEP( + lowered.layout.typ, root, lowered.layout.fieldIndex(slot.id), "", + ), + ) + param++ + } + } + builder.CreateStore(root, builder.CreateStructGEP(abi.contextType, context, 0, "")) + builder.CreateStore( + llvm.ConstNull(abi.ptr), + builder.CreateStructGEP(abi.contextType, context, 1, ""), + ) + builder.CreateBr(loopBlock) + + builder.SetInsertPointAtEnd(loopBlock) + topField := builder.CreateStructGEP(abi.contextType, context, 0, "") + top := builder.CreateLoad(abi.ptr, topField, "top") + builder.CreateCondBr( + builder.CreateICmp(llvm.IntNE, top, llvm.ConstNull(abi.ptr), ""), + resumeBlock, + doneBlock, + ) + + framePrefix := ctx.StructType([]llvm.Type{abi.ptr, abi.ptr, i32}, false) + builder.SetInsertPointAtEnd(resumeBlock) + descriptor := builder.CreateLoad( + abi.ptr, builder.CreateStructGEP(framePrefix, top, 1, ""), "descriptor", + ) + resume := builder.CreateLoad( + abi.ptr, builder.CreateStructGEP(abi.descriptorType, descriptor, 0, ""), "resume.entry", + ) + action := builder.CreateCall(abi.entryType, resume, []llvm.Value{context, top}, "action") + switchAction := builder.CreateSwitch(action, failedBlock, 3) + switchAction.AddCase(llvm.ConstInt(i8, actionContinue, false), loopBlock) + switchAction.AddCase(llvm.ConstInt(i8, actionReturn, false), popBlock) + switchAction.AddCase(llvm.ConstInt(i8, actionSuspend, false), loopBlock) + + builder.SetInsertPointAtEnd(popBlock) + parent := builder.CreateLoad( + abi.ptr, builder.CreateStructGEP(framePrefix, top, 0, ""), "parent", + ) + builder.CreateStore(parent, topField) + builder.CreateStore(top, builder.CreateStructGEP(abi.contextType, context, 1, "")) + builder.CreateBr(loopBlock) + + builder.SetInsertPointAtEnd(doneBlock) + builder.CreateRet(builder.CreateLoad( + i32, + builder.CreateStructGEP( + lowered.layout.typ, root, + lowered.layout.fieldIndex(lowered.layout.plan.resultSlot), "", + ), + "result", + )) + + builder.SetInsertPointAtEnd(failedBlock) + builder.CreateRet(llvm.ConstInt(i32, ^uint64(0), true)) + return run +} + +func TestLowerPrototypeSupportsDynamicAlloca(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("dynamic") + defer mod.Dispose() + targetData := llvm.NewTargetData("e-m:e-p:32:32-i64:64-n32:64-S128") + defer targetData.Dispose() + + i32 := ctx.Int32Type() + ptr := llvm.PointerType(i32, 0) + calleeType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr}, false) + callee := llvm.AddFunction(mod, "callee", calleeType) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), []llvm.Type{i32}, false)) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + local := builder.CreateArrayAlloca(i32, fn.Param(0), "local") + call := builder.CreateCall(calleeType, callee, []llvm.Value{local}, "") + markCall(ctx, call) + builder.CreateRetVoid() + + if _, err := lowerPrototype(mod, targetData); err != nil { + t.Fatal(err) + } + if ir := mod.String(); strings.Contains(ir, "%local = alloca") || + !strings.Contains(ir, "@__llgo_wasm_resume_alloc_dynamic") { + t.Fatalf("dynamic alloca was not lowered:\n%s", ir) + } +} + +func TestLowerPrototypeRejectsVariadicResumeCall(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("variadic") + defer mod.Dispose() + targetData := llvm.NewTargetData("e-m:e-p:32:32-i64:64-n32:64-S128") + defer targetData.Dispose() + + i32 := ctx.Int32Type() + variadicType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{i32}, true) + callee := llvm.AddFunction(mod, "callee", variadicType) + fn := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), nil, false)) + markFunction(ctx, fn) + block := ctx.AddBasicBlock(fn, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + call := builder.CreateCall( + variadicType, callee, []llvm.Value{llvm.ConstInt(i32, 1, false)}, "", + ) + markCall(ctx, call) + builder.CreateRetVoid() + + if _, err := lowerPrototype(mod, targetData); err == nil || + !strings.Contains(err.Error(), "variadic") { + t.Fatalf("lowerPrototype error = %v", err) + } +} diff --git a/internal/wasmresume/unwind.go b/internal/wasmresume/unwind.go new file mode 100644 index 0000000000..5fe33c0ed2 --- /dev/null +++ b/internal/wasmresume/unwind.go @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import ( + "fmt" + + "github.com/xgo-dev/llvm" +) + +func lowerUnwindMarkers( + ctx llvm.Context, entry llvm.Value, plan framePlan, unwindField llvm.Value, +) error { + var markers []llvm.Value + for block := entry.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instr := block.FirstInstruction(); !instr.IsNil(); instr = llvm.NextInstruction(instr) { + if instr.InstructionOpcode() != llvm.Call { + continue + } + switch instr.CalledValue().Name() { + case RegisterUnwindSymbol, ClearUnwindSymbol: + markers = append(markers, instr) + } + } + } + if len(markers) == 0 { + if plan.unwindSlot != 0 { + return fmt.Errorf("unwind frame has no registration marker") + } + return nil + } + if plan.unwindSlot == 0 || unwindField.IsNil() { + return fmt.Errorf("unwind marker has no frame slot") + } + + builder := ctx.NewBuilder() + defer builder.Dispose() + unwindType := plan.slots[plan.unwindSlot-1].typ + for _, marker := range markers { + builder.SetInsertPointBefore(marker) + value := llvm.ConstNull(unwindType) + if marker.CalledValue().Name() == RegisterUnwindSymbol { + if marker.OperandsCount() < 3 { + return fmt.Errorf("invalid unwind registration marker") + } + value = marker.Operand(0) + } + builder.CreateStore(value, unwindField) + marker.EraseFromParentAsInstruction() + } + return nil +} diff --git a/internal/wasmresume/unwind_test.go b/internal/wasmresume/unwind_test.go new file mode 100644 index 0000000000..2441b1bc14 --- /dev/null +++ b/internal/wasmresume/unwind_test.go @@ -0,0 +1,245 @@ +package wasmresume + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestLowerUnwindMarkersStoresAndClearsFrameSlot(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("unwind-markers") + defer mod.Dispose() + + ptr := llvm.PointerType(ctx.Int8Type(), 0) + entryType := llvm.FunctionType(ctx.Int8Type(), []llvm.Type{ptr, ptr}, false) + entry := llvm.AddFunction(mod, "resume", entryType) + block := ctx.AddBasicBlock(entry, "entry") + handler := ctx.AddBasicBlock(entry, "handler") + registerType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr, ptr}, false) + register := llvm.AddFunction(mod, RegisterUnwindSymbol, registerType) + clearType := llvm.FunctionType(ctx.VoidType(), nil, false) + clear := llvm.AddFunction(mod, ClearUnwindSymbol, clearType) + token := llvm.AddGlobal(mod, ctx.Int8Type(), "token") + + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + field := builder.CreateAlloca(ptr, "unwind.slot") + builder.CreateCall(registerType, register, []llvm.Value{ + token, + llvm.BlockAddress(entry, handler), + }, "") + builder.CreateCall(clearType, clear, nil, "") + builder.CreateRet(llvm.ConstInt(ctx.Int8Type(), actionReturn, false)) + builder.SetInsertPointAtEnd(handler) + builder.CreateRet(llvm.ConstInt(ctx.Int8Type(), actionReturn, false)) + + plan := framePlan{ + slots: []frameSlot{{id: 1, kind: slotUnwind, typ: ptr}}, + unwindSlot: 1, + } + if err := lowerUnwindMarkers(ctx, entry, plan, field); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyFunction(entry, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify lowered marker function: %v\n%s", err, mod.String()) + } + ir := mod.String() + if strings.Contains(ir, "call void @"+RegisterUnwindSymbol) || + strings.Contains(ir, "call void @"+ClearUnwindSymbol) { + t.Fatalf("unwind marker call remains:\n%s", ir) + } + for _, want := range []string{ + "store ptr @token, ptr %unwind.slot", + "store ptr null, ptr %unwind.slot", + } { + if !strings.Contains(ir, want) { + t.Fatalf("lowered unwind markers are missing %q:\n%s", want, ir) + } + } +} + +func TestLowerUnwindMarkersValidatesPlan(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + ptr := llvm.PointerType(ctx.Int8Type(), 0) + + t.Run("missing marker", func(t *testing.T) { + mod := ctx.NewModule("missing-marker") + defer mod.Dispose() + entry := llvm.AddFunction(mod, "resume", llvm.FunctionType(ctx.Int8Type(), nil, false)) + block := ctx.AddBasicBlock(entry, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + field := builder.CreateAlloca(ptr, "unwind.slot") + builder.CreateRet(llvm.ConstInt(ctx.Int8Type(), actionReturn, false)) + + plan := framePlan{ + slots: []frameSlot{{id: 1, kind: slotUnwind, typ: ptr}}, + unwindSlot: 1, + } + if err := lowerUnwindMarkers(ctx, entry, plan, field); err == nil { + t.Fatal("lowerUnwindMarkers accepted a missing registration") + } + if err := lowerUnwindMarkers(ctx, entry, framePlan{}, llvm.Value{}); err != nil { + t.Fatalf("marker-free frame returned %v", err) + } + }) + + t.Run("missing slot", func(t *testing.T) { + mod := ctx.NewModule("missing-slot") + defer mod.Dispose() + registerType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr, ptr}, false) + register := llvm.AddFunction(mod, RegisterUnwindSymbol, registerType) + entry := llvm.AddFunction(mod, "resume", llvm.FunctionType(ctx.Int8Type(), nil, false)) + block := ctx.AddBasicBlock(entry, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + builder.CreateCall(registerType, register, []llvm.Value{ + llvm.ConstNull(ptr), + llvm.ConstNull(ptr), + }, "") + builder.CreateRet(llvm.ConstInt(ctx.Int8Type(), actionReturn, false)) + if err := lowerUnwindMarkers(ctx, entry, framePlan{}, llvm.Value{}); err == nil { + t.Fatal("lowerUnwindMarkers accepted a marker without a slot") + } + }) + + t.Run("invalid register", func(t *testing.T) { + mod := ctx.NewModule("invalid-register") + defer mod.Dispose() + registerType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr}, false) + register := llvm.AddFunction(mod, RegisterUnwindSymbol, registerType) + entry := llvm.AddFunction(mod, "resume", llvm.FunctionType(ctx.Int8Type(), nil, false)) + block := ctx.AddBasicBlock(entry, "entry") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(block) + field := builder.CreateAlloca(ptr, "unwind.slot") + builder.CreateCall(registerType, register, []llvm.Value{llvm.ConstNull(ptr)}, "") + builder.CreateRet(llvm.ConstInt(ctx.Int8Type(), actionReturn, false)) + plan := framePlan{ + slots: []frameSlot{{id: 1, kind: slotUnwind, typ: ptr}}, + unwindSlot: 1, + } + if err := lowerUnwindMarkers(ctx, entry, plan, field); err == nil { + t.Fatal("lowerUnwindMarkers accepted an invalid registration") + } + }) +} + +func TestFindUnwindPlan(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + ptr := llvm.PointerType(ctx.Int8Type(), 0) + registerType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr, ptr}, false) + + newFunction := func(mod llvm.Module, handler llvm.Value) (llvm.Value, llvm.Value) { + register := llvm.AddFunction(mod, RegisterUnwindSymbol, registerType) + fn := llvm.AddFunction(mod, "f", llvm.FunctionType(ctx.VoidType(), nil, false)) + entry := ctx.AddBasicBlock(fn, "entry") + target := ctx.AddBasicBlock(fn, "target") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(entry) + call := builder.CreateCall(registerType, register, []llvm.Value{ + llvm.ConstNull(ptr), + handler, + }, "") + builder.CreateBr(target) + builder.SetInsertPointAtEnd(target) + builder.CreateRetVoid() + return fn, call + } + + mod := ctx.NewModule("valid-unwind") + fn := llvm.AddFunction(mod, "f", llvm.FunctionType(ctx.VoidType(), nil, false)) + entry := ctx.AddBasicBlock(fn, "entry") + target := ctx.AddBasicBlock(fn, "target") + register := llvm.AddFunction(mod, RegisterUnwindSymbol, registerType) + builder := ctx.NewBuilder() + builder.SetInsertPointAtEnd(entry) + builder.CreateCall(registerType, register, []llvm.Value{ + llvm.ConstNull(ptr), + llvm.BlockAddress(fn, target), + }, "") + builder.CreateBr(target) + builder.SetInsertPointAtEnd(target) + builder.CreateRetVoid() + builder.Dispose() + plan, err := findUnwindPlan(fn) + if err != nil || plan.block != target || plan.typ != ptr { + t.Fatalf("findUnwindPlan = %+v, %v", plan, err) + } + mod.Dispose() + + mod = ctx.NewModule("invalid-unwind") + fn, call := newFunction(mod, llvm.ConstNull(ptr)) + if _, err := findUnwindPlan(fn); err == nil { + t.Fatal("findUnwindPlan accepted a non-block handler") + } + call.EraseFromParentAsInstruction() + mod.Dispose() +} + +func TestUnwindOnlyFunctionUsesStateMachine(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("unwind-only") + defer mod.Dispose() + mod.SetTarget("wasm32-unknown-unknown") + mod.SetDataLayout("e-m:e-p:32:32-i64:64-n32:64-S128") + targetData := llvm.NewTargetData(mod.DataLayout()) + defer targetData.Dispose() + + ptr := llvm.PointerType(ctx.Int8Type(), 0) + registerType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptr, ptr}, false) + register := llvm.AddFunction(mod, RegisterUnwindSymbol, registerType) + clearType := llvm.FunctionType(ctx.VoidType(), nil, false) + clear := llvm.AddFunction(mod, ClearUnwindSymbol, clearType) + fn := llvm.AddFunction(mod, "with.c.defer", llvm.FunctionType(ctx.VoidType(), nil, false)) + markFunction(ctx, fn) + entry := ctx.AddBasicBlock(fn, "entry") + handler := ctx.AddBasicBlock(fn, "handler") + done := ctx.AddBasicBlock(fn, "done") + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(entry) + builder.CreateCall(registerType, register, []llvm.Value{ + llvm.ConstNull(ptr), + llvm.BlockAddress(fn, handler), + }, "") + builder.CreateBr(done) + builder.SetInsertPointAtEnd(handler) + builder.CreateCall(clearType, clear, nil, "") + builder.CreateRetVoid() + builder.SetInsertPointAtEnd(done) + builder.CreateCall(clearType, clear, nil, "") + builder.CreateRetVoid() + + if err := Lower(mod, targetData); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify unwind-only state machine: %v\n%s", err, mod.String()) + } + ir := mod.String() + for _, want := range []string{ + "define internal i8 @__llgo_wasm_resume.with.c.defer", + "define void @with.c.defer()", + "i32 1, label %handler", + } { + if !strings.Contains(ir, want) { + t.Fatalf("unwind-only state machine is missing %q:\n%s", want, ir) + } + } + if strings.Contains(ir, "call void @"+RegisterUnwindSymbol) || + strings.Contains(ir, "call void @"+ClearUnwindSymbol) { + t.Fatalf("unwind marker remains in unwind-only state machine:\n%s", ir) + } +} diff --git a/runtime/internal/runtime/goroutine_func_default.go b/runtime/internal/runtime/goroutine_func_default.go new file mode 100644 index 0000000000..42dd35cd03 --- /dev/null +++ b/runtime/internal/runtime/goroutine_func_default.go @@ -0,0 +1,11 @@ +//go:build !llgo || !wasm || !llgo.wasm_resume || (!js && !wasip1) || (wasip1 && llgo.wasi_threads) + +package runtime + +import "unsafe" + +// goroutineFunc is the target-independent entry ABI between compiler-generated +// goroutine wrappers and the selected stackful scheduler. +// +//llgo:type C +type goroutineFunc func(unsafe.Pointer) unsafe.Pointer diff --git a/runtime/internal/runtime/goroutine_func_wasm_resume.go b/runtime/internal/runtime/goroutine_func_wasm_resume.go new file mode 100644 index 0000000000..d2b7786c95 --- /dev/null +++ b/runtime/internal/runtime/goroutine_func_wasm_resume.go @@ -0,0 +1,14 @@ +//go:build llgo && wasm && llgo.wasm_resume && (js || wasip1) && !(wasip1 && llgo.wasi_threads) + +package runtime + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/internal/wasmresume" +) + +// goroutineFunc is a compiler-generated start entry for one resumable G. +// +//llgo:type C +type goroutineFunc func(*wasmresume.Context, unsafe.Pointer) *wasmresume.Frame diff --git a/runtime/internal/runtime/proc.go b/runtime/internal/runtime/proc.go index ae66de6057..fc38269f37 100644 --- a/runtime/internal/runtime/proc.go +++ b/runtime/internal/runtime/proc.go @@ -22,12 +22,6 @@ import ( c "github.com/goplus/llgo/runtime/internal/clite" ) -// goroutineFunc is the target-independent entry ABI between compiler-generated -// goroutine wrappers and the runtime scheduler. -// -//llgo:type C -type goroutineFunc func(unsafe.Pointer) unsafe.Pointer - // runtimeContext owns one G and its target-specific suspended execution state. // M and P ownership belongs to the selected scheduler backend and can outlive, // or be shared by, multiple runtime contexts. diff --git a/runtime/internal/runtime/proc_wasip1.go b/runtime/internal/runtime/proc_wasip1.go index cfa88861e4..6acb25b611 100644 --- a/runtime/internal/runtime/proc_wasip1.go +++ b/runtime/internal/runtime/proc_wasip1.go @@ -1,4 +1,4 @@ -//go:build llgo && wasip1 && wasm && !llgo.wasi_threads +//go:build llgo && wasip1 && wasm && !llgo.wasi_threads && !llgo.wasm_resume /* * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. diff --git a/runtime/internal/runtime/proc_wasm.go b/runtime/internal/runtime/proc_wasm.go index d0f07791d3..10eefcee2a 100644 --- a/runtime/internal/runtime/proc_wasm.go +++ b/runtime/internal/runtime/proc_wasm.go @@ -1,4 +1,4 @@ -//go:build llgo && js && wasm +//go:build llgo && js && wasm && !llgo.wasm_resume /* * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. diff --git a/runtime/internal/runtime/proc_wasm_resume.go b/runtime/internal/runtime/proc_wasm_resume.go new file mode 100644 index 0000000000..f66f5d6f24 --- /dev/null +++ b/runtime/internal/runtime/proc_wasm_resume.go @@ -0,0 +1,267 @@ +//go:build llgo && wasm && llgo.wasm_resume && (js || wasip1) && !(wasip1 && llgo.wasi_threads) + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + "unsafe" + + c "github.com/goplus/llgo/runtime/internal/clite" + "github.com/goplus/llgo/runtime/internal/runqueue" + "github.com/goplus/llgo/runtime/internal/wasmresume" +) + +type runtimeContextPlatform struct { + context wasmresume.Context + runqNext *g + runqQueued bool + unwind unsafe.Pointer +} + +var wasmSched struct { + m m + p p + runq runqueue.Queue[*g] + started bool + mainExited bool +} + +func initRuntimeContext(ctx *runtimeContext, callergp *g, status uint32) *g { + gp := initG(ctx, callergp, status) + if status == _Grunning { + initWasmScheduler(gp) + } + return gp +} + +func initWasmScheduler(gp *g) { + if wasmSched.started { + fatal("runtime: WebAssembly scheduler initialized twice") + return + } + wasmSched.started = true + mp := &wasmSched.m + pp := &wasmSched.p + mp.curg = gp + mp.p = pp + mp.id = nextMid(mp) + pp.id = nextPid(pp) + setpstatus(pp, _Prunning) + pp.m = mp + gp.m = mp +} + +//go:linkname wasmMainStart C.__llgo_wasm_start.__llgo_wasm_main +func wasmMainStart(*wasmresume.Context, unsafe.Pointer) *wasmresume.Frame + +// RunWasmMain runs package initialization and main.main through the resumable +// ABI while the host entry remains on its original stack. +func RunWasmMain() { + gp := getg() + if gp == nil || !gp.isMain { + fatal("runtime: invalid WebAssembly main goroutine") + return + } + gp.context.platform.context.Start( + wasmMainStart(&gp.context.platform.context, nil), + ) + + for { + action := runWasmResumeContext(gp) + status := readgstatus(gp) + if action == wasmresume.Return { + if status != _Grunning { + fatal("runtime: invalid completed WebAssembly goroutine") + return + } + casgstatus(gp, _Grunning, _Gdead) + status = _Gdead + if gp.isMain { + releaseWasmContext(gp) + return + } + } else if action != wasmresume.Suspend || status == _Grunning { + fatal("runtime: invalid WebAssembly resume action") + return + } + + releaseWasmOwnership(gp) + if status == _Gdead { + releaseWasmContext(gp) + } + + gp = wasmSched.runq.Pop() + if gp == nil { + if wasmSched.mainExited { + fatal("no goroutines (main called runtime.Goexit) - deadlock!") + } else { + fatal("all goroutines are asleep - deadlock!") + } + return + } + } +} + +func runWasmResumeContext(gp *g) wasmresume.Action { + if readgstatus(gp) == _Grunnable { + casgstatus(gp, _Grunnable, _Grunning) + } + mp := &wasmSched.m + pp := &wasmSched.p + mp.curg = gp + pp.m = mp + gp.m = mp + setg(gp) + + platform := &gp.context.platform + unwind := c.AllocaSigjmpBuf() + previous := platform.unwind + platform.unwind = unwind + if c.Sigsetjmp(unwind, 0) != 0 { + if !platform.context.Unwind(unsafe.Pointer(gp.defer_), FreeRoot) { + platform.unwind = previous + if gp.goexit { + casgstatus(gp, _Grunning, _Gdead) + if gp.isMain { + wasmSched.mainExited = true + } + return wasmresume.Suspend + } + Rethrow(nil) + return wasmresume.Return + } + } + action := platform.context.Run() + platform.unwind = previous + return action +} + +func releaseWasmOwnership(gp *g) { + if gp != nil { + gp.m = nil + } + wasmSched.m.curg = nil +} + +func newprocBackend(fn goroutineFunc, arg unsafe.Pointer, _ uintptr, callergp *g) { + gp := newproc1(fn, arg, callergp) + gp.startfn = nil + gp.startarg = nil + gp.context.platform.context.Start(fn(&gp.context.platform.context, arg)) + if !wasmSched.runq.Push(gp) { + fatal("runtime: invalid run queue insertion") + } +} + +func releaseWasmContext(gp *g) { + if gp == nil || gp.context == nil { + return + } + ctx := gp.context + ctx.platform.context.Close(FreeRoot) + freeRuntimeContext(ctx) +} + +func goschedBackend() { + gp := getg() + casgstatus(gp, _Grunning, _Grunnable) + if !wasmSched.runq.Push(gp) { + fatal("runtime: invalid run queue insertion") + return + } + wasmresume.SuspendCurrent() +} + +func gopark() { + gp := getg() + casgstatus(gp, _Grunning, _Gwaiting) + wasmresume.SuspendCurrent() +} + +func goready(gp *g) { + if gp == nil { + fatal("runtime: ready of nil goroutine") + return + } + casgstatus(gp, _Gwaiting, _Grunnable) + if !wasmSched.runq.Push(gp) { + fatal("runtime: invalid run queue insertion") + } +} + +func goexitBackend(gp *g) { + casgstatus(gp, _Grunning, _Gdead) + if gp.isMain { + wasmSched.mainExited = true + } + wasmresume.SuspendCurrent() + fatal("runtime: resumed dead WebAssembly goroutine") +} + +//go:linkname wasmResumeAlloc __llgo_wasm_resume_alloc +func wasmResumeAlloc(ctx *wasmresume.Context, size, align uintptr) unsafe.Pointer { + return ctx.AllocateFrame(size, align, AllocRoot) +} + +//go:linkname wasmResumeAllocDynamic __llgo_wasm_resume_alloc_dynamic +func wasmResumeAllocDynamic(ctx *wasmresume.Context, size, align uintptr) unsafe.Pointer { + return ctx.AllocateFrame(size, align, AllocRoot) +} + +//go:linkname wasmResumeFree __llgo_wasm_resume_free +func wasmResumeFree(ctx *wasmresume.Context, frame *wasmresume.Frame) { + ctx.ReleaseFrame(frame, FreeRoot) +} + +//go:linkname wasmResumeClose __llgo_wasm_resume_close +func wasmResumeClose(ctx *wasmresume.Context) { + ctx.Close(FreeRoot) +} + +// CurrentGForTesting returns an opaque handle suitable for ReadyForTesting. +func CurrentGForTesting() unsafe.Pointer { + return unsafe.Pointer(getg()) +} + +// ParkForTesting parks the current G until another G marks it runnable. +func ParkForTesting() { + gopark() +} + +// ReadyForTesting makes a G previously parked by ParkForTesting runnable. +func ReadyForTesting(handle unsafe.Pointer) { + goready((*g)(handle)) +} + +// SchedulerStateForTesting reports single-worker queue and ownership state. +func SchedulerStateForTesting() (runq uintptr, mid int64, pid int32) { + return wasmSched.runq.Len(), wasmSched.m.id, wasmSched.p.id +} + +func GMPForTesting() (goid, parentGoid uint64, mid int64, pid int32, gstatus, pstatus uint32, linked bool) { + gp := getg() + if gp == nil || gp.m == nil || gp.m.p == nil { + return + } + mp := gp.m + pp := mp.p + ctx := gp.context + return gp.goid, gp.parentGoid, mp.id, pp.id, readgstatus(gp), readpstatus(pp), + mp == &wasmSched.m && pp == &wasmSched.p && + mp.curg == gp && pp.m == mp && ctx != nil && &ctx.g == gp +} diff --git a/runtime/internal/runtime/z_default.go b/runtime/internal/runtime/z_default.go index 71823fa552..e4bcc48431 100644 --- a/runtime/internal/runtime/z_default.go +++ b/runtime/internal/runtime/z_default.go @@ -1,4 +1,4 @@ -//go:build !baremetal +//go:build !baremetal && !(llgo && wasm && llgo.wasm_resume && (js || wasip1) && !(wasip1 && llgo.wasi_threads)) package runtime diff --git a/runtime/internal/runtime/z_wasm_resume.go b/runtime/internal/runtime/z_wasm_resume.go new file mode 100644 index 0000000000..0c78c38bcf --- /dev/null +++ b/runtime/internal/runtime/z_wasm_resume.go @@ -0,0 +1,39 @@ +//go:build llgo && wasm && llgo.wasm_resume && (js || wasip1) && !(wasip1 && llgo.wasi_threads) + +package runtime + +import ( + c "github.com/goplus/llgo/runtime/internal/clite" + "github.com/goplus/llgo/runtime/internal/clite/debug" +) + +var ( + printFormatPrefixInt = c.Str("%lld") + printFormatPrefixUInt = c.Str("%llu") + printFormatPrefixHex = c.Str("%llx") +) + +// Rethrow transfers pending panic/Goexit processing to the scheduler's active +// native catch. The scheduler then redirects the explicit frame chain to the +// defer owner recorded by the compiler. +func Rethrow(link *Defer) { + gp := getg() + if gp.panic_ == nil && !gp.goexit { + return + } + gp.defer_ = link + if unwind := gp.context.platform.unwind; unwind != nil { + c.Siglongjmp(unwind, 1) + return + } + if ptr := gp.panic_; ptr != nil { + TracePanic(*(*any)(ptr)) + if PanicTraceback == nil || !PanicTraceback(2) { + debug.PrintStack(2) + } + c.Free(ptr) + c.Exit(2) + return + } + fatal("runtime: Goexit outside WebAssembly scheduler") +} diff --git a/runtime/internal/wasmresume/resume.go b/runtime/internal/wasmresume/resume.go new file mode 100644 index 0000000000..a03d3a72fd --- /dev/null +++ b/runtime/internal/wasmresume/resume.go @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package wasmresume defines the runtime half of LLGo's experimental +// WebAssembly resumable call ABI. +package wasmresume + +import "unsafe" + +// SuspendCurrent yields the active resumable frame to its scheduler. The +// compiler replaces calls to SuspendCurrent with a frame-PC transition; no +// function body is linked into the final WebAssembly module. +func SuspendCurrent() { + panic("wasmresume: SuspendCurrent was not lowered") +} + +// Action tells Context what a resume entry did. +type Action uint8 + +const ( + // Continue means that execution can continue immediately. The resume entry + // may have pushed a child frame or advanced within the current frame. + Continue Action = iota + + // Return means that the current frame completed normally. + Return + + // Suspend returns control to the scheduler without changing the frame chain. + Suspend +) + +// Resume is the non-suspending indirect-call signature for generated entries. +// +//llgo:type C +type Resume func(*Context, *Frame) Action + +// Allocator allocates one GC-scanned root block. +// +//llgo:type C +type Allocator func(uintptr) unsafe.Pointer + +// Releaser releases one block previously returned by Allocator. +// +//llgo:type C +type Releaser func(unsafe.Pointer) + +// Descriptor contains immutable state shared by every invocation of a +// generated function. +type Descriptor struct { + Resume Resume + FrameSize uintptr + FrameAlign uintptr + UnwindOffset uintptr + UnwindPC uint32 +} + +// Frame is the common prefix of every generated function frame. Generated +// frame types must embed Frame as their first field. +type Frame struct { + Parent *Frame + Descriptor *Descriptor + PC uint32 +} + +// Context owns the active frame chain for one logical goroutine. +type Context struct { + top *Frame + returned *Frame + storage frameStorage +} + +// Start installs the root frame of a new logical goroutine. +func (c *Context) Start(frame *Frame) { + if frame == nil || frame.Parent != nil || frame.Descriptor == nil || c.top != nil { + panic("wasmresume: invalid root frame") + } + c.returned = nil + c.top = frame +} + +// AllocateFrame allocates stable, root-scanned storage for a generated frame. +func (c *Context) AllocateFrame( + size, align uintptr, allocate Allocator, +) unsafe.Pointer { + return c.storage.allocate(size, align, allocate) +} + +// ReleaseFrame reclaims the most recently completed generated frame. +func (c *Context) ReleaseFrame(frame *Frame, release Releaser) { + if frame == nil || frame.Descriptor == nil { + panic("wasmresume: invalid completed frame") + } + c.storage.releaseFrame(unsafe.Pointer(frame), frame.Descriptor.FrameSize, release) +} + +// Unwind discards frames above the defer owner and redirects that owner to its +// generated panic/defer state. +func (c *Context) Unwind(deferFrame unsafe.Pointer, release Releaser) bool { + if deferFrame == nil { + return false + } + var owner *Frame + for frame := c.top; frame != nil; frame = frame.Parent { + descriptor := frame.Descriptor + if descriptor == nil || descriptor.UnwindOffset == 0 || + descriptor.UnwindPC == 0 { + continue + } + slot := (*unsafe.Pointer)(unsafe.Add(unsafe.Pointer(frame), descriptor.UnwindOffset)) + if *slot == deferFrame { + owner = frame + break + } + } + if owner == nil { + return false + } + for c.top != owner { + frame := c.top + c.top = frame.Parent + c.storage.releaseFrame(unsafe.Pointer(frame), frame.Descriptor.FrameSize, release) + } + c.returned = nil + owner.PC = owner.Descriptor.UnwindPC + return true +} + +// Close releases every frame storage segment owned by the context. +func (c *Context) Close(release Releaser) { + c.storage.close(release) + c.top = nil + c.returned = nil +} + +// Top returns the active frame. +func (c *Context) Top() *Frame { + return c.top +} + +// Push links frame as the active child of the current frame. +func (c *Context) Push(frame *Frame, descriptor *Descriptor) { + if c.top == nil { + c.returned = nil + } + frame.Parent = c.top + frame.Descriptor = descriptor + frame.PC = 0 + c.top = frame +} + +// TakeReturned returns the child frame that completed immediately before the +// active frame resumed. It also transfers ownership back to the caller. +func (c *Context) TakeReturned() *Frame { + frame := c.returned + c.returned = nil + return frame +} + +// Run resumes the active frame chain until it completes or suspends. +func (c *Context) Run() Action { + for c.top != nil { + frame := c.top + switch frame.Descriptor.Resume(c, frame) { + case Continue: + case Return: + c.top = frame.Parent + c.returned = frame + case Suspend: + return Suspend + default: + panic("wasmresume: invalid resume action") + } + } + return Return +} diff --git a/runtime/internal/wasmresume/resume_test.go b/runtime/internal/wasmresume/resume_test.go new file mode 100644 index 0000000000..2463429abe --- /dev/null +++ b/runtime/internal/wasmresume/resume_test.go @@ -0,0 +1,234 @@ +package wasmresume + +import ( + "testing" + "unsafe" +) + +type testRootFrame struct { + Frame + direct testLeafFrame + indirect testLeafFrame + value int +} + +type testLeafFrame struct { + Frame + value int +} + +var ( + testRootDescriptor = Descriptor{Resume: resumeTestRoot} + testAddDescriptor = Descriptor{Resume: resumeTestAdd} + testMulDescriptor = Descriptor{Resume: resumeTestMul} +) + +func resumeTestRoot(ctx *Context, raw *Frame) Action { + frame := (*testRootFrame)(unsafe.Pointer(raw)) + switch frame.PC { + case 0: + frame.PC = 1 + frame.direct.value = 4 + ctx.Push(&frame.direct.Frame, &testAddDescriptor) + return Continue + case 1: + if returned := ctx.TakeReturned(); returned != &frame.direct.Frame { + panic("unexpected direct child frame") + } + frame.value = frame.direct.value + frame.PC = 2 + frame.indirect.value = frame.value + descriptor := &testAddDescriptor + if frame.value == 7 { + descriptor = &testMulDescriptor + } + ctx.Push(&frame.indirect.Frame, descriptor) + return Continue + case 2: + if returned := ctx.TakeReturned(); returned != &frame.indirect.Frame { + panic("unexpected indirect child frame") + } + frame.value = frame.indirect.value + return Return + default: + panic("unexpected root resume PC") + } +} + +func resumeTestAdd(_ *Context, raw *Frame) Action { + frame := (*testLeafFrame)(unsafe.Pointer(raw)) + frame.value += 3 + return Return +} + +func resumeTestMul(_ *Context, raw *Frame) Action { + frame := (*testLeafFrame)(unsafe.Pointer(raw)) + switch frame.PC { + case 0: + frame.PC = 1 + return Suspend + case 1: + frame.value *= 2 + return Return + default: + panic("unexpected leaf resume PC") + } +} + +func TestContextRunDirectAndIndirectCalls(t *testing.T) { + var ( + ctx Context + frame testRootFrame + ) + ctx.Push(&frame.Frame, &testRootDescriptor) + + if action := ctx.Run(); action != Suspend { + t.Fatalf("first Run action = %d, want Suspend", action) + } + if ctx.Top() != &frame.indirect.Frame { + t.Fatal("suspended child is not the active frame") + } + if frame.indirect.Parent != &frame.Frame { + t.Fatal("child frame is not linked to its caller") + } + + if action := ctx.Run(); action != Return { + t.Fatalf("second Run action = %d, want Return", action) + } + if ctx.Top() != nil { + t.Fatal("completed frame chain remains active") + } + if returned := ctx.TakeReturned(); returned != &frame.Frame { + t.Fatal("completed root frame was not returned to its owner") + } + if frame.value != 14 { + t.Fatalf("result = %d, want 14", frame.value) + } +} + +func TestContextRunEmpty(t *testing.T) { + var ctx Context + if action := ctx.Run(); action != Return { + t.Fatalf("empty Run action = %d, want Return", action) + } + if returned := ctx.TakeReturned(); returned != nil { + t.Fatalf("empty Run returned frame %p", returned) + } +} + +func TestSuspendCurrentRequiresCompilerLowering(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("SuspendCurrent fallback did not panic") + } + }() + SuspendCurrent() +} + +func TestContextStart(t *testing.T) { + descriptor := &Descriptor{} + root := &Frame{Descriptor: descriptor} + var context Context + context.Start(root) + if context.Top() != root { + t.Fatalf("Top() = %p, want %p", context.Top(), root) + } + for _, frame := range []*Frame{ + nil, + {Descriptor: descriptor}, + {Parent: root, Descriptor: descriptor}, + {}, + } { + func() { + defer func() { + if recover() == nil { + t.Fatalf("Start(%+v) did not panic", frame) + } + }() + context.Start(frame) + }() + } +} + +func TestContextPushInitializesHeader(t *testing.T) { + parent := Frame{} + child := Frame{Parent: &parent, Descriptor: &testMulDescriptor, PC: 9} + ctx := Context{top: &parent} + ctx.Push(&child, &testAddDescriptor) + if child.Parent != &parent { + t.Fatal("Push did not link the parent frame") + } + if child.Descriptor != &testAddDescriptor { + t.Fatal("Push did not set the descriptor") + } + if child.PC != 0 { + t.Fatalf("Push PC = %d, want 0", child.PC) + } +} + +func TestDescriptorCarriesFrameLayout(t *testing.T) { + descriptor := Descriptor{ + Resume: resumeTestAdd, + FrameSize: unsafe.Sizeof(testLeafFrame{}), + FrameAlign: unsafe.Alignof(testLeafFrame{}), + UnwindOffset: unsafe.Sizeof(Frame{}), + UnwindPC: 3, + } + if descriptor.Resume == nil || descriptor.FrameSize != unsafe.Sizeof(testLeafFrame{}) || + descriptor.FrameAlign != unsafe.Alignof(testLeafFrame{}) || + descriptor.UnwindOffset != unsafe.Sizeof(Frame{}) || descriptor.UnwindPC != 3 { + t.Fatalf("descriptor = %+v", descriptor) + } +} + +func TestContextPushClearsCompletedChain(t *testing.T) { + var ( + ctx Context + first Frame + next Frame + ) + ctx.Push(&first, &testAddDescriptor) + if action := ctx.Run(); action != Return { + t.Fatalf("first Run action = %d, want Return", action) + } + ctx.Push(&next, &testAddDescriptor) + if returned := ctx.TakeReturned(); returned != nil { + t.Fatalf("new root retained completed frame %p", returned) + } +} + +func TestContextRejectsInvalidAction(t *testing.T) { + descriptor := Descriptor{Resume: func(*Context, *Frame) Action { + return Action(255) + }} + var ( + ctx Context + frame Frame + ) + ctx.Push(&frame, &descriptor) + defer func() { + if recover() == nil { + t.Fatal("Run accepted an invalid resume action") + } + }() + ctx.Run() +} + +func BenchmarkContextDispatch(b *testing.B) { + descriptor := Descriptor{Resume: func(_ *Context, frame *Frame) Action { + if frame.PC == 0 { + frame.PC = 1 + return Continue + } + return Return + }} + var ( + ctx Context + frame Frame + ) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + ctx.Push(&frame, &descriptor) + ctx.Run() + } +} diff --git a/runtime/internal/wasmresume/storage.go b/runtime/internal/wasmresume/storage.go new file mode 100644 index 0000000000..04db9c870c --- /dev/null +++ b/runtime/internal/wasmresume/storage.go @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import "unsafe" + +const defaultFrameBlockSize = uintptr(2 << 10) + +type frameBlock struct { + prev *frameBlock + begin, end uintptr + stackPointer uintptr +} + +type frameStorage struct { + current *frameBlock +} + +func (s *frameStorage) allocate( + size, align uintptr, allocate Allocator, +) unsafe.Pointer { + if size == 0 || align == 0 || align&(align-1) != 0 || allocate == nil { + return nil + } + if frame, ok := allocateFromBlock(s.current, size, align); ok { + return frame + } + + payload, ok := addUintptr(size, unsafe.Sizeof(uintptr(0))) + if !ok { + return nil + } + payload, ok = addUintptr(payload, align-1) + if !ok { + return nil + } + if payload < defaultFrameBlockSize { + payload = defaultFrameBlockSize + } + total, ok := addUintptr(unsafe.Sizeof(frameBlock{}), payload) + if !ok { + return nil + } + raw := allocate(total) + if raw == nil { + return nil + } + block := (*frameBlock)(raw) + block.prev = s.current + block.begin, ok = addUintptr(uintptr(raw), unsafe.Sizeof(frameBlock{})) + if !ok { + panic("wasmresume: frame block address overflow") + } + block.end, ok = addUintptr(uintptr(raw), total) + if !ok { + panic("wasmresume: frame block address overflow") + } + block.stackPointer = block.begin + s.current = block + frame, ok := allocateFromBlock(block, size, align) + if !ok { + panic("wasmresume: new frame block is too small") + } + return frame +} + +func allocateFromBlock(block *frameBlock, size, align uintptr) (unsafe.Pointer, bool) { + if block == nil { + return nil, false + } + header, ok := addUintptr(block.stackPointer, unsafe.Sizeof(uintptr(0))) + if !ok { + return nil, false + } + frame, ok := alignUintptr(header, align) + if !ok { + return nil, false + } + next, ok := addUintptr(frame, size) + if !ok || next > block.end { + return nil, false + } + *(*uintptr)(unsafe.Pointer(frame - unsafe.Sizeof(uintptr(0)))) = block.stackPointer + block.stackPointer = next + return unsafe.Pointer(frame), true +} + +func (s *frameStorage) releaseFrame( + frame unsafe.Pointer, size uintptr, release Releaser, +) { + if s.current == nil || frame == nil || size == 0 { + panic("wasmresume: invalid frame release") + } + address := uintptr(frame) + end, ok := addUintptr(address, size) + if !ok { + panic("wasmresume: invalid frame release") + } + + block := s.current + for block != nil && (address < block.begin || end > block.stackPointer) { + block = block.prev + } + if block == nil { + panic("wasmresume: frame is not owned by this context") + } + previous := *(*uintptr)(unsafe.Pointer(address - unsafe.Sizeof(uintptr(0)))) + if previous < block.begin || previous >= address { + panic("wasmresume: invalid frame allocation header") + } + if block != s.current && release == nil { + panic("wasmresume: missing frame block reclaimer") + } + for s.current != block { + current := s.current + s.current = current.prev + release(unsafe.Pointer(current)) + } + block.stackPointer = previous + if previous == block.begin && block.prev != nil { + if release == nil { + panic("wasmresume: missing frame block reclaimer") + } + s.current = block.prev + release(unsafe.Pointer(block)) + } +} + +func (s *frameStorage) close(release Releaser) { + if s.current != nil && release == nil { + panic("wasmresume: missing frame block reclaimer") + } + for block := s.current; block != nil; { + previous := block.prev + release(unsafe.Pointer(block)) + block = previous + } + s.current = nil +} + +func addUintptr(left, right uintptr) (uintptr, bool) { + sum := left + right + return sum, sum >= left +} + +func alignUintptr(value, align uintptr) (uintptr, bool) { + next, ok := addUintptr(value, align-1) + if !ok { + return 0, false + } + return next &^ (align - 1), true +} diff --git a/runtime/internal/wasmresume/storage_test.go b/runtime/internal/wasmresume/storage_test.go new file mode 100644 index 0000000000..76cfd3a00d --- /dev/null +++ b/runtime/internal/wasmresume/storage_test.go @@ -0,0 +1,290 @@ +package wasmresume + +import ( + "testing" + "unsafe" +) + +type testFrameRoots struct { + blocks map[unsafe.Pointer][]byte + allocs int + frees int +} + +type testUnwindFrame struct { + Frame + deferFrame unsafe.Pointer +} + +func (r *testFrameRoots) allocate(size uintptr) unsafe.Pointer { + block := make([]byte, size) + if len(block) == 0 { + return nil + } + ptr := unsafe.Pointer(&block[0]) + if r.blocks == nil { + r.blocks = make(map[unsafe.Pointer][]byte) + } + r.blocks[ptr] = block + r.allocs++ + return ptr +} + +func (r *testFrameRoots) release(ptr unsafe.Pointer) { + if _, ok := r.blocks[ptr]; !ok { + panic("released unknown root block") + } + delete(r.blocks, ptr) + r.frees++ +} + +func TestFrameStorageAlignsAndReusesFrames(t *testing.T) { + var ( + storage frameStorage + roots testFrameRoots + ) + first := storage.allocate(31, 8, roots.allocate) + second := storage.allocate(64, 64, roots.allocate) + if first == nil || second == nil { + t.Fatal("frame allocation failed") + } + if uintptr(first)%8 != 0 || uintptr(second)%64 != 0 { + t.Fatalf("unaligned frames: first=%p second=%p", first, second) + } + if roots.allocs != 1 { + t.Fatalf("root block allocations = %d, want 1", roots.allocs) + } + + storage.releaseFrame(second, 64, roots.release) + reused := storage.allocate(64, 64, roots.allocate) + if reused != second { + t.Fatalf("frame was not reused: got %p, want %p", reused, second) + } + storage.releaseFrame(reused, 64, roots.release) + storage.releaseFrame(first, 31, roots.release) + storage.close(roots.release) + if roots.frees != 1 || len(roots.blocks) != 0 { + t.Fatalf("released roots = %d, remaining = %d", roots.frees, len(roots.blocks)) + } +} + +func TestFrameStorageAddsAndReleasesSegments(t *testing.T) { + var ( + storage frameStorage + roots testFrameRoots + ) + first := storage.allocate(defaultFrameBlockSize, 16, roots.allocate) + second := storage.allocate(128, 16, roots.allocate) + if first == nil || second == nil || roots.allocs != 2 { + t.Fatalf("allocations = %d, first=%p second=%p", roots.allocs, first, second) + } + storage.releaseFrame(second, 128, roots.release) + if roots.frees != 1 { + t.Fatalf("released child segments = %d, want 1", roots.frees) + } + storage.releaseFrame(first, defaultFrameBlockSize, roots.release) + storage.close(roots.release) + if roots.frees != 2 || len(roots.blocks) != 0 { + t.Fatalf("released roots = %d, remaining = %d", roots.frees, len(roots.blocks)) + } +} + +func TestContextOwnsGeneratedFrameStorage(t *testing.T) { + var ( + ctx Context + roots testFrameRoots + ) + size := unsafe.Sizeof(testLeafFrame{}) + raw := ctx.AllocateFrame(size, unsafe.Alignof(testLeafFrame{}), roots.allocate) + if raw == nil { + t.Fatal("Context.AllocateFrame failed") + } + frame := (*testLeafFrame)(raw) + frame.Descriptor = &Descriptor{FrameSize: size} + ctx.ReleaseFrame(&frame.Frame, roots.release) + ctx.Close(roots.release) + if roots.allocs != 1 || roots.frees != 1 { + t.Fatalf("root lifecycle = %d allocs, %d frees", roots.allocs, roots.frees) + } +} + +func TestContextReleaseFrameDiscardsDynamicStorage(t *testing.T) { + var ( + ctx Context + roots testFrameRoots + ) + const frameSize = uintptr(32) + raw := ctx.AllocateFrame(frameSize, 8, roots.allocate) + frame := (*Frame)(raw) + frame.Descriptor = &Descriptor{FrameSize: frameSize} + if ctx.AllocateFrame(64, 16, roots.allocate) == nil || + ctx.AllocateFrame(defaultFrameBlockSize, 16, roots.allocate) == nil { + t.Fatal("dynamic frame storage allocation failed") + } + + ctx.ReleaseFrame(frame, roots.release) + reused := ctx.AllocateFrame(frameSize, 8, roots.allocate) + if reused != raw { + t.Fatalf("frame storage was not rewound: got %p, want %p", reused, raw) + } + ctx.Close(roots.release) + if roots.allocs != roots.frees || len(roots.blocks) != 0 { + t.Fatalf("root lifecycle = %d allocs, %d frees, %d remaining", + roots.allocs, roots.frees, len(roots.blocks)) + } +} + +func TestContextUnwindReclaimsChildrenAndRedirectsOwner(t *testing.T) { + var ( + ctx Context + roots testFrameRoots + token byte + ) + size := unsafe.Sizeof(testUnwindFrame{}) + align := unsafe.Alignof(testUnwindFrame{}) + owner := (*testUnwindFrame)(ctx.AllocateFrame(size, align, roots.allocate)) + child := (*testUnwindFrame)(ctx.AllocateFrame(size, align, roots.allocate)) + ownerDescriptor := &Descriptor{ + FrameSize: size, + UnwindOffset: unsafe.Offsetof(testUnwindFrame{}.deferFrame), + UnwindPC: 7, + } + childDescriptor := &Descriptor{FrameSize: size} + owner.Descriptor = ownerDescriptor + owner.deferFrame = unsafe.Pointer(&token) + child.Parent = &owner.Frame + child.Descriptor = childDescriptor + ctx.top = &child.Frame + + if !ctx.Unwind(unsafe.Pointer(&token), roots.release) { + t.Fatal("Context.Unwind did not find the defer owner") + } + if ctx.top != &owner.Frame || owner.PC != ownerDescriptor.UnwindPC { + t.Fatalf("unwind result: top=%p PC=%d", ctx.top, owner.PC) + } + reused := ctx.AllocateFrame(size, align, roots.allocate) + if reused != unsafe.Pointer(child) { + t.Fatalf("discarded child storage was not reused: got %p, want %p", reused, child) + } + ctx.Close(roots.release) +} + +func TestContextUnwindRejectsMissingOwner(t *testing.T) { + var ctx Context + if ctx.Unwind(nil, nil) || ctx.Unwind(unsafe.Pointer(new(byte)), nil) { + t.Fatal("Context.Unwind accepted a missing defer owner") + } +} + +func TestContextUnwindIgnoresIncompleteDescriptor(t *testing.T) { + var ( + ctx Context + token byte + frame testUnwindFrame + ) + frame.deferFrame = unsafe.Pointer(&token) + frame.Descriptor = &Descriptor{ + FrameSize: unsafe.Sizeof(frame), + UnwindOffset: unsafe.Offsetof(frame.deferFrame), + } + ctx.top = &frame.Frame + if ctx.Unwind(unsafe.Pointer(&token), nil) { + t.Fatal("Context.Unwind accepted a descriptor without an unwind PC") + } +} + +func TestContextKeepsGeneratedABIPrefix(t *testing.T) { + if got, want := unsafe.Offsetof(Context{}.storage), 2*unsafe.Sizeof(uintptr(0)); got != want { + t.Fatalf("Context storage offset = %d, want %d", got, want) + } +} + +func TestFrameStorageRejectsInvalidOperations(t *testing.T) { + var ( + storage frameStorage + roots testFrameRoots + ) + if storage.allocate(0, 8, roots.allocate) != nil || + storage.allocate(8, 3, roots.allocate) != nil || + storage.allocate(^uintptr(0), 8, roots.allocate) != nil || + storage.allocate(8, 8, nil) != nil { + t.Fatal("invalid allocation was accepted") + } + if storage.allocate(8, 8, func(uintptr) unsafe.Pointer { return nil }) != nil { + t.Fatal("failed root allocation returned a frame") + } + + storage.close(roots.release) +} + +func TestFrameStorageRejectsInvalidReleaseState(t *testing.T) { + assertPanic := func(name string, operation func()) { + t.Helper() + t.Run(name, func(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("operation did not panic") + } + }() + operation() + }) + } + + assertPanic("empty", func() { + var storage frameStorage + storage.releaseFrame(unsafe.Pointer(new(byte)), 1, nil) + }) + + var ( + storage frameStorage + roots testFrameRoots + ) + frame := storage.allocate(8, 8, roots.allocate) + assertPanic("nil frame", func() { + storage.releaseFrame(nil, 8, roots.release) + }) + assertPanic("zero size", func() { + storage.releaseFrame(frame, 0, roots.release) + }) + assertPanic("foreign frame", func() { + storage.releaseFrame(unsafe.Pointer(new(byte)), 1, roots.release) + }) + assertPanic("invalid header", func() { + header := unsafe.Pointer(uintptr(frame) - unsafe.Sizeof(uintptr(0))) + *(*uintptr)(header) = 0 + storage.releaseFrame(frame, 8, roots.release) + }) + storage.close(roots.release) + + var noRelease frameStorage + noRelease.allocate(8, 8, roots.allocate) + assertPanic("close without reclaimer", func() { + noRelease.close(nil) + }) + noRelease.close(roots.release) +} + +func TestContextRejectsFrameWithoutDescriptor(t *testing.T) { + var ctx Context + defer func() { + if recover() == nil { + t.Fatal("ReleaseFrame accepted an untyped frame") + } + }() + ctx.ReleaseFrame(&Frame{}, nil) +} + +func BenchmarkFrameStorageHotAllocateRelease(b *testing.B) { + var ( + storage frameStorage + roots testFrameRoots + ) + frame := storage.allocate(64, 16, roots.allocate) + storage.releaseFrame(frame, 64, roots.release) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + frame = storage.allocate(64, 16, roots.allocate) + storage.releaseFrame(frame, 64, roots.release) + } +} diff --git a/ssa/abitype.go b/ssa/abitype.go index 8d5e728c59..bec470ce16 100644 --- a/ssa/abitype.go +++ b/ssa/abitype.go @@ -511,6 +511,8 @@ func (b Builder) abiUncommonMethods(t types.Type, methods []*types.Selection) ll pSig := types.NewSignature(pRecv, mSig.Params(), mSig.Results(), mSig.Variadic()) ifn = b.abiMethodFunc(anonymous, pkg, mName, pSig).impl } + ifn = b.Pkg.wasmResumeStart(ifn) + tfn = b.Pkg.wasmResumeStart(tfn) var values []llvm.Value values = append(values, name) ftyp := funcType(prog, m.Type()) diff --git a/ssa/closure_wrap.go b/ssa/closure_wrap.go index 470a299caf..de6246a578 100644 --- a/ssa/closure_wrap.go +++ b/ssa/closure_wrap.go @@ -70,13 +70,24 @@ func closureWrapReturn(b Builder, sig *types.Signature, ret Expr) { // closureWrapDecl wraps a function declaration that lacks __llgo_ctx. // It directly calls the target symbol and ignores the ctx parameter. func (p Package) closureWrapDecl(fn Expr, sig *types.Signature) Function { + return p.closureWrapDeclFor(fn, sig, p.Prog.WasmResumeABIEnabled()) +} + +func (p Package) closureWrapDeclFor(fn Expr, sig *types.Signature, resumable bool) Function { name := closureStub + fn.impl.Name() + if p.Prog.WasmResumeABIEnabled() && !resumable { + name = closureStub + "sync." + fn.impl.Name() + } if wrap := p.FuncOf(name); wrap != nil { return wrap } ctx := types.NewParam(token.NoPos, nil, closureCtx, types.Typ[types.UnsafePointer]) sigCtx := FuncAddCtx(ctx, sig) - wrap := p.NewFunc(name, sigCtx, InC) + background := InC + if resumable { + background = InGo + } + wrap := p.NewFunc(name, sigCtx, background) wrap.impl.SetLinkage(llvm.LinkOnceAnyLinkage) b := wrap.MakeBody(1) args := closureWrapArgs(wrap) diff --git a/ssa/decl.go b/ssa/decl.go index a575dd1bee..7ea6f01efb 100644 --- a/ssa/decl.go +++ b/ssa/decl.go @@ -246,10 +246,11 @@ type aFunction struct { nextDeferID uintptr recov BasicBlock - params []Type - freeVars Expr - base int // base = 1 if hasFreeVars; base = 0 otherwise - hasVArg bool + params []Type + freeVars Expr + base int // base = 1 if hasFreeVars; base = 0 otherwise + hasVArg bool + background Background fakeUses []llvm.Value fakeUseSet map[llvm.Value]struct{} @@ -275,6 +276,7 @@ func (p Package) NewFuncEx(name string, sig *types.Signature, bg Background, has fn := llvm.AddFunction(p.mod, name, t.ll) if bg == InGo { fn.AddFunctionAttr(p.nullPointerIsValidAttr) + p.Prog.markWasmResumeFunction(fn) // Keep frame pointers so the runtime can walk real stacks (FP chain) // for Callers/panic tracebacks instead of shadow-stack bookkeeping. // Only where that unwinder exists: on embedded targets the attribute @@ -290,7 +292,7 @@ func (p Package) NewFuncEx(name string, sig *types.Signature, bg Background, has if p.isPreservedName(name) { p.markLLVMUsed(fn) } - ret := newFunction(fn, t, p, p.Prog, hasFreeVars) + ret := newFunction(fn, t, p, p.Prog, bg, hasFreeVars) p.fns[name] = ret return ret } @@ -300,7 +302,7 @@ func (p Package) FuncOf(name string) Function { return p.fns[name] } -func newFunction(fn llvm.Value, t Type, pkg Package, prog Program, hasFreeVars bool) Function { +func newFunction(fn llvm.Value, t Type, pkg Package, prog Program, bg Background, hasFreeVars bool) Function { params, hasVArg := newParams(t, prog) base := 0 if hasFreeVars { @@ -313,6 +315,7 @@ func newFunction(fn llvm.Value, t Type, pkg Package, prog Program, hasFreeVars b params: params, base: base, hasVArg: hasVArg, + background: bg, fakeUses: make([]llvm.Value, 0, 4), fakeUseSet: make(map[llvm.Value]struct{}), } diff --git a/ssa/eh.go b/ssa/eh.go index 989371c75c..62965faf91 100644 --- a/ssa/eh.go +++ b/ssa/eh.go @@ -202,14 +202,14 @@ func (b Builder) getDefer(kind DoAction) *aDefer { blks := self.MakeBlocks(2) procBlk, rethrowBlk := blks[0], blks[1] - deferState, link, retval := b.initDeferState(procBlk, rethrowBlk) - czero := b.Prog.IntVal(0, b.Prog.CInt()) if kind != DeferAlways { panicBlk = self.MakeBlock() } else { blks = self.MakeBlocks(2) next, panicBlk = blks[0], blks[1] } + deferState, link, retval := b.initDeferState(procBlk, rethrowBlk, panicBlk) + czero := b.Prog.IntVal(0, b.Prog.CInt()) b.If(b.BinOp(token.EQL, retval, czero), next, panicBlk) deferState.panicBlk = panicBlk @@ -244,7 +244,7 @@ func (b Builder) getDeferInCurrentBlock() *aDefer { logicalBlk := b.blk blks := self.MakeBlocks(4) procBlk, rethrowBlk, next, panicBlk := blks[0], blks[1], blks[2], blks[3] - deferState, link, retval := b.initDeferState(procBlk, rethrowBlk) + deferState, link, retval := b.initDeferState(procBlk, rethrowBlk, panicBlk) czero := b.Prog.IntVal(0, b.Prog.CInt()) b.If(b.BinOp(token.EQL, retval, czero), next, panicBlk) deferState.panicBlk = panicBlk @@ -261,15 +261,21 @@ func (b Builder) getDeferInCurrentBlock() *aDefer { return self.defer_ } -func (b Builder) initDeferState(procBlk, rethrowBlk BasicBlock) (*aDefer, Expr, Expr) { +func (b Builder) initDeferState( + procBlk, rethrowBlk, panicBlk BasicBlock, +) (*aDefer, Expr, Expr) { self := b.Func prog := b.Prog zero := prog.Val(uintptr(0)) link := b.Call(b.Pkg.rtFunc("GetThreadDefer")) - jb := b.AllocaSigjmpBuf() + jb := prog.Nil(prog.VoidPtr()) + if !b.wasmResumeFunctionEnabled() { + jb = b.AllocaSigjmpBuf() + } ptr := b.aggregateAllocU(prog.Defer(), jb.impl, zero.impl, link.impl, procBlk.Addr().impl) deferData := Expr{ptr, prog.DeferPtr()} b.Call(b.Pkg.rtFunc("SetThreadDefer"), deferData) + b.registerWasmResumeUnwind(deferData, panicBlk) bitsPtr := b.FieldAddr(deferData, deferBits) rethPtr := b.FieldAddr(deferData, deferRethrow) rundPtr := b.FieldAddr(deferData, deferRunDefers) @@ -279,7 +285,10 @@ func (b Builder) initDeferState(procBlk, rethrowBlk BasicBlock) (*aDefer, Expr, b.Store(argsPtr, prog.Nil(prog.VoidPtr())) czero := prog.IntVal(0, prog.CInt()) - retval := b.Sigsetjmp(jb, czero) + retval := czero + if !b.wasmResumeFunctionEnabled() { + retval = b.Sigsetjmp(jb, czero) + } self.defer_ = &aDefer{ data: deferData, @@ -619,6 +628,7 @@ func (p Function) endDefer(b Builder) { } link := b.getField(b.Load(self.data), deferLink) b.Call(b.Pkg.rtFunc("SetThreadDefer"), link) + b.clearWasmResumeUnwind() b.jumpRunDefersTarget(rundPtr, nexts) b.SetBlockEx(panicBlk, AtEnd, false) // panicBlk: exec runDefers and rethrow diff --git a/ssa/expr.go b/ssa/expr.go index 6f476c9eac..c906346469 100644 --- a/ssa/expr.go +++ b/ssa/expr.go @@ -1202,7 +1202,15 @@ func (b Builder) MakeClosure(fn Expr, bindings []Expr) Expr { ptr := b.aggregateAllocU(prog.rawType(tctx), llvmFields(bindings, tctx, b)...) data = ptr } - return b.aggregateValue(prog.Closure(removeCtx(sig)), fn.impl, data) + code := fn.impl + resumable := b.wasmResumeFunctionEnabled() + if prog.WasmResumeABIEnabled() && closureCtxParam(sig) == nil { + code = b.Pkg.closureWrapDeclFor(fn, sig, resumable).impl + } + if resumable { + code = b.Pkg.wasmResumeStart(code) + } + return b.aggregateValue(prog.Closure(removeCtx(sig)), code, data) } // ----------------------------------------------------------------------------- @@ -1245,6 +1253,7 @@ func (b Builder) Call(fn Expr, args ...Expr) (ret Expr) { } ll = b.Prog.FuncDecl(sigCtx, InC).ll ret.impl = llvm.CreateCall(b.impl, ll, fn.impl, llvmParamsEx(data, args, sigCtx.Params(), b)) + b.markWasmResumeCall(ret.impl, InGo) return ret case vkFuncPtr: sig = raw.Underlying().(*types.Signature) @@ -1264,6 +1273,7 @@ func (b Builder) Call(fn Expr, args ...Expr) (ret Expr) { } ret.Type = b.Prog.retType(sig) ret.impl = llvm.CreateCall(b.impl, ll, fn.impl, llvmParamsEx(data, args, sig.Params(), b)) + b.markWasmResumeCall(ret.impl, b.directCallBackground(fn)) if reflectCheck.Kind&ReflectMethodByName != 0 && reflectCheck.Name == "" { nameArgIndex := len(args) - 1 if !data.IsNil() { @@ -1744,6 +1754,9 @@ func checkExpr(v Expr, t types.Type, b Builder) Expr { v, data = b.Pkg.closureStub(b, v, sig, origKind) } } + if origKind == vkFuncDecl && b.wasmResumeFunctionEnabled() { + v.impl = b.Pkg.wasmResumeStart(v.impl) + } return b.aggregateValue(tclosure, v.impl, data.impl) } if types.Identical(v.raw.Type, t) || !types.AssignableTo(v.raw.Type, t) { diff --git a/ssa/goroutine.go b/ssa/goroutine.go index 398a5ee376..029dc3d94e 100644 --- a/ssa/goroutine.go +++ b/ssa/goroutine.go @@ -54,11 +54,17 @@ func (b Builder) Go(fn Expr, buildCall func(Builder, Expr, ...Expr) Expr, args . if fn != Nil && fn.kind != vkBuiltin { offset = 1 } + resumableDirectCall := offset == 1 && + b.wasmResumeFunctionEnabled() && + b.directCallBackground(fn) == InGo typs := make([]Type, len(args)+offset) flds := make([]llvm.Value, len(args)+offset) if offset == 1 { typs[0] = fn.Type flds[0] = fn.impl + if resumableDirectCall { + flds[0] = pkg.wasmResumeStart(flds[0]) + } } for i, arg := range args { typs[i+offset] = arg.Type @@ -73,7 +79,12 @@ func (b Builder) Go(fn Expr, buildCall func(Builder, Expr, ...Expr) Expr, args . aggregateInit(b.impl, dataPtr, t.ll, flds...) data := Expr{dataPtr, voidPtr} stackSize := prog.IntVal(prog.pthreadStackSize, prog.Uintptr()) - b.Call(pkg.rtFunc("NewProc"), pkg.routine(t, fn, buildCall, len(args)), data, stackSize) + b.Call( + pkg.rtFunc("NewProc"), + pkg.routine(t, fn, buildCall, len(args), resumableDirectCall), + data, + stackSize, + ) } func (p Package) routineName() string { @@ -81,9 +92,19 @@ func (p Package) routineName() string { return p.Path() + "._llgo_routine$" + strconv.Itoa(p.iRoutine) } -func (p Package) routine(t Type, fn Expr, buildCall func(Builder, Expr, ...Expr) Expr, n int) Expr { +func (p Package) routine( + t Type, + fn Expr, + buildCall func(Builder, Expr, ...Expr) Expr, + n int, + resumableDirectCall bool, +) Expr { prog := p.Prog - routine := p.NewFunc(p.routineName(), prog.tyRoutine(), InC) + background := InC + if prog.WasmResumeABIEnabled() { + background = InGo + } + routine := p.NewFunc(p.routineName(), prog.tyRoutine(), background) b := routine.MakeBody(1) var localCtx, previousLocalCtx Expr hasLocalContext := prog.NeedsLocalContext() @@ -102,7 +123,10 @@ func (p Package) routine(t Type, fn Expr, buildCall func(Builder, Expr, ...Expr) args[i] = b.getField(data, i+offset) } b.Call(p.rtFunc("FreeRoot"), param) - buildCall(b, fn, args...) + call := buildCall(b, fn, args...) + if resumableDirectCall && !call.impl.IsNil() { + b.markWasmResumeCall(call.impl, InGo) + } lastInst := b.impl.GetInsertBlock().LastInstruction() if lastInst.IsNil() || lastInst.IsAUnreachableInst().IsNil() { if hasLocalContext { @@ -110,5 +134,9 @@ func (p Package) routine(t Type, fn Expr, buildCall func(Builder, Expr, ...Expr) } b.Return(prog.Nil(prog.VoidPtr())) } - return routine.Expr + ret := routine.Expr + if prog.WasmResumeABIEnabled() { + ret.impl = p.wasmResumeStart(ret.impl) + } + return ret } diff --git a/ssa/package.go b/ssa/package.go index db41bae4ab..4c56b0fdce 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -243,6 +243,7 @@ type aProgram struct { enableFuncInfoMetadata bool enableFuncInfoSites bool + enableWasmResumeABI bool debugInfoOptimized bool } @@ -920,7 +921,7 @@ func (p Package) closureStub(b Builder, fn Expr, sig *types.Signature, origKind prog := b.Prog switch origKind { case vkFuncDecl: - wrap := p.closureWrapDecl(fn, sig) + wrap := p.closureWrapDeclFor(fn, sig, b.wasmResumeFunctionEnabled()) return wrap.Expr, prog.Nil(prog.VoidPtr()) case vkFuncPtr: wrap := p.closureWrapPtr(sig) diff --git a/ssa/wasm_resume.go b/ssa/wasm_resume.go new file mode 100644 index 0000000000..52b5d84eae --- /dev/null +++ b/ssa/wasm_resume.go @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import ( + "github.com/goplus/llgo/internal/wasmresume" + "github.com/xgo-dev/llvm" +) + +// EnableWasmResumeABI controls emission of the function and call inventory +// consumed by the experimental WebAssembly resumable ABI lowering. +func (p Program) EnableWasmResumeABI(enable bool) { + p.enableWasmResumeABI = enable +} + +// WasmResumeABIEnabled reports whether resumable ABI lowering is enabled for a +// WebAssembly target. +func (p Program) WasmResumeABIEnabled() bool { + return p.enableWasmResumeABI && p.target != nil && p.target.GOARCH == "wasm" +} + +func (p Program) markWasmResumeFunction(fn llvm.Value) { + if !p.WasmResumeABIEnabled() || + wasmresume.IsRuntimeABIImplementation(fn.Name()) || + wasmresume.IsNonSuspendingBoundary(fn.Name()) { + return + } + fn.AddFunctionAttr(p.ctx.CreateStringAttribute(wasmresume.FunctionAttribute, "1")) +} + +func (p Package) wasmResumeStart(fn llvm.Value) llvm.Value { + if !p.Prog.WasmResumeABIEnabled() || + wasmresume.IsRuntimeABIImplementation(fn.Name()) || + wasmresume.IsNonSuspendingBoundary(fn.Name()) { + return fn + } + name := wasmresume.StartSymbol(fn.Name()) + if start := p.mod.NamedFunction(name); !start.IsNil() { + return start + } + fnType := fn.GlobalValueType() + params := append([]llvm.Type{p.Prog.tyVoidPtr()}, fnType.ParamTypes()...) + startType := llvm.FunctionType(p.Prog.tyVoidPtr(), params, false) + return llvm.AddFunction(p.mod, name, startType) +} + +func (b Builder) markWasmResumeCall(call llvm.Value, background Background) { + if background != InGo || !b.wasmResumeFunctionEnabled() { + return + } + callee := call.CalledValue() + if !callee.IsAFunction().IsNil() && + wasmresume.IsNonSuspendingBoundary(callee.Name()) { + return + } + kind := b.Prog.ctx.MDKindID(wasmresume.CallMetadata) + version := llvm.ConstInt(b.Prog.Int32().ll, wasmresume.MarkerVersion, false).ConstantAsMetadata() + call.SetMetadata(kind, b.Prog.ctx.MDNode([]llvm.Metadata{version})) +} + +func (b Builder) wasmResumeFunctionEnabled() bool { + return b != nil && b.Prog.WasmResumeABIEnabled() && + b.Func != nil && b.Func.background == InGo && + !wasmresume.IsRuntimeABIImplementation(b.Func.Name()) && + !wasmresume.IsNonSuspendingBoundary(b.Func.Name()) +} + +func (b Builder) registerWasmResumeUnwind(frame Expr, handler BasicBlock) { + if !b.wasmResumeFunctionEnabled() { + return + } + ctx := b.Prog.ctx + typ := llvm.FunctionType(ctx.VoidType(), []llvm.Type{ + b.Prog.tyVoidPtr(), + b.Prog.tyVoidPtr(), + }, false) + fn := b.Pkg.mod.NamedFunction(wasmresume.RegisterUnwindSymbol) + if fn.IsNil() { + fn = llvm.AddFunction(b.Pkg.mod, wasmresume.RegisterUnwindSymbol, typ) + } + llvm.CreateCall(b.impl, typ, fn, []llvm.Value{frame.impl, handler.Addr().impl}) +} + +func (b Builder) clearWasmResumeUnwind() { + if !b.wasmResumeFunctionEnabled() { + return + } + ctx := b.Prog.ctx + typ := llvm.FunctionType(ctx.VoidType(), nil, false) + fn := b.Pkg.mod.NamedFunction(wasmresume.ClearUnwindSymbol) + if fn.IsNil() { + fn = llvm.AddFunction(b.Pkg.mod, wasmresume.ClearUnwindSymbol, typ) + } + llvm.CreateCall(b.impl, typ, fn, nil) +} + +func (b Builder) directCallBackground(fn Expr) Background { + if fn.impl.IsNil() || fn.impl.IsAFunction().IsNil() { + return inUnknown + } + if decl := b.Pkg.FuncOf(fn.impl.Name()); decl != nil { + return decl.background + } + return inUnknown +} diff --git a/ssa/wasm_resume_test.go b/ssa/wasm_resume_test.go new file mode 100644 index 0000000000..58d163b002 --- /dev/null +++ b/ssa/wasm_resume_test.go @@ -0,0 +1,407 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import ( + "go/importer" + "go/token" + "go/types" + "regexp" + "strings" + "testing" + + "github.com/goplus/llgo/internal/wasmresume" + "github.com/xgo-dev/llvm" +) + +func TestWasmResumeABIInventoriesGoCalls(t *testing.T) { + prog := NewProgram(&Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + prog.EnableWasmResumeABI(true) + + pkg := prog.NewPackage("p", "example.com/p") + goFn := pkg.NewFunc("goFn", NoArgsNoRet, InGo) + gb := goFn.MakeBody(1) + gb.Return() + + cFn := pkg.NewFunc("cFn", NoArgsNoRet, InC) + caller := pkg.NewFunc("caller", NoArgsNoRet, InGo) + b := caller.MakeBody(1) + b.Call(goFn.Expr) + b.Call(b.MakeClosure(goFn.Expr, nil)) + b.Call(cFn.Expr) + b.Return() + + ir := pkg.String() + if got := strings.Count(ir, "!"+wasmresume.CallMetadata); got != 3 { + t.Fatalf("resumable call marker count = %d, want 3:\n%s", got, ir) + } + if !strings.Contains(ir, `"`+wasmresume.FunctionAttribute+`"="1"`) { + t.Fatalf("Go functions are not marked for resumable lowering:\n%s", ir) + } + if !strings.Contains(ir, "@"+wasmresume.StartSymbol("__llgo_stub.goFn")) { + t.Fatalf("closure does not reference its resumable start entry:\n%s", ir) + } + var foundCCall bool + for _, line := range strings.Split(ir, "\n") { + if strings.Contains(line, "call void @cFn") && strings.Contains(line, wasmresume.CallMetadata) { + t.Fatalf("C call was marked resumable: %s", line) + } + if strings.Contains(line, "call void @cFn") { + foundCCall = true + } + } + if !foundCCall { + t.Fatalf("C call is missing from test IR:\n%s", ir) + } + if got := b.directCallBackground(Builtin("len")); got != inUnknown { + t.Fatalf("builtin call background = %d, want unknown", got) + } + delete(pkg.fns, cFn.Name()) + if got := b.directCallBackground(cFn.Expr); got != inUnknown { + t.Fatalf("untracked declaration background = %d, want unknown", got) + } +} + +func TestWasmResumeABIDoesNotChangeDefaultOrNativeIR(t *testing.T) { + tests := []struct { + name string + target *Target + enable bool + }{ + {name: "wasm disabled", target: &Target{GOOS: "wasip1", GOARCH: "wasm"}}, + {name: "native enabled", target: &Target{GOOS: "darwin", GOARCH: "arm64"}, enable: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + prog := NewProgram(test.target) + defer prog.Dispose() + prog.EnableWasmResumeABI(test.enable) + pkg := prog.NewPackage("p", "example.com/p") + callee := pkg.NewFunc("callee", NoArgsNoRet, InGo) + caller := pkg.NewFunc("caller", NoArgsNoRet, InGo) + b := caller.MakeBody(1) + b.Call(callee.Expr) + b.Call(b.MakeClosure(callee.Expr, nil)) + b.Return() + + ir := pkg.String() + if strings.Contains(ir, wasmresume.FunctionAttribute) || + strings.Contains(ir, wasmresume.CallMetadata) || + strings.Contains(ir, wasmresume.StartSymbol("")) { + t.Fatalf("inactive resumable ABI changed IR:\n%s", ir) + } + }) + } +} + +func TestWasmResumeABIClosureWithContextUsesStartEntry(t *testing.T) { + prog := NewProgram(&Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + prog.EnableWasmResumeABI(true) + prog.TypeSizes(types.SizesFor("gc", "wasm")) + prog.SetRuntime(func() *types.Package { + pkg, err := importer.For("source", nil).Import(PkgRuntime) + if err != nil { + t.Fatal(err) + } + return pkg + }) + + pkg := prog.NewPackage("p", "example.com/p") + fields := []*types.Var{ + types.NewField(token.NoPos, nil, "value", types.Typ[types.Int], false), + } + ctxType := types.NewStruct(fields, nil) + ctx := types.NewParam(token.NoPos, nil, closureCtx, types.NewPointer(ctxType)) + sig := types.NewSignatureType( + nil, nil, nil, types.NewTuple(ctx), nil, false, + ) + inner := pkg.NewFunc("inner", sig, InGo) + inner.MakeBody(1).Return() + + outer := pkg.NewFunc("outer", NoArgsNoRet, InGo) + b := outer.MakeBody(1) + b.Call(b.MakeClosure(inner.Expr, []Expr{prog.Val(42)})) + b.Return() + + ir := pkg.String() + if !strings.Contains(ir, "@"+wasmresume.StartSymbol("inner")) { + t.Fatalf("capturing closure does not reference its start entry:\n%s", ir) + } + if strings.Contains(ir, wasmresume.StartSymbol(closureStub+"inner")) { + t.Fatalf("capturing closure was wrapped unnecessarily:\n%s", ir) + } +} + +func TestWasmResumeABIMethodMetadataUsesStartEntries(t *testing.T) { + prog := NewProgram(&Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + prog.EnableWasmResumeABI(true) + prog.TypeSizes(types.SizesFor("gc", "wasm")) + prog.SetRuntime(func() *types.Package { + pkg, err := importer.For("source", nil).Import(PkgRuntime) + if err != nil { + t.Fatal(err) + } + return pkg + }) + + pkg := prog.NewPackage("p", "example.com/p") + goPkg := types.NewPackage("example.com/p", "p") + named := types.NewNamed( + types.NewTypeName(token.NoPos, goPkg, "S", nil), + types.NewStruct(nil, nil), + nil, + ) + recv := types.NewVar(token.NoPos, goPkg, "", named) + method := types.NewFunc( + token.NoPos, + goPkg, + "M", + types.NewSignatureType(recv, nil, nil, nil, nil, false), + ) + named.AddMethod(method) + + use := pkg.NewFunc("use", NoArgsNoRet, InGo) + b := use.MakeBody(1) + b.abiType(named) + b.Return() + + ir := pkg.String() + for _, want := range []string{ + wasmresume.StartSymbol("example.com/p.(*S).M"), + wasmresume.StartSymbol(closureStub + "example.com/p.S.M"), + } { + if !strings.Contains(ir, want) { + t.Fatalf("method metadata does not reference %s:\n%s", want, ir) + } + } +} + +func TestWasmResumeABILowersSuspendCurrent(t *testing.T) { + prog := NewProgram(&Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + prog.EnableWasmResumeABI(true) + + pkg := prog.NewPackage("p", "example.com/p") + suspend := pkg.NewFunc(wasmresume.SuspendSymbol, NoArgsNoRet, InGo) + caller := pkg.NewFunc("caller", NoArgsNoRet, InGo) + b := caller.MakeBody(1) + b.Call(suspend.Expr) + b.Return() + + if err := wasmresume.Lower(pkg.Module(), prog.TargetData()); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify lowered suspend module: %v\n%s", err, pkg.String()) + } + ir := pkg.String() + if strings.Contains(ir, "call void @"+wasmresume.SuspendSymbol) { + t.Fatalf("SuspendCurrent call remains after lowering:\n%s", ir) + } + for _, want := range []string{ + "ret i8 2", + "i32 1, label %resume.1", + } { + if !strings.Contains(ir, want) { + t.Fatalf("lowered suspend module is missing %q:\n%s", want, ir) + } + } +} + +func TestWasmResumeABILowersDeferUnwindState(t *testing.T) { + prog := NewProgram(&Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + prog.EnableWasmResumeABI(true) + prog.TypeSizes(types.SizesFor("gc", "wasm")) + prog.SetRuntime(func() *types.Package { + pkg, err := importer.For("source", nil).Import(PkgRuntime) + if err != nil { + t.Fatal(err) + } + return pkg + }) + + pkg := prog.NewPackage("p", "example.com/p") + deferred := pkg.NewFunc("deferred", NoArgsNoRet, InGo) + deferred.MakeBody(1).Return() + suspend := pkg.NewFunc(wasmresume.SuspendSymbol, NoArgsNoRet, InGo) + fn := pkg.NewFunc("withDefer", NoArgsNoRet, InGo) + b := fn.MakeBody(1) + recoverBlock := fn.MakeBlock() + fn.SetRecover(recoverBlock) + b.SetBlockEx(recoverBlock, AtEnd, true) + b.Return() + b.SetBlockEx(fn.Block(0), AtEnd, true) + b.Defer(DeferAlways, deferred.Expr, Builder.Call) + b.Call(suspend.Expr) + b.RunDefers() + b.Return() + b.EndBuild() + + before := pkg.String() + if strings.Contains(before, "setjmp") { + t.Fatalf("resumable defer allocated a native jump buffer:\n%s", before) + } + for _, marker := range []string{ + wasmresume.RegisterUnwindSymbol, + wasmresume.ClearUnwindSymbol, + } { + if !strings.Contains(before, marker) { + t.Fatalf("resumable defer is missing %s:\n%s", marker, before) + } + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify defer module before lowering: %v\n%s", err, before) + } + + if err := wasmresume.Lower(pkg.Module(), prog.TargetData()); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify lowered defer module: %v\n%s", err, pkg.String()) + } + ir := pkg.String() + for _, marker := range []string{ + wasmresume.RegisterUnwindSymbol, + wasmresume.ClearUnwindSymbol, + } { + if strings.Contains(ir, "call void @"+marker) { + t.Fatalf("unwind marker %s remains after lowering:\n%s", marker, ir) + } + } + descriptor := regexp.MustCompile( + `@__llgo_wasm_resume_desc\.withDefer = constant \{ ptr, i32, i32, i32, i32 \} ` + + `\{ ptr @__llgo_wasm_resume\.withDefer, i32 [1-9][0-9]*, i32 [1-9][0-9]*, ` + + `i32 [1-9][0-9]*, i32 [1-9][0-9]* \}`, + ) + if !descriptor.MatchString(ir) { + t.Fatalf("resumable defer descriptor has no unwind state:\n%s", ir) + } +} + +func TestWasmResumeABIGoroutineUsesResumableTarget(t *testing.T) { + prog := NewProgram(&Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + prog.EnableWasmResumeABI(true) + prog.TypeSizes(types.SizesFor("gc", "wasm")) + prog.SetRuntime(func() *types.Package { + pkg, err := importer.For("source", nil).Import(PkgRuntime) + if err != nil { + t.Fatal(err) + } + return pkg + }) + + pkg := prog.NewPackage("p", "example.com/p") + worker := pkg.NewFunc("worker", NoArgsNoRet, InGo) + worker.MakeBody(1).Return() + caller := pkg.NewFunc("caller", NoArgsNoRet, InGo) + b := caller.MakeBody(1) + workerPointer := worker.Expr + workerPointer.kind = vkFuncPtr + b.Go(workerPointer, func(b Builder, fn Expr, args ...Expr) Expr { + return b.Call(fn, args...) + }) + b.Return() + + ir := pkg.String() + if !strings.Contains(ir, "store ptr @"+wasmresume.StartSymbol("worker")) { + t.Fatalf("goroutine startup record does not contain the worker start entry:\n%s", ir) + } + routine := pkg.FuncOf("example.com/p._llgo_routine$1") + if routine == nil { + t.Fatalf("goroutine wrapper is missing:\n%s", ir) + } + kind := prog.ctx.MDKindID(wasmresume.CallMetadata) + var marked bool + for block := routine.impl.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.HasMetadata() && !instruction.Metadata(kind).IsNil() { + marked = true + } + } + } + if !marked { + t.Fatalf("goroutine wrapper call is not resumable:\n%s", ir) + } +} + +func TestWasmResumeABIKeepsRuntimeBoundariesSynchronous(t *testing.T) { + prog := NewProgram(&Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + prog.EnableWasmResumeABI(true) + + pkg := prog.NewPackage("p", "example.com/p") + ordinary := pkg.NewFunc("ordinary", NoArgsNoRet, InGo) + ordinary.MakeBody(1).Return() + + boundary := pkg.NewFunc( + "github.com/goplus/llgo/runtime/internal/wasmresume.Context.Run", + NoArgsNoRet, + InGo, + ) + b := boundary.MakeBody(1) + b.Call(ordinary.Expr) + b.Call(b.MakeClosure(ordinary.Expr, nil)) + b.Return() + + root := pkg.NewFunc("root", NoArgsNoRet, InC) + b = root.MakeBody(1) + b.Call(ordinary.Expr) + b.Return() + + caller := pkg.NewFunc("caller", NoArgsNoRet, InGo) + b = caller.MakeBody(1) + b.Call(boundary.Expr) + b.Return() + + for _, function := range []Function{boundary, root} { + for _, attr := range function.impl.GetFunctionAttributes() { + if attr.IsString() && attr.GetStringKind() == wasmresume.FunctionAttribute { + t.Fatalf("%s was marked resumable:\n%s", function.Name(), pkg.String()) + } + } + } + var callerMarked bool + for _, attr := range caller.impl.GetFunctionAttributes() { + if attr.IsString() && attr.GetStringKind() == wasmresume.FunctionAttribute { + callerMarked = true + } + } + if !callerMarked { + t.Fatalf("ordinary Go caller was not marked resumable:\n%s", pkg.String()) + } + if ir := pkg.String(); !strings.Contains(ir, "@"+closureStub+"sync.ordinary") || + strings.Contains(ir, "@"+wasmresume.StartSymbol(closureStub+"sync.ordinary")) { + t.Fatalf("runtime boundary function value does not use its synchronous wrapper:\n%s", ir) + } + kind := prog.ctx.MDKindID(wasmresume.CallMetadata) + for _, function := range []Function{boundary, root, caller} { + for block := function.impl.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instr := block.FirstInstruction(); !instr.IsNil(); instr = llvm.NextInstruction(instr) { + if instr.HasMetadata() && !instr.Metadata(kind).IsNil() { + t.Fatalf("%s contains a resumable boundary call:\n%s", function.Name(), pkg.String()) + } + } + } + } +}