From a94dfb9cc51e27968d47d6023d6e053d77484478 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 15:52:29 +0800 Subject: [PATCH 01/32] feat(coro): gate plain function dispatch ABI --- cl/compilation.go | 14 +++++++++++++- cl/compilation_test.go | 24 ++++++++++++++++++++++++ cl/coro_entry.go | 3 +++ internal/build/build.go | 19 ++++++++++++++++++- internal/build/collect.go | 3 ++- internal/build/coro_plan_test.go | 30 ++++++++++++++++++++++++++---- internal/coro/plan_digest.go | 5 +++++ 7 files changed, 91 insertions(+), 7 deletions(-) diff --git a/cl/compilation.go b/cl/compilation.go index a30049f54a..7d5a5e8047 100644 --- a/cl/compilation.go +++ b/cl/compilation.go @@ -62,6 +62,11 @@ type Compilation struct { // suspends itself; a matching scheduler owns every resume and destroy // operation. EnableCoroChildAwait bool + // EnableCoroPlainDispatch permits the first descriptor/context function-value + // ABI. Only a no-capture, non-suspending plain target at an ordinary scalar + // call is accepted by this capability; every wider dynamic form remains an + // unsupported preflight error. + EnableCoroPlainDispatch bool // EnableCoroProgramBootstrapRun selects the program-root scheduler ABI for // package identities. The factory itself lives in the uncached entry module, // but every linked archive must agree with the runtime driver contract. @@ -109,6 +114,13 @@ func (c *Compilation) validateCoroABIIdentity(required bool) error { } wantSchedulerABI = coro.SchedulerProgramBootstrapABIV1 } + if c.EnableCoroPlainDispatch && !c.EnableCoroEntryResolution { + return fmt.Errorf("coroutine plain dispatch requires coroutine entry resolution") + } + wantFuncRepABI := coro.FuncRepABIV0 + if c.EnableCoroPlainDispatch { + wantFuncRepABI = coro.FuncRepABIV1 + } checks := []struct { name string got string @@ -117,7 +129,7 @@ func (c *Compilation) validateCoroABIIdentity(required bool) error { {"coroutine", c.CoroABI, wantCoroABI}, {"scheduler", c.SchedulerABI, wantSchedulerABI}, {"panic", c.PanicABI, coro.PanicLegacyABIV0}, - {"function representation", c.FuncRepABI, coro.FuncRepABIV0}, + {"function representation", c.FuncRepABI, wantFuncRepABI}, } if !required { populated := false diff --git a/cl/compilation_test.go b/cl/compilation_test.go index 6bd8089f78..fd8ec9be98 100644 --- a/cl/compilation_test.go +++ b/cl/compilation_test.go @@ -94,6 +94,30 @@ func TestCompilationCoroABIIdentityValidation(t *testing.T) { if err := (&Compilation{EnableCoroEntryResolution: true, EnableCoroPhysicalABI: true}).validateCoroABIIdentity(false); err != nil { t.Fatalf("omitted source ABI identity should use current defaults: %v", err) } + plainDispatch := &Compilation{ + EnableCoroEntryResolution: true, + EnableCoroPlainDispatch: true, + CoroABI: coro.EntryResolutionABIV0, + SchedulerABI: coro.SchedulerNoneABIV0, + PanicABI: coro.PanicLegacyABIV0, + FuncRepABI: coro.FuncRepABIV1, + } + if err := plainDispatch.validateCoroABIIdentity(false); err != nil { + t.Fatalf("complete plain-dispatch ABI identity: %v", err) + } + wrongPlainDispatch := *plainDispatch + wrongPlainDispatch.FuncRepABI = coro.FuncRepABIV0 + if err := wrongPlainDispatch.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "function representation ABI") { + t.Fatalf("plain-dispatch function representation mismatch = %v", err) + } + withoutEntry := *plainDispatch + withoutEntry.EnableCoroEntryResolution = false + if err := withoutEntry.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "requires coroutine entry resolution") { + t.Fatalf("plain-dispatch dependency error = %v", err) + } + if err := withoutEntry.preflightCoroPlan(); err == nil || !strings.Contains(err.Error(), "requires coroutine entry resolution") { + t.Fatalf("plain-dispatch preflight dependency error = %v", err) + } newChildAwait := func() *Compilation { return &Compilation{ EnableCoroEntryResolution: true, diff --git a/cl/coro_entry.go b/cl/coro_entry.go index 93c0d494b6..b7c396fe96 100644 --- a/cl/coro_entry.go +++ b/cl/coro_entry.go @@ -191,6 +191,9 @@ func (c *Compilation) preflightCoroPlan() error { if c.EnableCoroChildAwait && !c.EnableCoroPhysicalABI { return fmt.Errorf("coroutine child await requires coroutine physical ABI") } + if c.EnableCoroPlainDispatch && !c.EnableCoroEntryResolution { + return fmt.Errorf("coroutine plain dispatch requires coroutine entry resolution") + } if !c.EnableCoroEntryResolution { return nil } diff --git a/internal/build/build.go b/internal/build/build.go index 31683fcde8..daac2c931f 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -459,6 +459,12 @@ type Config struct { // async root receives a typed factory descriptor. It requires the physical // ABI and does not enable a runtime scheduler, spawn, park, or preemption. EnableCoroChildAwait bool + // EnableCoroPlainDispatch enables the v1 descriptor/context ABI for the + // narrowly supported ordinary call of a no-capture, non-suspending plain Go + // function value. It requires entry resolution and does not authorize + // coroutine, interface, reflect, method, go/defer, aggregate, or captured + // closure dispatch. + EnableCoroPlainDispatch bool // EnableCoroProgramBootstrapABI emits the target-neutral v1 startup table // for an executable after the exact init/main entries have been validated // against the frozen whole-program plan. It does not replace the legacy @@ -903,6 +909,9 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { if ctx.buildConf.EnableCoroChildAwait && !ctx.buildConf.EnableCoroPhysicalABI { return fmt.Errorf("enable coroutine child await: coroutine physical ABI is required") } + if ctx.buildConf.EnableCoroPlainDispatch && !ctx.buildConf.EnableCoroEntryResolution { + return fmt.Errorf("enable coroutine plain dispatch: coroutine entry resolution is required") + } if ctx.buildConf.EnableCoroChildAwait && ctx.buildConf.BuildMode == BuildModeCArchive { return fmt.Errorf("enable coroutine child await: c-archive requires flattened package members and an explicit host bootstrap extraction contract") } @@ -997,6 +1006,7 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { EnableCoroEntryResolution: ctx.buildConf.EnableCoroEntryResolution, EnableCoroPhysicalABI: ctx.buildConf.EnableCoroPhysicalABI, EnableCoroChildAwait: ctx.buildConf.EnableCoroChildAwait, + EnableCoroPlainDispatch: ctx.buildConf.EnableCoroPlainDispatch, EnableCoroProgramBootstrapRun: ctx.buildConf.EnableCoroProgramBootstrapRun, CoroPlanDigest: digest, CoroABI: metadata.CoroABI, @@ -1040,6 +1050,13 @@ func activeCoroSchedulerABIVersion(conf *Config) string { return coro.SchedulerNoneABIV0 } +func activeCoroFuncRepABIVersion(conf *Config) string { + if conf != nil && conf.EnableCoroPlainDispatch { + return coro.FuncRepABIV1 + } + return coro.FuncRepABIV0 +} + // requiredCoroProgramRuntimePlan returns the Go bodies referenced only by // compiler-generated entry/coroutine IR and their exact static call closure. // They are not visible from the application's source roots. The closure is a @@ -1339,7 +1356,7 @@ func buildCoroPlanDigestMetadata(ctx *context) (coro.PlanDigestMetadata, error) CoroABI: activeCoroABIVersion(ctx.buildConf), SchedulerABI: activeCoroSchedulerABIVersion(ctx.buildConf), PanicABI: coro.PanicLegacyABIV0, - FuncRepABI: coro.FuncRepABIV0, + FuncRepABI: activeCoroFuncRepABIVersion(ctx.buildConf), TargetTriple: target.Triple, TargetCPU: target.CPU, TargetFeatures: target.Features, diff --git a/internal/build/collect.go b/internal/build/collect.go index 1b004a33e2..ad3bc9ab10 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -376,6 +376,7 @@ func (c *context) canUsePackageCache() bool { return c.clCompilation.EnableCoroEntryResolution && c.clCompilation.EnableCoroPhysicalABI == c.buildConf.EnableCoroPhysicalABI && c.clCompilation.EnableCoroChildAwait == c.buildConf.EnableCoroChildAwait && + c.clCompilation.EnableCoroPlainDispatch == c.buildConf.EnableCoroPlainDispatch && c.clCompilation.CoroABI == metadata.CoroABI && c.clCompilation.SchedulerABI == metadata.SchedulerABI && c.clCompilation.PanicABI == metadata.PanicABI && @@ -383,7 +384,7 @@ func (c *context) canUsePackageCache() bool { metadata.CoroABI == activeCoroABIVersion(c.buildConf) && metadata.SchedulerABI == activeCoroSchedulerABIVersion(c.buildConf) && metadata.PanicABI == coro.PanicLegacyABIV0 && - metadata.FuncRepABI == coro.FuncRepABIV0 && + metadata.FuncRepABI == activeCoroFuncRepABIVersion(c.buildConf) && metadata.TargetTriple != "" && metadata.PointerBits > 0 && (metadata.Endianness == "little" || metadata.Endianness == "big") && metadata.DataLayout != "" diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index 00bb9efea0..22cc064cf7 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -1198,11 +1198,13 @@ func TestActiveCoroABIVersions(t *testing.T) { config *Config coroABI string scheduler string + funcRep string }{ - {"entry resolution", &Config{}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0}, - {"physical leaf", &Config{EnableCoroPhysicalABI: true}, coro.PhysicalABIV0, coro.SchedulerNoneABIV0}, - {"child await", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true}, coro.PhysicalABIV1, coro.SchedulerChildAwaitABIV0}, - {"program bootstrap runtime", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroProgramBootstrapRun: true}, coro.PhysicalABIV1, coro.SchedulerProgramBootstrapABIV1}, + {"entry resolution", &Config{}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0, coro.FuncRepABIV0}, + {"physical leaf", &Config{EnableCoroPhysicalABI: true}, coro.PhysicalABIV0, coro.SchedulerNoneABIV0, coro.FuncRepABIV0}, + {"plain dispatch", &Config{EnableCoroPlainDispatch: true}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0, coro.FuncRepABIV1}, + {"child await", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true}, coro.PhysicalABIV1, coro.SchedulerChildAwaitABIV0, coro.FuncRepABIV0}, + {"program bootstrap runtime with plain dispatch", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroPlainDispatch: true, EnableCoroProgramBootstrapRun: true}, coro.PhysicalABIV1, coro.SchedulerProgramBootstrapABIV1, coro.FuncRepABIV1}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -1212,6 +1214,9 @@ func TestActiveCoroABIVersions(t *testing.T) { if got := activeCoroSchedulerABIVersion(test.config); got != test.scheduler { t.Fatalf("scheduler ABI = %q, want %q", got, test.scheduler) } + if got := activeCoroFuncRepABIVersion(test.config); got != test.funcRep { + t.Fatalf("function representation ABI = %q, want %q", got, test.funcRep) + } }) } } @@ -1321,6 +1326,11 @@ func TestBuildCoroPlanErrors(t *testing.T) { conf Config want string }{ + { + name: "plain dispatch requires entry resolution", + conf: Config{EnableCoroPlainDispatch: true}, + want: "plain dispatch: coroutine entry resolution is required", + }, { name: "program bootstrap runtime requires descriptor ABI", conf: Config{BuildMode: BuildModeExe, EnableCoroEntryResolution: true, EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroProgramBootstrapRun: true}, @@ -1596,6 +1606,18 @@ func TestCoroEntryResolutionUsesPlanMatchedPackageCache(t *testing.T) { if !seedCtx.tryLoadFromCache(matchingPkg) || !matchingPkg.CacheHit { t.Fatal("matching coroutine plan did not reuse the package archive") } + dispatchCtx := newContext(digestA) + dispatchCtx.buildConf.EnableCoroPlainDispatch = true + dispatchCtx.clCompilation.EnableCoroPlainDispatch = true + dispatchCtx.clCompilation.FuncRepABI = coro.FuncRepABIV1 + dispatchCtx.coroPlanMetadata.FuncRepABI = coro.FuncRepABIV1 + if !dispatchCtx.canUsePackageCache() { + t.Fatal("matching plain-dispatch ABI unexpectedly disabled package cache") + } + dispatchCtx.clCompilation.EnableCoroPlainDispatch = false + if dispatchCtx.canUsePackageCache() { + t.Fatal("plain-dispatch capability mismatch unexpectedly permits package cache") + } if !matchingPkg.NeedRt || !matchingPkg.NeedPyInit { t.Fatalf("cache metadata runtime flags = %v/%v, want true/true", matchingPkg.NeedRt, matchingPkg.NeedPyInit) } diff --git a/internal/coro/plan_digest.go b/internal/coro/plan_digest.go index 7cf1b0fd7c..5dd064bff8 100644 --- a/internal/coro/plan_digest.go +++ b/internal/coro/plan_digest.go @@ -52,6 +52,11 @@ const ( SchedulerProgramBootstrapABIV1 = "llgo.coro.scheduler.program-bootstrap.v1" PanicLegacyABIV0 = "llgo.coro.panic.legacy.v0" FuncRepABIV0 = "llgo.coro.func-rep.v0" + // FuncRepABIV1 introduces an explicit descriptor/context representation for + // dynamically consumed Go function values. The first producer/consumer slice + // supports only one no-capture, non-suspending plain body; unsupported value + // shapes and call capabilities remain fail-closed. + FuncRepABIV1 = "llgo.coro.func-rep.v1" ) // PlanDigestMetadata contains every effective ABI and target input that may From a37912076fd62ba836ed47d003a6b3cd9280ff34 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 16:05:34 +0800 Subject: [PATCH 02/32] feat(coro): certify closed dynamic call targets --- internal/coro/closed_dynamic_call_test.go | 230 ++++++++++++++++++++++ internal/coro/func_flow.go | 42 ++++ internal/coro/ssa_plan.go | 121 +++++++++++- 3 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 internal/coro/closed_dynamic_call_test.go diff --git a/internal/coro/closed_dynamic_call_test.go b/internal/coro/closed_dynamic_call_test.go new file mode 100644 index 0000000000..d1c446ddc5 --- /dev/null +++ b/internal/coro/closed_dynamic_call_test.go @@ -0,0 +1,230 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "strings" + "testing" + + "golang.org/x/tools/go/ssa" +) + +func TestAnalyzeSSAClosedDynamicCallCertificates(t *testing.T) { + prog, pkg := buildClosedDynamicCallTestSSA(t) + dynamicPlain := packageFunction(t, pkg, "dynamicPlain") + nilOnly := packageFunction(t, pkg, "nilOnly") + dynamicSuspend := packageFunction(t, pkg, "dynamicSuspend") + plain := packageFunction(t, pkg, "plain") + suspend := packageFunction(t, pkg, "suspend") + + plan, err := AnalyzeSSA(prog, Roots{ + {Function: dynamicPlain, Demand: AsyncDemand}, + {Function: nilOnly, Demand: AsyncDemand}, + {Function: dynamicSuspend, Demand: AsyncDemand}, + }, SSAConfig{ + ClassifyClosedDynamicCall: func(caller *ssa.Function, _ ssa.CallInstruction) (SSAClosedDynamicCallCertificate, bool, error) { + switch caller { + case dynamicPlain: + return SSAClosedDynamicCallCertificate{Targets: []*ssa.Function{plain}, MayBeNil: true}, true, nil + case nilOnly: + return SSAClosedDynamicCallCertificate{MayBeNil: true}, true, nil + case dynamicSuspend: + return SSAClosedDynamicCallCertificate{Targets: []*ssa.Function{suspend}, MayBeNil: true}, true, nil + default: + return SSAClosedDynamicCallCertificate{}, false, nil + } + }, + }) + if err != nil { + t.Fatal(err) + } + + plainCall := onlyNonBuiltinCall(t, dynamicPlain) + assertClosedDynamicCall(t, plan, plainCall, true, plain) + plainValue, ok := plan.ValuePlan(plainCall.Common().Value) + if !ok || len(plainValue.Funcs) != 1 || plainValue.Funcs[0].Rep != Dispatch || + !plainValue.Funcs[0].MayBeNil || len(plainValue.Funcs[0].Targets) != 1 { + t.Fatalf("certified plain callee value = %+v, present=%t; want nullable singleton Dispatch", plainValue, ok) + } + if got := functionPlanFor(t, plan, dynamicPlain); got.Effect != NoSuspend || got.Effect.IsOpaque() { + t.Fatalf("certified plain caller effect = %s, want no-suspend", got.Effect) + } + if got := functionPlanFor(t, plan, plain); got.FuncRep != Dispatch || got.Primary != PrimaryPlain { + t.Fatalf("certified plain target plan = %+v, want descriptor-backed plain target", got) + } + + nilCall := onlyNonBuiltinCall(t, nilOnly) + assertClosedDynamicCall(t, plan, nilCall, true) + if got := functionPlanFor(t, plan, nilOnly); got.Effect != NoSuspend || got.Effect.IsOpaque() { + t.Fatalf("closed nil-only caller effect = %s, want no-suspend", got.Effect) + } + + suspendCall := onlyNonBuiltinCall(t, dynamicSuspend) + assertClosedDynamicCall(t, plan, suspendCall, true, suspend) + if got := functionPlanFor(t, plan, dynamicSuspend); got.Effect.IsOpaque() || !got.Effect.Contains(MayPark) { + t.Fatalf("certified suspending caller effect = %s, want known MayPark", got.Effect) + } + if got := functionPlanFor(t, plan, suspend); got.Demand != AsyncDemand || got.FuncRep != Dispatch || !got.Effect.Contains(MayPark) { + t.Fatalf("certified suspending target plan = %+v, want demanded descriptor target with MayPark", got) + } +} + +func TestAnalyzeSSAClosedDynamicCallCertificateRejectsInvalidProof(t *testing.T) { + prog, pkg := buildClosedDynamicCallTestSSA(t) + dynamicPlain := packageFunction(t, pkg, "dynamicPlain") + plain := packageFunction(t, pkg, "plain") + suspend := packageFunction(t, pkg, "suspend") + wrongSignature := packageFunction(t, pkg, "wrongSignature") + external := packageFunction(t, pkg, "external") + + tests := []struct { + name string + caller *ssa.Function + certificate SSAClosedDynamicCallCertificate + want string + }{ + { + name: "static", + caller: packageFunction(t, pkg, "staticCall"), + certificate: SSAClosedDynamicCallCertificate{Targets: []*ssa.Function{plain}}, + want: "cannot identify a static call", + }, + { + name: "invoke", + caller: packageFunction(t, pkg, "interfaceInvoke"), + certificate: SSAClosedDynamicCallCertificate{MayBeNil: true}, + want: "cannot identify an interface invoke", + }, + { + name: "go", + caller: packageFunction(t, pkg, "goDynamic"), + certificate: SSAClosedDynamicCallCertificate{MayBeNil: true}, + want: "ordinary *ssa.Call", + }, + { + name: "defer", + caller: packageFunction(t, pkg, "deferDynamic"), + certificate: SSAClosedDynamicCallCertificate{MayBeNil: true}, + want: "ordinary *ssa.Call", + }, + { + name: "multiple targets", + caller: dynamicPlain, + certificate: SSAClosedDynamicCallCertificate{Targets: []*ssa.Function{plain, suspend}, MayBeNil: true}, + want: "has 2 targets", + }, + { + name: "signature mismatch", + caller: dynamicPlain, + certificate: SSAClosedDynamicCallCertificate{Targets: []*ssa.Function{wrongSignature}, MayBeNil: true}, + want: "has signature", + }, + { + name: "external target", + caller: dynamicPlain, + certificate: SSAClosedDynamicCallCertificate{Targets: []*ssa.Function{external}, MayBeNil: true}, + want: "not an external target", + }, + { + name: "empty non-nil", + caller: dynamicPlain, + certificate: SSAClosedDynamicCallCertificate{}, + want: "neither a target nor nil", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + call := onlyNonBuiltinCall(t, test.caller) + _, err := AnalyzeSSA(prog, Roots{{Function: test.caller, Demand: AsyncDemand}}, SSAConfig{ + ClassifyClosedDynamicCall: func(_ *ssa.Function, candidate ssa.CallInstruction) (SSAClosedDynamicCallCertificate, bool, error) { + if candidate != call { + return SSAClosedDynamicCallCertificate{}, false, nil + } + return test.certificate, true, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("AnalyzeSSA error = %v, want substring %q", err, test.want) + } + }) + } +} + +func TestAnalyzeSSAClosedDynamicCallCertificateRejectsTargetOutsideUniverse(t *testing.T) { + prog, pkg := buildClosedDynamicCallTestSSA(t) + caller := packageFunction(t, pkg, "dynamicPlain") + target := packageFunction(t, pkg, "plain") + universe, err := NewSSAEmissionUniverse(prog, []*ssa.Function{caller}) + if err != nil { + t.Fatal(err) + } + _, err = AnalyzeSSA(prog, Roots{{Function: caller, Demand: AsyncDemand}}, SSAConfig{ + EmissionUniverse: universe, + ClassifyClosedDynamicCall: func(_ *ssa.Function, _ ssa.CallInstruction) (SSAClosedDynamicCallCertificate, bool, error) { + return SSAClosedDynamicCallCertificate{Targets: []*ssa.Function{target}, MayBeNil: true}, true, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "outside the effective emission universe") { + t.Fatalf("outside-universe certificate error = %v", err) + } +} + +func assertClosedDynamicCall(t *testing.T, plan *SSAPlan, call ssa.CallInstruction, mayBeNil bool, targets ...*ssa.Function) { + t.Helper() + got, ok := plan.CallPlan(call) + if !ok { + t.Fatalf("certified call %s has no CallPlan", call) + } + if got.Rep != Dispatch || got.Open || got.MayBeNil != mayBeNil || len(got.Targets) != len(targets) { + t.Fatalf("certified call plan = %+v, want closed Dispatch nil=%t targets=%d", got, mayBeNil, len(targets)) + } + for i, target := range targets { + id, ok := plan.FunctionID(target) + if !ok { + t.Fatalf("certified target %q has no FunctionID", target.Name()) + } + if got.Targets[i] != id { + t.Fatalf("certified call target[%d] = %s, want %s", i, got.Targets[i], id) + } + } +} + +func buildClosedDynamicCallTestSSA(t *testing.T) (*ssa.Program, *ssa.Package) { + t.Helper() + return buildCoroTestSSA(t, "closed_dynamic_call.go", `package coroid + +var channel chan int + +func plain(int) {} +func suspend(int) { <-channel } +func wrongSignature(string) {} +func external(int) + +func dynamicPlain(fn func(int)) { fn(1) } +func nilOnly(fn func(int)) { fn(2) } +func dynamicSuspend(fn func(int)) { fn(3) } +func staticCall() { plain(4) } + +type Interface interface { Method() } +func interfaceInvoke(value Interface) { value.Method() } +func goDynamic(fn func(int)) { go fn(5) } +func deferDynamic(fn func(int)) { defer fn(6) } +`) +} diff --git a/internal/coro/func_flow.go b/internal/coro/func_flow.go index 4e8e8c2945..c99888ceb2 100644 --- a/internal/coro/func_flow.go +++ b/internal/coro/func_flow.go @@ -161,6 +161,7 @@ type ssaFuncFlow struct { canonicalizer *ssaFunctionCanonicalizer directPlainArgs map[ssaCallArgumentUse]struct{} directPlainOrder []ssaCallArgumentUse + closedValues map[ssa.Value]SSAClosedDynamicCallCertificate } type ssaCallArgumentUse struct { @@ -176,6 +177,7 @@ func analyzeSSAFunctionFlow( dynamicResolution DynamicResolution, canonicalizer *ssaFunctionCanonicalizer, directPlainArgs []ssaCallArgumentUse, + closedDynamicCalls map[ssa.CallInstruction]SSAClosedDynamicCallCertificate, ) (*ssaFuncFlow, error) { directPlainSet := make(map[ssaCallArgumentUse]struct{}, len(directPlainArgs)) for _, use := range directPlainArgs { @@ -192,6 +194,20 @@ func analyzeSSAFunctionFlow( canonicalizer: canonicalizer, directPlainArgs: directPlainSet, directPlainOrder: append([]ssaCallArgumentUse(nil), directPlainArgs...), + closedValues: make(map[ssa.Value]SSAClosedDynamicCallCertificate, len(closedDynamicCalls)), + } + for call, certificate := range closedDynamicCalls { + value := call.Common().Value + if previous, exists := flow.closedValues[value]; exists { + if !sameSSAClosedDynamicCallCertificate(previous, certificate) { + return nil, fmt.Errorf("conflicting closed dynamic call certificates for callee value in %q", call.Parent().Name()) + } + continue + } + flow.closedValues[value] = SSAClosedDynamicCallCertificate{ + Targets: append([]*ssa.Function(nil), certificate.Targets...), + MayBeNil: certificate.MayBeNil, + } } for _, fn := range functions { @@ -255,6 +271,20 @@ func analyzeSSAFunctionFlow( if !isScalarFuncType(value.Type()) { continue } + if certificate, certified := flow.closedValues[value]; certified { + for _, target := range certificate.Targets { + if err := flow.addTarget(value, target); err != nil { + return nil, fmt.Errorf("resolve certified function-value target %q: %w", target.Name(), err) + } + } + if certificate.MayBeNil { + flow.markMayBeNil(value) + } + // The proof closes the target set, not the physical representation: + // this value crossed canonical storage and must retain Dispatch. + flow.markBoundary(value) + continue + } switch value := value.(type) { case *ssa.Function: if err := flow.addTarget(value, value); err != nil { @@ -303,6 +333,18 @@ func analyzeSSAFunctionFlow( return flow, nil } +func sameSSAClosedDynamicCallCertificate(left, right SSAClosedDynamicCallCertificate) bool { + if left.MayBeNil != right.MayBeNil || len(left.Targets) != len(right.Targets) { + return false + } + for i := range left.Targets { + if left.Targets[i] != right.Targets[i] { + return false + } + } + return true +} + func (f *ssaFuncFlow) recordValue(value ssa.Value) { if value == nil || value.Type() == nil { return diff --git a/internal/coro/ssa_plan.go b/internal/coro/ssa_plan.go index 23fa620c2c..5e36c12fcf 100644 --- a/internal/coro/ssa_plan.go +++ b/internal/coro/ssa_plan.go @@ -101,6 +101,20 @@ type SSAFunctionPolicy struct { // refer to the replaced SSA declaration. type SSAFunctionResolver func(fn *ssa.Function) (canonical *ssa.Function, ok bool, err error) +// SSAClosedDynamicCallCertificate is a trusted frontend proof for one exact +// ordinary dynamic call. V0 intentionally accepts at most one non-nil target: +// the narrow form is sufficient for fields whose whole-program writes are +// proven to contain either nil or one descriptor-backed function value. +// +// Targets is copied and validated before analysis. An empty Targets slice is a +// closed nil-only value and therefore requires MayBeNil. A singleton may be +// either nullable or non-null. The target must be an exact canonical, owned Go +// body in the effective emission universe with the call's exact signature. +type SSAClosedDynamicCallCertificate struct { + Targets []*ssa.Function + MayBeNil bool +} + // SSAConfig controls the SSA-to-Graph analysis bridge. It deliberately has no // lowering or runtime switches. type SSAConfig struct { @@ -165,6 +179,19 @@ type SSAConfig struct { // arguments. Frontends should reserve it for source-level ABI facts such as // a named //llgo:type C callback parameter. ClassifyDirectPlainCallArgument func(caller *ssa.Function, call ssa.CallInstruction, argument int) (bool, error) + + // ClassifyClosedDynamicCall supplies a frozen whole-program proof for one + // exact ordinary dynamic *ssa.Call whose callee value crosses descriptor + // storage but has a closed nil-or-singleton target set. This is not a general + // points-to hint: AnalyzeSSA rejects static calls, invokes, go/defer sites, + // multiple targets, captured functions, signature mismatches, aliases, + // external declarations, and targets outside the effective universe. + // + // A certified callee remains Dispatch because it crossed canonical storage; + // the certificate only closes its graph edge and CallPlan target set. The + // callback is trusted to have rejected every unknown physical write or escape + // that could reach the exact value loaded at call. + ClassifyClosedDynamicCall func(caller *ssa.Function, call ssa.CallInstruction) (SSAClosedDynamicCallCertificate, bool, error) } // SSAFunctionPlan binds an immutable FunctionPlan back to its SSA function. @@ -593,7 +620,11 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err if err != nil { return nil, err } - flow, err := analyzeSSAFunctionFlow(bodyFunctions, includedSet, ids, dynamicCandidates, config.DynamicResolution, canonicalizer, directPlainCallArguments) + closedDynamicCalls, err := classifySSAClosedDynamicCalls(bodyFunctions, includedSet, bodyFunctionSet, trustedPolicies, canonicalizer, config) + if err != nil { + return nil, err + } + flow, err := analyzeSSAFunctionFlow(bodyFunctions, includedSet, ids, dynamicCandidates, config.DynamicResolution, canonicalizer, directPlainCallArguments, closedDynamicCalls) if err != nil { return nil, fmt.Errorf("coro: analyze SSA function-value flow: %w", err) } @@ -881,6 +912,94 @@ func classifySSADirectPlainCallArguments(functions []*ssa.Function, config SSACo return result, nil } +func classifySSAClosedDynamicCalls( + functions []*ssa.Function, + included map[*ssa.Function]bool, + bodyFunctions map[*ssa.Function]bool, + policies map[*ssa.Function]SSAFunctionPolicy, + canonicalizer *ssaFunctionCanonicalizer, + config SSAConfig, +) (map[ssa.CallInstruction]SSAClosedDynamicCallCertificate, error) { + result := make(map[ssa.CallInstruction]SSAClosedDynamicCallCertificate) + if config.ClassifyClosedDynamicCall == nil { + return result, nil + } + for _, caller := range functions { + for _, block := range caller.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(ssa.CallInstruction) + if !ok { + continue + } + certificate, certified, err := config.ClassifyClosedDynamicCall(caller, call) + if err != nil { + return nil, fmt.Errorf("coro: classify closed dynamic call in %q: %w", caller.Name(), err) + } + if !certified { + if len(certificate.Targets) != 0 || certificate.MayBeNil { + return nil, fmt.Errorf("coro: unclassified dynamic call in %q returned non-empty certificate facts", caller.Name()) + } + continue + } + common := call.Common() + if _, direct := call.(*ssa.Call); !direct || common == nil || call.Parent() != caller { + return nil, fmt.Errorf("coro: closed dynamic call certificate in %q must identify an exact ordinary *ssa.Call", caller.Name()) + } + if common.StaticCallee() != nil { + return nil, fmt.Errorf("coro: closed dynamic call certificate in %q cannot identify a static call", caller.Name()) + } + if common.IsInvoke() { + return nil, fmt.Errorf("coro: closed dynamic call certificate in %q cannot identify an interface invoke", caller.Name()) + } + if _, builtin := common.Value.(*ssa.Builtin); builtin || common.Value == nil || !isScalarFuncType(common.Value.Type()) { + return nil, fmt.Errorf("coro: closed dynamic call certificate in %q requires a scalar Go function callee", caller.Name()) + } + if len(certificate.Targets) > 1 { + return nil, fmt.Errorf("coro: closed dynamic call certificate in %q has %d targets; only nil or one exact target is supported", caller.Name(), len(certificate.Targets)) + } + if len(certificate.Targets) == 0 && !certificate.MayBeNil { + return nil, fmt.Errorf("coro: closed dynamic call certificate in %q has neither a target nor nil", caller.Name()) + } + + cloned := SSAClosedDynamicCallCertificate{MayBeNil: certificate.MayBeNil} + if len(certificate.Targets) == 1 { + target := certificate.Targets[0] + if target == nil { + return nil, fmt.Errorf("coro: closed dynamic call certificate in %q has a nil target entry", caller.Name()) + } + if target.Prog != caller.Prog { + return nil, fmt.Errorf("coro: closed dynamic call certificate in %q targets function %q from another SSA program", caller.Name(), target.Name()) + } + canonical, resolved, resolveErr := canonicalizer.resolve(target) + if resolveErr != nil { + return nil, fmt.Errorf("coro: resolve closed dynamic target %q in %q: %w", target.Name(), caller.Name(), resolveErr) + } + if !resolved || canonical == nil || !included[canonical] { + return nil, fmt.Errorf("coro: closed dynamic target %q in %q is outside the effective emission universe", target.Name(), caller.Name()) + } + if canonical != target { + return nil, fmt.Errorf("coro: closed dynamic target %q in %q is not the exact canonical function", target.Name(), caller.Name()) + } + policy := policies[target] + if !bodyFunctions[target] || len(target.Blocks) == 0 || policy.IgnoreBody || (policy.OverrideExternal && policy.External != Defined) { + return nil, fmt.Errorf("coro: closed dynamic target %q in %q must be an owned emitted Go body, not an external target", target.Name(), caller.Name()) + } + if len(target.FreeVars) != 0 { + return nil, fmt.Errorf("coro: closed dynamic target %q in %q has %d captured variables", target.Name(), caller.Name(), len(target.FreeVars)) + } + callSignature := common.Signature() + if callSignature == nil || target.Signature == nil || !types.Identical(callSignature, target.Signature) { + return nil, fmt.Errorf("coro: closed dynamic target %q in %q has signature %v, want %v", target.Name(), caller.Name(), target.Signature, callSignature) + } + cloned.Targets = []*ssa.Function{target} + } + result[call] = cloned + } + } + } + return result, nil +} + func classifySSAElidedCalls(functions []*ssa.Function, config SSAConfig) (map[ssa.CallInstruction]bool, error) { result := make(map[ssa.CallInstruction]bool) if config.ClassifyElidedCall == nil { From 2d4f25cc9829c854d5347bc955741a2a4ff6aae9 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 16:18:27 +0800 Subject: [PATCH 03/32] feat(coro): lower plain dispatch descriptors --- ssa/coro_dispatch.go | 487 ++++++++++++++++++++++++++++++++++++++ ssa/coro_dispatch_test.go | 382 ++++++++++++++++++++++++++++++ ssa/type.go | 8 + 3 files changed, 877 insertions(+) create mode 100644 ssa/coro_dispatch.go create mode 100644 ssa/coro_dispatch_test.go diff --git a/ssa/coro_dispatch.go b/ssa/coro_dispatch.go new file mode 100644 index 0000000000..ed2ef90391 --- /dev/null +++ b/ssa/coro_dispatch.go @@ -0,0 +1,487 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import ( + "encoding/binary" + "fmt" + "go/token" + "go/types" + "strings" + + "github.com/xgo-dev/llvm" +) + +// Coro plain-dispatch version and capability flags are linker-visible ABI. +// HasCoro is reserved by v1 even though the first production slice emits only +// the exact HasPlain|NoCapture capability set. +const CoroPlainDispatchVersionV1 uint32 = 1 + +const ( + CoroDispatchFlagHasPlain uint32 = 1 << iota + CoroDispatchFlagHasCoro + CoroDispatchFlagNoCapture + + CoroPlainDispatchFlagsV1 = CoroDispatchFlagHasPlain | CoroDispatchFlagNoCapture +) + +const coroPlainDispatchThunkPrefix = "__llgo_coro_func_plain_v1." + +// CoroPlainDispatchThunkName derives the dedicated v1 thunk symbol for one +// target-specific symbol identity. The frontend should pass its final planned +// symbol name, which is the only place target FunctionID identity belongs. +func CoroPlainDispatchThunkName(targetSymbol string) string { + if targetSymbol == "" { + panic("ssa: coroutine plain dispatch thunk requires a target symbol") + } + return coroPlainDispatchThunkPrefix + targetSymbol +} + +// CoroPlainDispatchDescriptorOptions describes one v1 plain-only function +// descriptor. ABIHash is supplied by the frontend and deliberately does not +// include the target FunctionID. Target identity belongs only in Name and +// ThunkName, which lets identical ABI contracts share the same hash. +// +// Signature is the source Go signature. PlainTarget is its one compiler-owned +// plain body. Result is the canonical result-slot layout. This API creates a +// target-specific (env, args)->results thunk; callers must not pass or reuse +// the legacy closure stub. +type CoroPlainDispatchDescriptorOptions struct { + Version uint32 + Flags uint32 + ABIHash [16]byte + PlainTarget Expr + Signature *types.Signature + ThunkName string + Result Type +} + +// CoroPlainDispatchCallOptions is the caller's exact expected v1 contract. +// Result is the canonical result-slot layout used by the ABI hash and layout +// guards; it is distinct from the direct LLVM call's return type. +type CoroPlainDispatchCallOptions struct { + Version uint32 + Flags uint32 + ABIHash [16]byte + Result Type +} + +// NewCoroPlainDispatchDescriptor defines a link-once constant descriptor: +// +// { version i32, flags i32, hashLo i64, hashHi i64, +// plainEntry ptr, coroEntry ptr, resultSize uintptr, +// resultAlign uintptr } +// +// plainEntry is a target-specific context thunk and coroEntry is null. The +// descriptor is returned as a pointer. Hash words use big-endian byte order so +// their textual IR form is deterministic across hosts. +func (p Package) NewCoroPlainDispatchDescriptor( + name string, opts CoroPlainDispatchDescriptorOptions, +) Expr { + if name == "" { + panic("ssa: coroutine plain dispatch descriptor requires a name") + } + validateCoroPlainDispatchContract(opts.Version, opts.Flags) + if opts.Signature == nil { + panic("ssa: coroutine plain dispatch descriptor requires a signature") + } + if err := validateCoroPlainDispatchSignature(p.Prog, opts.Signature); err != nil { + panic("ssa: coroutine plain dispatch descriptor: " + err.Error()) + } + if opts.ThunkName == "" { + panic("ssa: coroutine plain dispatch descriptor requires a target-specific thunk name") + } + if opts.ThunkName == name { + panic("ssa: coroutine plain dispatch descriptor and thunk require distinct symbols") + } + if strings.HasPrefix(opts.ThunkName, closureStub) { + panic("ssa: coroutine plain dispatch thunk must not reuse the legacy closure stub namespace") + } + if opts.Result == nil || opts.Result.kind == vkInvalid || + opts.Result.ll.Context().C != p.Prog.ctx.C { + panic("ssa: coroutine plain dispatch descriptor requires a result layout from the same program") + } + target := coroPlainDispatchFunction(opts.PlainTarget.impl) + if opts.PlainTarget.IsNil() || opts.PlainTarget.kind != vkFuncDecl || + target.IsNil() || target.GlobalParent().C != p.mod.C { + panic("ssa: coroutine plain dispatch requires a plain target from the same package module") + } + targetFn := p.FuncOf(target.Name()) + if targetFn == nil || targetFn.impl.C != target.C || targetFn.base != 0 { + panic("ssa: coroutine plain dispatch requires a no-capture plain target") + } + physicalSig := p.Prog.PhysicalFuncDecl(opts.Signature, InGo) + if closureCtxParam(physicalSig) != nil || + !types.Identical(opts.PlainTarget.RawType(), physicalSig) { + panic("ssa: coroutine plain dispatch target does not match the lowered signature") + } + if descriptor := p.VarOf(name); descriptor != nil { + thunk := p.FuncOf(opts.ThunkName) + if thunk != nil && p.matchesCoroPlainDispatchDescriptor( + descriptor, thunk, target, physicalSig, opts, + ) { + return descriptor.Expr + } + panic(fmt.Sprintf("ssa: coroutine plain dispatch symbol %q conflicts with an existing descriptor", name)) + } + for _, symbol := range []string{name, opts.ThunkName} { + _, knownGlobal := p.vars[symbol] + _, knownFunction := p.fns[symbol] + if knownGlobal || knownFunction || + !p.mod.NamedGlobal(symbol).IsNil() || !p.mod.NamedFunction(symbol).IsNil() { + panic(fmt.Sprintf("ssa: coroutine plain dispatch symbol %q already exists", symbol)) + } + } + + thunk := p.newCoroPlainDispatchThunk(opts.ThunkName, opts.PlainTarget, physicalSig) + descriptorType := p.Prog.coroPlainDispatchDescriptorType() + descriptor := p.NewVarEx(name, p.Prog.Pointer(descriptorType)) + fields := []llvm.Value{ + p.Prog.IntVal(uint64(opts.Version), p.Prog.Uint32()).impl, + p.Prog.IntVal(uint64(opts.Flags), p.Prog.Uint32()).impl, + p.Prog.IntVal(binary.BigEndian.Uint64(opts.ABIHash[:8]), p.Prog.Uint64()).impl, + p.Prog.IntVal(binary.BigEndian.Uint64(opts.ABIHash[8:]), p.Prog.Uint64()).impl, + thunk.impl, + p.Prog.Nil(p.Prog.VoidPtr()).impl, + p.Prog.IntVal(p.Prog.SizeOf(opts.Result), p.Prog.Uintptr()).impl, + p.Prog.IntVal(p.Prog.AlignOf(opts.Result), p.Prog.Uintptr()).impl, + } + descriptor.impl.SetInitializer(p.Prog.ctx.ConstStruct(fields, false)) + descriptor.impl.SetGlobalConstant(true) + descriptor.impl.SetLinkage(llvm.LinkOnceODRLinkage) + descriptor.impl.SetUnnamedAddr(true) + return descriptor.Expr +} + +// MakeCoroPlainDispatchValue constructs the canonical two-pointer function +// value {descriptor, nil}. The descriptor occupies the existing code word; +// LLVM opaque pointers keep the physical closure layout unchanged. +func (b Builder) MakeCoroPlainDispatchValue( + sig *types.Signature, descriptor Expr, +) Expr { + if sig == nil { + panic("ssa: coroutine plain dispatch value requires a signature") + } + if err := validateCoroPlainDispatchSignature(b.Prog, sig); err != nil { + panic("ssa: coroutine plain dispatch value: " + err.Error()) + } + if !b.Pkg.isCoroPlainDispatchDescriptor(descriptor) { + panic("ssa: coroutine plain dispatch value requires a descriptor from the same package module") + } + return b.aggregateValue( + b.Prog.Closure(sig), descriptor.impl, b.Prog.Nil(b.Prog.VoidPtr()).impl, + ) +} + +// CallCoroPlainDispatch validates and calls an ordinary v1 plain-only dynamic +// function value. A nil descriptor uses the same recoverable Go nil-call panic +// path as the legacy closure call. Invalid or forged non-nil representation +// state traps. All checks precede the descriptor entry call; success performs +// a typed (env,args)->results indirect call. +func (b Builder) CallCoroPlainDispatch( + fn Expr, args []Expr, opts CoroPlainDispatchCallOptions, +) (ret Expr) { + validateCoroPlainDispatchContract(opts.Version, opts.Flags) + if fn.IsNil() || fn.kind != vkClosure { + panic("ssa: coroutine plain dispatch call requires a closure value") + } + sig, ok := b.Prog.Field(fn.Type, 0).RawType().(*types.Signature) + if !ok { + panic("ssa: coroutine plain dispatch call has no function signature") + } + if err := validateCoroPlainDispatchPhysicalSignature(b.Prog, sig); err != nil { + panic("ssa: coroutine plain dispatch call: " + err.Error()) + } + if len(args) != sig.Params().Len() { + panic(fmt.Sprintf( + "ssa: coroutine plain dispatch call has %d arguments, want %d", + len(args), sig.Params().Len(), + )) + } + wantResult := b.Prog.retType(sig) + if opts.Result == nil || opts.Result.kind == vkInvalid || + opts.Result.ll.Context().C != b.Prog.ctx.C { + panic("ssa: coroutine plain dispatch call requires a result layout from the same program") + } + + descriptorWord := b.Field(fn, 0) + env := b.Field(fn, 1) + // Preserve Go's recoverable nil function-call semantics. AssertNilDeref + // returns only on the non-nil path, so the descriptor load below is safe. + b.AssertNilDeref(descriptorWord) + envNonNil := llvm.CreateICmp( + b.impl, llvm.IntNE, env.impl, llvm.ConstNull(env.impl.Type()), + ) + envNonNil.SetName("coro.dispatch.env.nonnull") + b.coroPlainDispatchTrapIf(envNonNil) + + descriptorType := b.Prog.coroPlainDispatchDescriptorType() + descriptorPtr := Expr{descriptorWord.impl, b.Prog.Pointer(descriptorType)} + descriptor := b.Load(descriptorPtr) + fields := make([]Expr, 8) + for i := range fields { + fields[i] = b.Field(descriptor, i) + } + + expected := []llvm.Value{ + b.Prog.IntVal(uint64(opts.Version), b.Prog.Uint32()).impl, + b.Prog.IntVal(uint64(opts.Flags), b.Prog.Uint32()).impl, + b.Prog.IntVal(binary.BigEndian.Uint64(opts.ABIHash[:8]), b.Prog.Uint64()).impl, + b.Prog.IntVal(binary.BigEndian.Uint64(opts.ABIHash[8:]), b.Prog.Uint64()).impl, + } + var invalid llvm.Value + for i, want := range expected { + mismatch := llvm.CreateICmp(b.impl, llvm.IntNE, fields[i].impl, want) + mismatch.SetName(fmt.Sprintf("coro.dispatch.field.%d.invalid", i)) + invalid = coroPlainDispatchOr(b.impl, invalid, mismatch) + } + plainNil := llvm.CreateICmp( + b.impl, llvm.IntEQ, fields[4].impl, llvm.ConstNull(fields[4].impl.Type()), + ) + plainNil.SetName("coro.dispatch.plain.nil") + invalid = coroPlainDispatchOr(b.impl, invalid, plainNil) + coroNonNil := llvm.CreateICmp( + b.impl, llvm.IntNE, fields[5].impl, llvm.ConstNull(fields[5].impl.Type()), + ) + coroNonNil.SetName("coro.dispatch.coro.nonnull") + invalid = coroPlainDispatchOr(b.impl, invalid, coroNonNil) + resultSizeInvalid := llvm.CreateICmp( + b.impl, llvm.IntNE, fields[6].impl, + b.Prog.IntVal(b.Prog.SizeOf(opts.Result), b.Prog.Uintptr()).impl, + ) + resultSizeInvalid.SetName("coro.dispatch.result.size.invalid") + invalid = coroPlainDispatchOr(b.impl, invalid, resultSizeInvalid) + resultAlignInvalid := llvm.CreateICmp( + b.impl, llvm.IntNE, fields[7].impl, + b.Prog.IntVal(b.Prog.AlignOf(opts.Result), b.Prog.Uintptr()).impl, + ) + resultAlignInvalid.SetName("coro.dispatch.result.align.invalid") + invalid = coroPlainDispatchOr(b.impl, invalid, resultAlignInvalid) + b.coroPlainDispatchTrapIf(invalid) + + ctx := types.NewParam(token.NoPos, nil, closureCtx, types.Typ[types.UnsafePointer]) + sigCtx := FuncAddCtx(ctx, sig) + ret.Type = wantResult + ret.impl = llvm.CreateCall( + b.impl, b.Prog.FuncDecl(sigCtx, InC).ll, fields[4].impl, + llvmParamsEx(env, args, sigCtx.Params(), b), + ) + return +} + +func (p Program) coroPlainDispatchDescriptorType() Type { + return p.Struct( + p.Uint32(), + p.Uint32(), + p.Uint64(), + p.Uint64(), + p.VoidPtr(), + p.VoidPtr(), + p.Uintptr(), + p.Uintptr(), + ) +} + +func (p Package) newCoroPlainDispatchThunk( + name string, target Expr, physicalSig *types.Signature, +) Function { + ctx := types.NewParam(token.NoPos, nil, closureCtx, types.Typ[types.UnsafePointer]) + thunk := p.NewFunc(name, FuncAddCtx(ctx, physicalSig), InC) + thunk.impl.SetLinkage(llvm.LinkOnceODRLinkage) + thunk.impl.SetUnnamedAddr(true) + b := thunk.MakeBody(1) + ret := b.Call(target, closureWrapArgs(thunk)...) + closureWrapReturn(b, physicalSig, ret) + b.EndBuild() + b.Dispose() + return thunk +} + +func (p Package) isCoroPlainDispatchDescriptor(descriptor Expr) bool { + if descriptor.IsNil() || descriptor.kind != vkPtr || + !descriptor.impl.IsAConstantPointerNull().IsNil() { + return false + } + global := coroPlainDispatchGlobal(descriptor.impl) + if global.IsNil() || global.GlobalParent().C != p.mod.C || + !global.IsGlobalConstant() || global.Linkage() != llvm.LinkOnceODRLinkage { + return false + } + want := p.Prog.Pointer(p.Prog.coroPlainDispatchDescriptorType()) + return types.Identical(descriptor.RawType(), want.RawType()) +} + +func (p Package) matchesCoroPlainDispatchDescriptor( + descriptor Global, + thunk Function, + target llvm.Value, + physicalSig *types.Signature, + opts CoroPlainDispatchDescriptorOptions, +) bool { + if descriptor == nil || thunk == nil || + !p.isCoroPlainDispatchDescriptor(descriptor.Expr) || + thunk.impl.GlobalParent().C != p.mod.C || + thunk.impl.Linkage() != llvm.LinkOnceODRLinkage || + !types.Identical( + thunk.RawType(), + FuncAddCtx( + types.NewParam(token.NoPos, nil, closureCtx, types.Typ[types.UnsafePointer]), + physicalSig, + ), + ) || !coroPlainDispatchThunkCalls(thunk, target) { + return false + } + initializer := descriptor.impl.Initializer() + if initializer.IsAConstantStruct().IsNil() || initializer.OperandsCount() != 8 { + return false + } + wantFixed := []uint64{ + uint64(opts.Version), + uint64(opts.Flags), + binary.BigEndian.Uint64(opts.ABIHash[:8]), + binary.BigEndian.Uint64(opts.ABIHash[8:]), + } + for i, want := range wantFixed { + if initializer.Operand(i).ZExtValue() != want { + return false + } + } + plain := coroPlainDispatchFunction(initializer.Operand(4)) + if plain.IsNil() || plain.C != thunk.impl.C || + initializer.Operand(5).IsAConstantPointerNull().IsNil() { + return false + } + return initializer.Operand(6).ZExtValue() == p.Prog.SizeOf(opts.Result) && + initializer.Operand(7).ZExtValue() == p.Prog.AlignOf(opts.Result) +} + +func coroPlainDispatchThunkCalls(thunk Function, target llvm.Value) bool { + calls := 0 + for _, block := range thunk.impl.BasicBlocks() { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.InstructionOpcode() != llvm.Call { + continue + } + called := coroPlainDispatchFunction(instruction.CalledValue()) + if called.IsNil() || called.C != target.C { + return false + } + calls++ + } + } + return calls == 1 +} + +func validateCoroPlainDispatchContract(version, flags uint32) { + if version != CoroPlainDispatchVersionV1 { + panic(fmt.Sprintf( + "ssa: coroutine plain dispatch version is %d, want %d", + version, CoroPlainDispatchVersionV1, + )) + } + if flags != CoroPlainDispatchFlagsV1 { + panic(fmt.Sprintf( + "ssa: coroutine plain dispatch flags are %#x, want exact HasPlain|NoCapture (%#x)", + flags, CoroPlainDispatchFlagsV1, + )) + } +} + +func validateCoroPlainDispatchSignature(prog Program, sig *types.Signature) error { + if sig.Recv() != nil { + return fmt.Errorf("methods are not supported") + } + if sig.Variadic() { + return fmt.Errorf("variadic signatures are not supported") + } + if params := sig.TypeParams(); params != nil && params.Len() != 0 { + return fmt.Errorf("generic signatures are not supported") + } + if params := sig.RecvTypeParams(); params != nil && params.Len() != 0 { + return fmt.Errorf("generic receiver signatures are not supported") + } + return validateCoroPlainDispatchPhysicalSignature(prog, prog.PhysicalFuncDecl(sig, InGo)) +} + +func validateCoroPlainDispatchPhysicalSignature(prog Program, sig *types.Signature) error { + if sig == nil || sig.Recv() != nil || sig.Variadic() { + return fmt.Errorf("requires an ordinary non-variadic function signature") + } + if sig.Results().Len() > 1 { + return fmt.Errorf("multiple results are not supported") + } + for _, item := range []struct { + role string + tuple *types.Tuple + }{ + {"parameter", sig.Params()}, + {"result", sig.Results()}, + } { + role, tuple := item.role, item.tuple + for i := 0; i < tuple.Len(); i++ { + if !isCoroPlainDispatchScalar(prog.rawType(tuple.At(i).Type())) { + return fmt.Errorf("%s %d is not a supported scalar", role, i) + } + } + } + return nil +} + +func isCoroPlainDispatchScalar(typ Type) bool { + switch typ.ll.TypeKind() { + case llvm.IntegerTypeKind, + llvm.FloatTypeKind, + llvm.DoubleTypeKind, + llvm.X86_FP80TypeKind, + llvm.FP128TypeKind, + llvm.PPC_FP128TypeKind, + llvm.PointerTypeKind: + return true + default: + return false + } +} + +func (b Builder) coroPlainDispatchTrapIf(invalid llvm.Value) { + b.IfThen(Expr{invalid, b.Prog.Bool()}, func() { + b.impl.CreateIntrinsic( + b.Prog.Void().ll, llvm.LookupIntrinsicID("llvm.trap"), nil, "", + ) + b.Unreachable() + }) +} + +func coroPlainDispatchOr(b llvm.Builder, left, right llvm.Value) llvm.Value { + if left.IsNil() { + return right + } + return b.CreateOr(left, right, "coro.dispatch.invalid") +} + +func coroPlainDispatchFunction(value llvm.Value) llvm.Value { + for !value.IsAConstantExpr().IsNil() && value.OperandsCount() == 1 { + value = value.Operand(0) + } + return value.IsAFunction() +} + +func coroPlainDispatchGlobal(value llvm.Value) llvm.Value { + for !value.IsAConstantExpr().IsNil() && value.OperandsCount() == 1 { + value = value.Operand(0) + } + return value.IsAGlobalVariable() +} diff --git a/ssa/coro_dispatch_test.go b/ssa/coro_dispatch_test.go new file mode 100644 index 0000000000..d6ccf8ea43 --- /dev/null +++ b/ssa/coro_dispatch_test.go @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import ( + "fmt" + "go/token" + "go/types" + "regexp" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type coroPlainDispatchTestFixture struct { + prog Program + pkg Package + signature *types.Signature + result Type + hash [16]byte + descriptor Expr + descriptor2 Expr + thunkName string + thunkName2 string +} + +func TestCoroPlainDispatchV1TargetLayoutAndLowering(t *testing.T) { + Initialize(InitAll) + tests := []struct { + name string + target *Target + pointerSize int + descriptorSize uint64 + coroEntryOffset uint64 + resultSizeOffset uint64 + resultAlignOffset uint64 + }{ + { + name: "native64", + pointerSize: 8, + descriptorSize: 56, + coroEntryOffset: 32, + resultSizeOffset: 40, + resultAlignOffset: 48, + }, + { + name: "wasm32", + target: &Target{GOOS: "wasip1", GOARCH: "wasm"}, + pointerSize: 4, + descriptorSize: 40, + coroEntryOffset: 28, + resultSizeOffset: 32, + resultAlignOffset: 36, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixture := newCoroPlainDispatchTestFixture(t, test.target) + prog, pkg := fixture.prog, fixture.pkg + + if got := prog.PointerSize(); got != test.pointerSize { + t.Fatalf("pointer size = %d, want %d", got, test.pointerSize) + } + closureType := prog.Closure(fixture.signature) + if got, want := prog.SizeOf(closureType), uint64(test.pointerSize*2); got != want { + t.Fatalf("dispatch value size = %d, want two pointers (%d)", got, want) + } + + descriptor := fixture.descriptor + if descriptor.kind != vkPtr || !descriptor.impl.IsGlobalConstant() { + t.Fatalf("descriptor is not a constant global pointer: %v", descriptor.impl) + } + if got := descriptor.impl.Linkage(); got != llvm.LinkOnceODRLinkage { + t.Fatalf("descriptor linkage = %v, want linkonce_odr", got) + } + descriptorType := prog.Elem(descriptor.Type) + if got := prog.SizeOf(descriptorType); got != test.descriptorSize { + t.Fatalf("descriptor size = %d, want %d", got, test.descriptorSize) + } + if got := prog.OffsetOf(descriptorType, 4); got != 24 { + t.Fatalf("plainEntry offset = %d, want 24", got) + } + if got := prog.OffsetOf(descriptorType, 5); got != test.coroEntryOffset { + t.Fatalf("coroEntry offset = %d, want %d", got, test.coroEntryOffset) + } + if got := prog.OffsetOf(descriptorType, 6); got != test.resultSizeOffset { + t.Fatalf("resultSize offset = %d, want %d", got, test.resultSizeOffset) + } + if got := prog.OffsetOf(descriptorType, 7); got != test.resultAlignOffset { + t.Fatalf("resultAlign offset = %d, want %d", got, test.resultAlignOffset) + } + if got, want := descriptor.impl.Alignment(), int(prog.AlignOf(descriptorType)); got != want { + t.Fatalf("descriptor alignment = %d, want %d", got, want) + } + + initializer := descriptor.impl.Initializer() + if initializer.IsAConstantStruct().IsNil() || initializer.OperandsCount() != 8 { + t.Fatalf("descriptor initializer is not an eight-field constant: %v", initializer) + } + wantFixed := []uint64{ + uint64(CoroPlainDispatchVersionV1), + uint64(CoroPlainDispatchFlagsV1), + 0x0102030405060708, + 0x090a0b0c0d0e0f10, + } + for i, want := range wantFixed { + if got := initializer.Operand(i).ZExtValue(); got != want { + t.Fatalf("descriptor field %d = %#x, want %#x", i, got, want) + } + } + plain := coroPlainDispatchFunction(initializer.Operand(4)) + if plain.IsNil() || plain.Name() != fixture.thunkName { + t.Fatalf("plainEntry = %v, want target-specific thunk %q", plain, fixture.thunkName) + } + if initializer.Operand(5).IsAConstantPointerNull().IsNil() { + t.Fatalf("coroEntry is not null: %v", initializer.Operand(5)) + } + if got, want := initializer.Operand(6).ZExtValue(), prog.SizeOf(fixture.result); got != want { + t.Fatalf("resultSize = %d, want %d", got, want) + } + if got, want := initializer.Operand(7).ZExtValue(), prog.AlignOf(fixture.result); got != want { + t.Fatalf("resultAlign = %d, want %d", got, want) + } + + // Repeated materialization of the same target is idempotent. This is + // needed when multiple exact SSA producers name one planned target. + again := pkg.NewCoroPlainDispatchDescriptor( + descriptor.Name(), fixture.descriptorOptions("plain_target", fixture.thunkName), + ) + if again.impl.C != descriptor.impl.C { + t.Fatal("identical descriptor materialization did not reuse the global") + } + + // Function identity changes the symbol names, not the ABI hash. + initializer2 := fixture.descriptor2.impl.Initializer() + for i := 2; i <= 3; i++ { + if got, want := initializer2.Operand(i).ZExtValue(), initializer.Operand(i).ZExtValue(); got != want { + t.Fatalf("second target hash field %d = %#x, want ABI-only hash %#x", i, got, want) + } + } + plain2 := coroPlainDispatchFunction(initializer2.Operand(4)) + if plain2.IsNil() || plain2.Name() != fixture.thunkName2 || plain2.C == plain.C { + t.Fatalf("second target did not receive a distinct thunk: %v versus %v", plain2, plain) + } + + ir := pkg.String() + if !strings.Contains(ir, "@plain_descriptor = linkonce_odr unnamed_addr constant") { + t.Fatalf("descriptor is not an unnamed_addr linkonce_odr constant:\n%s", ir) + } + for _, thunk := range []string{fixture.thunkName, fixture.thunkName2} { + body := coroPlainDispatchIRFunction(ir, thunk) + if body == "" || !strings.Contains(body, "linkonce_odr") || + !strings.Contains(body, "(ptr ") { + t.Fatalf("missing target-specific (ctx,args) thunk %q:\n%s", thunk, ir) + } + if strings.Contains(thunk, closureStub) { + t.Fatalf("dispatch thunk reused legacy closure stub namespace: %q", thunk) + } + } + if body := coroPlainDispatchIRFunction(ir, fixture.thunkName); !strings.Contains(body, "@plain_target(") { + t.Fatalf("target-specific thunk does not directly call its one plain target:\n%s", body) + } + producerBody := coroPlainDispatchIRFunction(ir, "dispatch_value") + if !strings.Contains(producerBody, "ret { ptr, ptr } { ptr @plain_descriptor, ptr null }") { + t.Fatalf("producer did not materialize canonical {descriptor,nil} value:\n%s", producerBody) + } + + callBody := coroPlainDispatchIRFunction(ir, "dispatch_call") + if callBody == "" { + t.Fatalf("missing dispatch caller:\n%s", ir) + } + if !regexp.MustCompile(`call void @[^\n]*AssertNilDeref[^\n]*\(i1`).MatchString(callBody) { + t.Fatalf("nil function call does not use the recoverable Go nil-deref path:\n%s", callBody) + } + if got := strings.Count(callBody, "call void @llvm.trap()"); got != 2 { + t.Fatalf("ABI guards emitted %d trap sites, want env and descriptor traps:\n%s", got, callBody) + } + if got := strings.Count(callBody, "unreachable"); got != 2 { + t.Fatalf("ABI guards emitted %d unreachable terminators, want 2:\n%s", got, callBody) + } + for _, guard := range []string{ + "coro.dispatch.env.nonnull", + "coro.dispatch.field.0.invalid", + "coro.dispatch.field.1.invalid", + "coro.dispatch.field.2.invalid", + "coro.dispatch.field.3.invalid", + "coro.dispatch.plain.nil", + "coro.dispatch.coro.nonnull", + "coro.dispatch.result.size.invalid", + "coro.dispatch.result.align.invalid", + } { + if !strings.Contains(callBody, guard) { + t.Fatalf("dispatch caller is missing guard %q:\n%s", guard, callBody) + } + } + assertCall := strings.Index(callBody, "AssertNilDeref") + envBranch := strings.Index(callBody, "br i1 %coro.dispatch.env.nonnull") + descriptorLoad := strings.Index(callBody, "load { i32, i32, i64, i64, ptr, ptr") + if assertCall < 0 || envBranch < 0 || descriptorLoad < 0 || + assertCall > envBranch || envBranch > descriptorLoad { + t.Fatalf("descriptor is loaded before nil/env validation:\n%s", callBody) + } + if !regexp.MustCompile(`call i32 %[^\n]*\(ptr [^,]+, i32 `).MatchString(callBody) { + t.Fatalf("success path has no typed (env,args)->result indirect call:\n%s", callBody) + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify plain dispatch module: %v\n%s", err, ir) + } + }) + } +} + +func TestCoroPlainDispatchV1RejectsNonExactContract(t *testing.T) { + Initialize(InitAll) + fixture := newCoroPlainDispatchTestFixture(t, nil) + options := fixture.descriptorOptions("plain_target", "__llgo_stub.plain_target") + coroPlainDispatchMustPanicContains(t, "legacy closure stub", func() { + fixture.pkg.NewCoroPlainDispatchDescriptor("legacy_stub_descriptor", options) + }) + options = fixture.descriptorOptions("plain_target", "unique_thunk") + options.Flags = CoroDispatchFlagHasPlain | CoroDispatchFlagHasCoro | CoroDispatchFlagNoCapture + coroPlainDispatchMustPanicContains(t, "exact HasPlain|NoCapture", func() { + fixture.pkg.NewCoroPlainDispatchDescriptor("bad_flags_descriptor", options) + }) +} + +func newCoroPlainDispatchTestFixture(t *testing.T, target *Target) *coroPlainDispatchTestFixture { + t.Helper() + prog := NewProgram(target) + installCoroPlainDispatchTestRuntime(prog) + pkg := prog.NewPackage("corodispatch", "coro/dispatch") + t.Cleanup(func() { + pkg.Module().Dispose() + prog.Dispose() + }) + + signature := coroPlainDispatchTestSignature( + []types.Type{types.Typ[types.Uint32]}, + []types.Type{types.Typ[types.Uint32]}, + ) + result := prog.Struct(prog.Uint32()) + hash := [16]byte{ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + } + makeTarget := func(name string) Function { + fn := pkg.NewFunc(name, signature, InGo) + b := fn.MakeBody(1) + b.Return(fn.Param(0)) + b.EndBuild() + b.Dispose() + return fn + } + target1 := makeTarget("plain_target") + target2 := makeTarget("plain_target_2") + thunkName := "__llgo_coro_func_plain_v1.target1" + thunkName2 := "__llgo_coro_func_plain_v1.target2" + fixture := &coroPlainDispatchTestFixture{ + prog: prog, + pkg: pkg, + signature: signature, + result: result, + hash: hash, + thunkName: thunkName, + thunkName2: thunkName2, + } + fixture.descriptor = pkg.NewCoroPlainDispatchDescriptor( + "plain_descriptor", fixture.descriptorOptions(target1.Name(), thunkName), + ) + fixture.descriptor2 = pkg.NewCoroPlainDispatchDescriptor( + "plain_descriptor_2", fixture.descriptorOptions(target2.Name(), thunkName2), + ) + + producerSig := coroPlainDispatchTestSignature(nil, []types.Type{signature}) + producer := pkg.NewFunc("dispatch_value", producerSig, InGo) + pb := producer.MakeBody(1) + pb.Return(pb.MakeCoroPlainDispatchValue(signature, fixture.descriptor)) + pb.EndBuild() + pb.Dispose() + + callerSig := coroPlainDispatchTestSignature( + []types.Type{signature, types.Typ[types.Uint32]}, + []types.Type{types.Typ[types.Uint32]}, + ) + caller := pkg.NewFunc("dispatch_call", callerSig, InGo) + cb := caller.MakeBody(1) + ret := cb.CallCoroPlainDispatch( + caller.Param(0), []Expr{caller.Param(1)}, CoroPlainDispatchCallOptions{ + Version: CoroPlainDispatchVersionV1, + Flags: CoroPlainDispatchFlagsV1, + ABIHash: hash, + Result: result, + }, + ) + cb.Return(ret) + cb.EndBuild() + cb.Dispose() + return fixture +} + +func (f *coroPlainDispatchTestFixture) descriptorOptions( + targetName, thunkName string, +) CoroPlainDispatchDescriptorOptions { + target := f.pkg.FuncOf(targetName) + if target == nil { + panic("missing plain dispatch test target " + targetName) + } + return CoroPlainDispatchDescriptorOptions{ + Version: CoroPlainDispatchVersionV1, + Flags: CoroPlainDispatchFlagsV1, + ABIHash: f.hash, + PlainTarget: target.Expr, + Signature: f.signature, + ThunkName: thunkName, + Result: f.result, + } +} + +func installCoroPlainDispatchTestRuntime(prog Program) { + runtimePkg := types.NewPackage(PkgRuntime, "runtime") + sig := coroPlainDispatchTestSignature([]types.Type{types.Typ[types.Bool]}, nil) + runtimePkg.Scope().Insert(types.NewFunc(token.NoPos, runtimePkg, "AssertNilDeref", sig)) + runtimePkg.MarkComplete() + prog.SetRuntime(runtimePkg) +} + +func coroPlainDispatchIRFunction(ir, name string) string { + marker := "@" + name + "(" + call := strings.Index(ir, marker) + if call < 0 { + return "" + } + start := strings.LastIndex(ir[:call], "define ") + if start < 0 { + return "" + } + end := strings.Index(ir[call:], "\n}\n") + if end < 0 { + return "" + } + return ir[start : call+end+3] +} + +func coroPlainDispatchTestSignature(params, results []types.Type) *types.Signature { + tuple := func(values []types.Type) *types.Tuple { + vars := make([]*types.Var, len(values)) + for i, value := range values { + vars[i] = types.NewVar(token.NoPos, nil, "", value) + } + return types.NewTuple(vars...) + } + return types.NewSignatureType(nil, nil, nil, tuple(params), tuple(results), false) +} + +func coroPlainDispatchMustPanicContains(t *testing.T, want string, fn func()) { + t.Helper() + defer func() { + value := recover() + if value == nil { + t.Fatalf("operation did not panic; want substring %q", want) + } + if got := fmt.Sprint(value); !strings.Contains(got, want) { + t.Fatalf("panic = %q, want substring %q", got, want) + } + }() + fn() +} diff --git a/ssa/type.go b/ssa/type.go index f90f8de380..eeb1412d38 100644 --- a/ssa/type.go +++ b/ssa/type.go @@ -194,6 +194,14 @@ func (p Program) SizeOf(typ Type, n ...int64) uint64 { return size } +// AlignOf returns the target ABI alignment of typ in bytes. +func (p Program) AlignOf(typ Type) uint64 { + if typ == nil { + panic("ssa: AlignOf requires a type") + } + return uint64(p.td.ABITypeAlignment(typ.ll)) +} + // OffsetOf returns the offset of a field in a struct. func (p Program) OffsetOf(typ Type, i int) uint64 { return p.td.ElementOffset(typ.ll, i) From 188ee4443f22300f1393daa32e2b150d676dabcf Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 16:20:28 +0800 Subject: [PATCH 04/32] feat(coro): emit plain function dispatch --- cl/compilation_test.go | 21 +- cl/compile.go | 79 +++-- cl/coro_dispatch.go | 625 +++++++++++++++++++++++++++++++++++++++ cl/coro_dispatch_test.go | 284 ++++++++++++++++++ cl/coro_entry.go | 44 ++- 5 files changed, 1002 insertions(+), 51 deletions(-) create mode 100644 cl/coro_dispatch.go create mode 100644 cl/coro_dispatch_test.go diff --git a/cl/compilation_test.go b/cl/compilation_test.go index fd8ec9be98..5b0a41da75 100644 --- a/cl/compilation_test.go +++ b/cl/compilation_test.go @@ -94,23 +94,26 @@ func TestCompilationCoroABIIdentityValidation(t *testing.T) { if err := (&Compilation{EnableCoroEntryResolution: true, EnableCoroPhysicalABI: true}).validateCoroABIIdentity(false); err != nil { t.Fatalf("omitted source ABI identity should use current defaults: %v", err) } - plainDispatch := &Compilation{ - EnableCoroEntryResolution: true, - EnableCoroPlainDispatch: true, - CoroABI: coro.EntryResolutionABIV0, - SchedulerABI: coro.SchedulerNoneABIV0, - PanicABI: coro.PanicLegacyABIV0, - FuncRepABI: coro.FuncRepABIV1, + newPlainDispatch := func() *Compilation { + return &Compilation{ + EnableCoroEntryResolution: true, + EnableCoroPlainDispatch: true, + CoroABI: coro.EntryResolutionABIV0, + SchedulerABI: coro.SchedulerNoneABIV0, + PanicABI: coro.PanicLegacyABIV0, + FuncRepABI: coro.FuncRepABIV1, + } } + plainDispatch := newPlainDispatch() if err := plainDispatch.validateCoroABIIdentity(false); err != nil { t.Fatalf("complete plain-dispatch ABI identity: %v", err) } - wrongPlainDispatch := *plainDispatch + wrongPlainDispatch := newPlainDispatch() wrongPlainDispatch.FuncRepABI = coro.FuncRepABIV0 if err := wrongPlainDispatch.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "function representation ABI") { t.Fatalf("plain-dispatch function representation mismatch = %v", err) } - withoutEntry := *plainDispatch + withoutEntry := newPlainDispatch() withoutEntry.EnableCoroEntryResolution = false if err := withoutEntry.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "requires coroutine entry resolution") { t.Fatalf("plain-dispatch dependency error = %v", err) diff --git a/cl/compile.go b/cl/compile.go index c92df02ed7..82104bfe85 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -186,6 +186,7 @@ type context struct { sourceParamBase int // hidden physical parameters before source params currentCoro *coroBodyContext coroRootFactories []coroRootFactoryRegistration + coroPlainDescriptors map[string]llssa.Expr patches Patches blkInfos []blocks.Info @@ -1217,11 +1218,13 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue } switch v := iv.(type) { case *ssa.Call: - if value, handled := p.tryCompileCoroStaticAwait(b, v); handled { + if value, handled := p.tryCompileCoroPlainDispatchCall(b, v); handled { ret = value - break + } else if value, handled := p.tryCompileCoroStaticAwait(b, v); handled { + ret = value + } else { + ret = p.call(b, llssa.Call, &v.Call) } - ret = p.call(b, llssa.Call, &v.Call) if p.rangeFuncCallNeedsDeferDrain(&v.Call) { b.DeferStackDrain() } @@ -1446,7 +1449,20 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue } ret = b.MakeMap(t, nReserve) case *ssa.MakeClosure: - fn := p.compileValue(b, v.Fn) + if value, handled := p.tryCompileCoroPlainDispatchClosure(b, v); handled { + ret = value + break + } + var fn llssa.Expr + if target, ok := v.Fn.(*ssa.Function); ok && p.compilation != nil && p.compilation.EnableCoroEntryResolution { + // The target's own ValuePlan may require a descriptor at another + // producer. MakeClosure still needs the raw body entry; feeding a + // descriptor-backed closure to Builder.MakeClosure would reinterpret + // the descriptor pointer as executable code. + fn = p.compileRawFunctionValue(target) + } else { + fn = p.compileValue(b, v.Fn) + } bindings := p.compileValues(b, v.Bindings, 0) ret = b.MakeClosure(fn, bindings) case *ssa.TypeAssert: @@ -1717,29 +1733,10 @@ func (p *context) compileValue(b llssa.Builder, v ssa.Value) llssa.Expr { } } case *ssa.Function: - if p.compilation != nil && p.compilation.EnableCoroEntryResolution && p.compilation.EmissionUniverse != nil { - canonical, ok := p.compilation.EmissionUniverse.Resolve(v) - if !ok { - panic(fmt.Errorf("coroutine entry resolution: function value %q is absent from the prepared emission universe", v.Name())) - } - v = canonical + if value, handled := p.tryCompileCoroPlainDispatchFunctionValue(b, v); handled { + return value } - if _, _, ftype := p.funcName(v); ftype == llgoInstr { - if p.compilation != nil && p.compilation.EnableCoroEntryResolution && p.compilation.EmissionUniverse != nil { - wrapper, ok := p.compilation.EmissionUniverse.intrinsicWrapper(p.goPkg, v) - if !ok { - panic(fmt.Errorf("coroutine entry resolution: intrinsic function value %q was not materialized before codegen", v.Name())) - } - v = wrapper - } else { - v = ssawrap.MakeCallWrapper(p.goProg, v) - } - } - aFn, pyFn, _ := p.compileFunction(v) - if aFn != nil { - return aFn.Expr - } - return pyFn.Expr + return p.compileRawFunctionValue(v) case *ssa.Global: varName := v.Name() val := p.varOf(b, v) @@ -2110,6 +2107,36 @@ func (p *context) observeCoroPlan() { } } +// compileRawFunctionValue returns the selected body entry without applying a +// function-value representation conversion. Static calls and MakeClosure use +// this path even when a different exact producer for the same SSA target is +// descriptor-backed. +func (p *context) compileRawFunctionValue(v *ssa.Function) llssa.Expr { + if p.compilation != nil && p.compilation.EnableCoroEntryResolution && p.compilation.EmissionUniverse != nil { + canonical, ok := p.compilation.EmissionUniverse.Resolve(v) + if !ok { + panic(fmt.Errorf("coroutine entry resolution: function value %q is absent from the prepared emission universe", v.Name())) + } + v = canonical + } + if _, _, ftype := p.funcName(v); ftype == llgoInstr { + if p.compilation != nil && p.compilation.EnableCoroEntryResolution && p.compilation.EmissionUniverse != nil { + wrapper, ok := p.compilation.EmissionUniverse.intrinsicWrapper(p.goPkg, v) + if !ok { + panic(fmt.Errorf("coroutine entry resolution: intrinsic function value %q was not materialized before codegen", v.Name())) + } + v = wrapper + } else { + v = ssawrap.MakeCallWrapper(p.goProg, v) + } + } + aFn, pyFn, _ := p.compileFunction(v) + if aFn != nil { + return aFn.Expr + } + return pyFn.Expr +} + func initFnNameOfHasPatch(name string) string { return name + "$hasPatch" } diff --git a/cl/coro_dispatch.go b/cl/coro_dispatch.go new file mode 100644 index 0000000000..385312dcff --- /dev/null +++ b/cl/coro_dispatch.go @@ -0,0 +1,625 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "go/token" + "go/types" + "strconv" + "strings" + + "github.com/goplus/llgo/internal/coro" + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +const ( + coroPlainDispatchVersion = llssa.CoroPlainDispatchVersionV1 + coroPlainDispatchFlags = llssa.CoroPlainDispatchFlagsV1 + coroPlainDispatchDescriptorPrefix = "__llgo_coro_func_descriptor_v1." + coroPlainDispatchThunkPrefix = "__llgo_coro_func_plain_v1." +) + +// coroPlainDispatchABI is deliberately target independent of the selected +// function body. Every function with the same canonical callable ABI receives +// the same hash, while its FunctionID digest is used only to make the descriptor +// and thunk symbols target-specific. +type coroPlainDispatchABI struct { + hash [16]byte + signature *types.Signature + resultSlotType types.Type +} + +func validateCoroPlainDispatchTarget(fn *ssa.Function, plan coro.FunctionPlan) error { + fail := func(format string, args ...any) error { + return fmt.Errorf("coroutine plain dispatch ABI: function %q: %s", plan.ID, fmt.Sprintf(format, args...)) + } + if fn == nil || plan.External != coro.Defined || len(fn.Blocks) == 0 { + return fail("requires one defined SSA body") + } + if plan.Emission != coro.EmitPlain || plan.Primary != coro.PrimaryPlain || plan.FuncRep != coro.Dispatch { + return fail("requires plain descriptor emission, got emission=%s primary=%s representation=%s", plan.Emission, plan.Primary, plan.FuncRep) + } + if plan.Effect != coro.NoSuspend || plan.Effect.IsOpaque() { + return fail("requires an exact non-suspending effect, got %s", plan.Effect) + } + if plan.Exec.Contains(coro.NeedsPreempt) || plan.Exec.IsOpaque() { + return fail("execution flags %s require coroutine or open dispatch lowering", plan.Exec) + } + if len(fn.FreeVars) != 0 { + return fail("captured closures require an environment descriptor") + } + if fn.Signature == nil || fn.Signature.Recv() != nil { + return fail("methods require receiver-aware dispatch lowering") + } + if fn.Signature.Variadic() { + return fail("variadic dispatch is not implemented") + } + if directive := coroLeafABIDirective(fn); directive != "" { + return fail("ABI directive %q requires an explicit boundary adapter", directive) + } + if isCgoExternSymbol(fn) { + return fail("cgo entry requires a foreign adapter") + } + if fn.Synthetic != "" { + return fail("synthetic function %q is outside the plain dispatch ABI", fn.Synthetic) + } + if params := fn.TypeParams(); params != nil && params.Len() != 0 { + return fail("generic declarations are not materialized dispatch bodies") + } + if len(fn.TypeArgs()) != 0 || fn.Origin() != nil { + return fail("generic instances require a frozen instantiated dispatch ABI") + } + if path, ok := nestedFunctionTypePath(fn.Signature); ok { + return fail("nested function type at %s requires recursive function-representation lowering", path) + } + if err := validateCoroPlainDispatchSignatureShape(fn.Signature); err != nil { + return fail("signature: %v", err) + } + return nil +} + +func validateCoroPlainDispatchSignatureShape(sig *types.Signature) error { + if sig == nil { + return fmt.Errorf("missing signature") + } + if sig.Results().Len() > 1 { + return fmt.Errorf("multiple results are not implemented") + } + for _, item := range []struct { + role string + tuple *types.Tuple + }{ + {"parameter", sig.Params()}, + {"result", sig.Results()}, + } { + for i := 0; i < item.tuple.Len(); i++ { + if !coroPlainDispatchSourceScalar(item.tuple.At(i).Type()) { + return fmt.Errorf("%s %d type %s is not a supported scalar", item.role, i, item.tuple.At(i).Type()) + } + } + } + return nil +} + +func coroPlainDispatchSourceScalar(typ types.Type) bool { + typ = types.Unalias(typ) + if named, ok := typ.(*types.Named); ok { + return coroPlainDispatchSourceScalar(named.Underlying()) + } + switch value := typ.Underlying().(type) { + case *types.Basic: + info := value.Info() + return value.Kind() == types.UnsafePointer || info&(types.IsBoolean|types.IsInteger|types.IsFloat) != 0 + case *types.Pointer, *types.Map, *types.Chan: + return true + default: + return false + } +} + +func validateCoroPlainDispatchConsumers(plan *coro.SSAPlan) error { + if plan == nil { + return fmt.Errorf("coroutine plain dispatch ABI requires a compilation plan") + } + for _, function := range plan.Functions() { + if function.Plan.Emission != coro.EmitPlain && function.Plan.Emission != coro.EmitCoroutine { + continue + } + fn := function.Function + for _, param := range fn.Params { + if err := validateCoroPlainDispatchValue(plan, fn, param); err != nil { + return err + } + } + for _, free := range fn.FreeVars { + if err := validateCoroPlainDispatchValue(plan, fn, free); err != nil { + return err + } + } + for _, block := range fn.Blocks { + for _, instr := range block.Instrs { + if value, ok := instr.(ssa.Value); ok { + if err := validateCoroPlainDispatchValue(plan, fn, value); err != nil { + return err + } + } + for _, operand := range instr.Operands(nil) { + if operand != nil && *operand != nil { + if err := validateCoroPlainDispatchValue(plan, fn, *operand); err != nil { + return err + } + } + } + if boxed, ok := instr.(*ssa.MakeInterface); ok { + if valuePlan, found := plan.ValuePlan(boxed.X); found && funcRepMapContains(valuePlan.Funcs, coro.Dispatch) { + return coroPlainDispatchInstructionError(fn, instr, "interface boxing of a descriptor-backed function value is not implemented") + } + } + call, ok := instr.(ssa.CallInstruction) + if !ok || plan.ElidesCall(call) { + continue + } + common := call.Common() + if common != nil { + if _, builtin := common.Value.(*ssa.Builtin); builtin { + continue + } + } + callPlan, found := plan.CallPlan(call) + if !found { + return coroPlainDispatchInstructionError(fn, instr, "call has no compilation CallPlan") + } + if callPlan.Rep != coro.Dispatch { + continue + } + if err := validateCoroPlainDispatchCall(plan, fn, call, callPlan); err != nil { + return err + } + } + } + } + return nil +} + +func validateCoroPlainDispatchValue(plan *coro.SSAPlan, owner *ssa.Function, value ssa.Value) error { + valuePlan, found := plan.ValuePlan(value) + if !found || !funcRepMapContains(valuePlan.Funcs, coro.Dispatch) { + return nil + } + if len(valuePlan.Funcs) != 1 || len(valuePlan.Funcs[0].Path) != 0 { + // Aggregate storage does not change the physical width of a function + // leaf: both direct and descriptor-backed values remain two pointers. + // Every aggregate leaf is canonical Dispatch, while exact scalar + // producers and consumers are validated separately. Interface boxing is + // still rejected at its instruction boundary below. + for _, leaf := range valuePlan.Funcs { + if leaf.Rep != coro.Dispatch { + return fmt.Errorf("coroutine plain dispatch ABI: function %q: aggregate value %q has non-Dispatch function leaf", owner.Name(), value.Name()) + } + } + return nil + } + leaf := valuePlan.Funcs[0] + if leaf.Rep != coro.Dispatch { + return fmt.Errorf("coroutine plain dispatch ABI: function %q: value %q has a mixed function representation", owner.Name(), value.Name()) + } + if _, ok := types.Unalias(value.Type()).Underlying().(*types.Signature); !ok { + return fmt.Errorf("coroutine plain dispatch ABI: function %q: value %q is not a scalar function value", owner.Name(), value.Name()) + } + if len(leaf.Targets) > 1 { + return fmt.Errorf("coroutine plain dispatch ABI: function %q: value %q has %d targets; multi-target dispatch is not implemented", owner.Name(), value.Name(), len(leaf.Targets)) + } + if len(leaf.Targets) == 0 { + if !leaf.MayBeNil { + return fmt.Errorf("coroutine plain dispatch ABI: function %q: value %q has no target and is not nil", owner.Name(), value.Name()) + } + return nil + } + target, targetPlan, err := coroPlainDispatchPlanTarget(plan, leaf.Targets[0]) + if err != nil { + return fmt.Errorf("coroutine plain dispatch ABI: function %q: value %q: %w", owner.Name(), value.Name(), err) + } + return validateCoroPlainDispatchTarget(target, targetPlan) +} + +func validateCoroPlainDispatchCall(plan *coro.SSAPlan, owner *ssa.Function, call ssa.CallInstruction, callPlan coro.SSACallPlan) error { + fail := func(format string, args ...any) error { + return coroPlainDispatchInstructionError(owner, call, fmt.Sprintf(format, args...)) + } + direct, ordinary := call.(*ssa.Call) + if !ordinary || direct == nil || callPlan.Kind != coro.CallDirect { + return fail("descriptor dispatch is supported only for an ordinary direct call instruction") + } + common := direct.Common() + if common == nil || common.StaticCallee() != nil || common.IsInvoke() || common.Method != nil { + return fail("descriptor dispatch requires an ordinary dynamic function call") + } + if callPlan.Open || callPlan.Unresolved == coro.UnknownForeign { + return fail("open or foreign descriptor dispatch is not implemented") + } + if len(callPlan.Targets) > 1 { + return fail("multi-target descriptor dispatch is not implemented") + } + if len(callPlan.Targets) == 0 { + if !callPlan.MayBeNil { + return fail("closed descriptor call has no target and is not nil") + } + } else { + targetFn, targetPlan, err := coroPlainDispatchPlanTarget(plan, callPlan.Targets[0]) + if err != nil { + return fail("%v", err) + } + if err := validateCoroPlainDispatchTarget(targetFn, targetPlan); err != nil { + return fail("%v", err) + } + if !types.Identical(common.Signature(), targetFn.Signature) { + return fail("call signature %s does not match target %q signature %s", common.Signature(), targetPlan.ID, targetFn.Signature) + } + } + valuePlan, found := plan.ValuePlan(common.Value) + if !found || len(valuePlan.Funcs) != 1 || len(valuePlan.Funcs[0].Path) != 0 || valuePlan.Funcs[0].Rep != coro.Dispatch { + return fail("callee has no exact scalar Dispatch ValuePlan") + } + leaf := valuePlan.Funcs[0] + if len(leaf.Targets) != len(callPlan.Targets) { + return fail("callee target count %d conflicts with CallPlan target count %d", len(leaf.Targets), len(callPlan.Targets)) + } + for i := range leaf.Targets { + if leaf.Targets[i] != callPlan.Targets[i] { + return fail("callee target %q conflicts with CallPlan target %q", leaf.Targets[i], callPlan.Targets[i]) + } + } + if leaf.MayBeNil != callPlan.MayBeNil { + return fail("callee nilability %t conflicts with CallPlan nilability %t", leaf.MayBeNil, callPlan.MayBeNil) + } + return nil +} + +func funcRepMapContains(reps coro.FuncRepMap, want coro.FuncRep) bool { + for _, leaf := range reps { + if leaf.Rep == want { + return true + } + } + return false +} + +func coroPlainDispatchPlanTarget(plan *coro.SSAPlan, id coro.FunctionID) (*ssa.Function, coro.FunctionPlan, error) { + target, found := plan.Function(id) + if !found || target == nil { + return nil, coro.FunctionPlan{}, fmt.Errorf("target %q is absent from the compilation plan", id) + } + targetPlan, found := plan.FunctionPlan(target) + if !found || targetPlan.ID != id { + return nil, coro.FunctionPlan{}, fmt.Errorf("target %q has no canonical function plan", id) + } + return target, targetPlan, nil +} + +func coroPlainDispatchInstructionError(fn *ssa.Function, instr ssa.Instruction, reason string) error { + position := token.Position{} + if fn != nil && fn.Prog != nil && fn.Prog.Fset != nil && instr != nil { + position = fn.Prog.Fset.Position(instr.Pos()) + } + return fmt.Errorf("coroutine plain dispatch ABI: function %q at %s: %s", fn.Name(), position, reason) +} + +func nestedFunctionTypePath(typ types.Type) (string, bool) { + seen := make(map[types.Type]bool) + var visit func(types.Type, string, bool) (string, bool) + visit = func(typ types.Type, path string, root bool) (string, bool) { + if typ == nil { + return "", false + } + typ = types.Unalias(typ) + if seen[typ] { + return "", false + } + seen[typ] = true + switch value := typ.(type) { + case *types.Signature: + if !root { + return path, true + } + for i := 0; i < value.Params().Len(); i++ { + if found, ok := visit(value.Params().At(i).Type(), fmt.Sprintf("param[%d]", i), false); ok { + return found, true + } + } + for i := 0; i < value.Results().Len(); i++ { + if found, ok := visit(value.Results().At(i).Type(), fmt.Sprintf("result[%d]", i), false); ok { + return found, true + } + } + case *types.Named: + return visit(value.Underlying(), path+".underlying", false) + case *types.Pointer: + // Pointer identity is part of the canonical logical signature, while + // its physical layout terminates at one opaque pointer. + return "", false + case *types.Array: + return visit(value.Elem(), path+".elem", false) + case *types.Slice: + return visit(value.Elem(), path+".elem", false) + case *types.Map: + if found, ok := visit(value.Key(), path+".key", false); ok { + return found, true + } + return visit(value.Elem(), path+".elem", false) + case *types.Chan: + return visit(value.Elem(), path+".elem", false) + case *types.Struct: + for i := 0; i < value.NumFields(); i++ { + if found, ok := visit(value.Field(i).Type(), fmt.Sprintf("%s.field[%d]", path, i), false); ok { + return found, true + } + } + case *types.Interface: + for i := 0; i < value.NumExplicitMethods(); i++ { + if found, ok := visit(value.ExplicitMethod(i).Type(), fmt.Sprintf("%s.method[%d]", path, i), false); ok { + return found, true + } + } + } + return "", false + } + return visit(typ, "signature", true) +} + +func newCoroPlainDispatchABI(p *context, signature *types.Signature) (coroPlainDispatchABI, error) { + if p == nil || p.prog == nil || signature == nil { + return coroPlainDispatchABI{}, fmt.Errorf("coroutine plain dispatch ABI requires a program and signature") + } + if path, ok := nestedFunctionTypePath(signature); ok { + return coroPlainDispatchABI{}, fmt.Errorf("nested function type at %s is unsupported", path) + } + patched, ok := p.patchType(signature).(*types.Signature) + if !ok { + return coroPlainDispatchABI{}, fmt.Errorf("patched dispatch signature is %T", p.patchType(signature)) + } + patched = canonicalCoroPlainDispatchSignature(patched) + physical := p.prog.PhysicalFuncDecl(patched, llssa.InGo) + resultFields := make([]*types.Var, physical.Results().Len()) + for i := range resultFields { + resultFields[i] = types.NewField(token.NoPos, nil, fmt.Sprintf("r%d", i), physical.Results().At(i).Type(), false) + } + resultSlot := types.NewStruct(resultFields, nil) + + qualified := func(pkg *types.Package) string { + if pkg == nil { + return "" + } + return llssa.PathOf(pkg) + } + var key strings.Builder + writeDispatchHashField(&key, "domain", "llgo.coro.func-dispatch.v1") + writeDispatchHashField(&key, "version", strconv.FormatUint(uint64(coroPlainDispatchVersion), 10)) + writeDispatchHashField(&key, "flags", strconv.FormatUint(uint64(coroPlainDispatchFlags), 10)) + writeDispatchHashField(&key, "closure", "two-pointer:descriptor,env;entry=(env,args)->results;env=nil") + writeDispatchHashField(&key, "panic", activeCompilationABI(p.compilation, func(c *Compilation) string { return c.PanicABI }, coro.PanicLegacyABIV0)) + writeDispatchHashField(&key, "func-rep", activeCompilationABI(p.compilation, func(c *Compilation) string { return c.FuncRepABI }, coro.FuncRepABIV1)) + target := p.prog.TargetSpec() + writeDispatchHashField(&key, "triple", target.Triple) + writeDispatchHashField(&key, "cpu", target.CPU) + writeDispatchHashField(&key, "features", target.Features) + writeDispatchHashField(&key, "target-abi", target.TargetABI) + writeDispatchHashField(&key, "data-layout", p.prog.DataLayout()) + writeDispatchHashField(&key, "pointer-bytes", strconv.Itoa(p.prog.PointerSize())) + writeDispatchHashField(&key, "byte-order", strconv.Itoa(int(p.prog.TargetData().ByteOrder()))) + writeDispatchHashField(&key, "logical-signature", types.TypeString(patched, qualified)) + writeDispatchHashField(&key, "physical-signature", types.TypeString(physical, qualified)) + if err := appendCoroPlainDispatchTupleLayout(&key, p.prog, "params", physical.Params(), qualified); err != nil { + return coroPlainDispatchABI{}, err + } + if err := appendCoroPlainDispatchTupleLayout(&key, p.prog, "results", physical.Results(), qualified); err != nil { + return coroPlainDispatchABI{}, err + } + if err := appendCoroPlainDispatchTypeLayout(&key, p.prog, "result-slot", resultSlot, qualified, make(map[types.Type]bool)); err != nil { + return coroPlainDispatchABI{}, err + } + sum := sha256.Sum256([]byte(key.String())) + var hash [16]byte + copy(hash[:], sum[:len(hash)]) + return coroPlainDispatchABI{hash: hash, signature: patched, resultSlotType: resultSlot}, nil +} + +// canonicalCoroPlainDispatchSignature removes source parameter/result names. +// go/types identity ignores those names, and a target declaration commonly has +// them while a function-typed parameter at the exact dynamic call does not. +// Letting names enter the descriptor hash would make two ABI-identical sites +// disagree at runtime. +func canonicalCoroPlainDispatchSignature(sig *types.Signature) *types.Signature { + params := make([]*types.Var, sig.Params().Len()) + for i := range params { + params[i] = types.NewParam(token.NoPos, nil, "", sig.Params().At(i).Type()) + } + results := make([]*types.Var, sig.Results().Len()) + for i := range results { + results[i] = types.NewParam(token.NoPos, nil, "", sig.Results().At(i).Type()) + } + return types.NewSignatureType(nil, nil, nil, types.NewTuple(params...), types.NewTuple(results...), false) +} + +func activeCompilationABI(c *Compilation, value func(*Compilation) string, fallback string) string { + if c != nil { + if current := value(c); current != "" { + return current + } + } + return fallback +} + +func writeDispatchHashField(builder *strings.Builder, name, value string) { + builder.WriteString(strconv.Itoa(len(name))) + builder.WriteByte(':') + builder.WriteString(name) + builder.WriteByte('=') + builder.WriteString(strconv.Itoa(len(value))) + builder.WriteByte(':') + builder.WriteString(value) + builder.WriteByte('\n') +} + +func appendCoroPlainDispatchTupleLayout(builder *strings.Builder, prog llssa.Program, path string, tuple *types.Tuple, qualified types.Qualifier) error { + writeDispatchHashField(builder, path+".count", strconv.Itoa(tuple.Len())) + for i := 0; i < tuple.Len(); i++ { + if err := appendCoroPlainDispatchTypeLayout(builder, prog, fmt.Sprintf("%s[%d]", path, i), tuple.At(i).Type(), qualified, make(map[types.Type]bool)); err != nil { + return err + } + } + return nil +} + +func appendCoroPlainDispatchTypeLayout(builder *strings.Builder, prog llssa.Program, path string, typ types.Type, qualified types.Qualifier, visiting map[types.Type]bool) error { + if typ == nil { + return fmt.Errorf("coroutine plain dispatch ABI: nil type at %s", path) + } + typ = types.Unalias(typ) + writeDispatchHashField(builder, path+".type", types.TypeString(typ, qualified)) + physical := prog.Type(typ, llssa.InC) + writeDispatchHashField(builder, path+".size", strconv.FormatUint(prog.SizeOf(physical), 10)) + writeDispatchHashField(builder, path+".align", strconv.FormatUint(prog.AlignOf(physical), 10)) + if visiting[typ] { + writeDispatchHashField(builder, path+".cycle", "true") + return nil + } + visiting[typ] = true + defer delete(visiting, typ) + switch value := typ.(type) { + case *types.Named: + return appendCoroPlainDispatchTypeLayout(builder, prog, path+".underlying", value.Underlying(), qualified, visiting) + case *types.Pointer: + writeDispatchHashField(builder, path+".pointer", "opaque") + case *types.Struct: + writeDispatchHashField(builder, path+".fields", strconv.Itoa(value.NumFields())) + for i := 0; i < value.NumFields(); i++ { + writeDispatchHashField(builder, fmt.Sprintf("%s.field[%d].offset", path, i), strconv.FormatUint(prog.OffsetOf(physical, i), 10)) + if err := appendCoroPlainDispatchTypeLayout(builder, prog, fmt.Sprintf("%s.field[%d]", path, i), value.Field(i).Type(), qualified, visiting); err != nil { + return err + } + } + case *types.Array: + writeDispatchHashField(builder, path+".length", strconv.FormatInt(value.Len(), 10)) + return appendCoroPlainDispatchTypeLayout(builder, prog, path+".element", value.Elem(), qualified, visiting) + case *types.Signature: + return fmt.Errorf("coroutine plain dispatch ABI: nested signature at %s", path) + } + return nil +} + +func (p *context) tryCompileCoroPlainDispatchFunctionValue(b llssa.Builder, value *ssa.Function) (llssa.Expr, bool) { + if p.compilation == nil || !p.compilation.EnableCoroPlainDispatch || p.compilation.CoroPlan == nil { + return llssa.Expr{}, false + } + valuePlan, found := p.compilation.CoroPlan.ValuePlan(value) + if !found || len(valuePlan.Funcs) != 1 || len(valuePlan.Funcs[0].Path) != 0 || valuePlan.Funcs[0].Rep != coro.Dispatch { + return llssa.Expr{}, false + } + return p.emitCoroPlainDispatchValue(b, value, valuePlan.Funcs[0]), true +} + +func (p *context) tryCompileCoroPlainDispatchClosure(b llssa.Builder, closure *ssa.MakeClosure) (llssa.Expr, bool) { + if p.compilation == nil || !p.compilation.EnableCoroPlainDispatch || p.compilation.CoroPlan == nil { + return llssa.Expr{}, false + } + valuePlan, found := p.compilation.CoroPlan.ValuePlan(closure) + if !found || len(valuePlan.Funcs) != 1 || len(valuePlan.Funcs[0].Path) != 0 || valuePlan.Funcs[0].Rep != coro.Dispatch { + return llssa.Expr{}, false + } + target, ok := closure.Fn.(*ssa.Function) + if !ok || len(closure.Bindings) != 0 || len(target.FreeVars) != 0 { + panic(fmt.Errorf("coroutine plain dispatch ABI: closure %q requires an unsupported captured or non-function producer", closure.Name())) + } + return p.emitCoroPlainDispatchValue(b, target, valuePlan.Funcs[0]), true +} + +func (p *context) emitCoroPlainDispatchValue(b llssa.Builder, target *ssa.Function, leaf coro.FuncRepLeaf) llssa.Expr { + if len(leaf.Targets) != 1 { + panic(fmt.Errorf("coroutine plain dispatch ABI: producer %q requires one target, got %d", target.Name(), len(leaf.Targets))) + } + entry := p.mustFunctionSymbol(target) + if entry.plan.ID != leaf.Targets[0] { + panic(fmt.Errorf("coroutine plain dispatch ABI: producer %q target %q conflicts with plan %q", target.Name(), leaf.Targets[0], entry.plan.ID)) + } + if err := validateCoroPlainDispatchTarget(entry.function, entry.plan); err != nil { + panic(err) + } + abi, err := newCoroPlainDispatchABI(p, entry.function.Signature) + if err != nil { + panic(err) + } + plain, py, ftype := p.compileFunction(entry.function) + if ftype != goFunc || plain == nil || py != nil { + panic(fmt.Errorf("coroutine plain dispatch ABI: target %q did not compile as one Go function", entry.plan.ID)) + } + targetHash := sha256.Sum256([]byte(entry.plan.ID)) + targetKey := hex.EncodeToString(targetHash[:8]) + "." + hex.EncodeToString(abi.hash[:]) + result := p.prog.Type(abi.resultSlotType, llssa.InC) + descriptorName := coroPlainDispatchDescriptorPrefix + targetKey + descriptor, found := p.coroPlainDescriptors[descriptorName] + if !found { + descriptor = p.pkg.NewCoroPlainDispatchDescriptor( + descriptorName, + llssa.CoroPlainDispatchDescriptorOptions{ + Version: coroPlainDispatchVersion, + Flags: coroPlainDispatchFlags, + ABIHash: abi.hash, + PlainTarget: plain.Expr, + Signature: abi.signature, + ThunkName: coroPlainDispatchThunkPrefix + targetKey, + Result: result, + }, + ) + if p.coroPlainDescriptors == nil { + p.coroPlainDescriptors = make(map[string]llssa.Expr) + } + p.coroPlainDescriptors[descriptorName] = descriptor + } + return b.MakeCoroPlainDispatchValue(abi.signature, descriptor) +} + +func (p *context) tryCompileCoroPlainDispatchCall(b llssa.Builder, call *ssa.Call) (llssa.Expr, bool) { + if p.compilation == nil || !p.compilation.EnableCoroPlainDispatch || p.compilation.CoroPlan == nil || call == nil { + return llssa.Expr{}, false + } + callPlan, found := p.compilation.CoroPlan.CallPlan(call) + if !found || callPlan.Rep != coro.Dispatch { + return llssa.Expr{}, false + } + if err := validateCoroPlainDispatchCall(p.compilation.CoroPlan, call.Parent(), call, callPlan); err != nil { + panic(err) + } + p.recordCallerLocationForCall(b, &call.Call) + p.emitPCLineLabel(b, call.Pos()) + fn := p.compileValue(b, call.Call.Value) + args := p.compileValues(b, call.Call.Args, fnNormal) + abi, err := newCoroPlainDispatchABI(p, call.Call.Signature()) + if err != nil { + panic(err) + } + result := p.prog.Type(abi.resultSlotType, llssa.InC) + return b.CallCoroPlainDispatch(fn, args, llssa.CoroPlainDispatchCallOptions{ + Version: coroPlainDispatchVersion, + Flags: coroPlainDispatchFlags, + ABIHash: abi.hash, + Result: result, + }), true +} diff --git a/cl/coro_dispatch_test.go b/cl/coro_dispatch_test.go new file mode 100644 index 0000000000..c5ecb5758c --- /dev/null +++ b/cl/coro_dispatch_test.go @@ -0,0 +1,284 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "strings" + "testing" + + "github.com/goplus/llgo/internal/coro" + "github.com/goplus/llgo/internal/goembed" + "github.com/xgo-dev/llvm" + "golang.org/x/tools/go/ssa" +) + +func TestCoroPlainDispatchCompilesClosedSingletonFunctionValue(t *testing.T) { + const source = `package foo + +func Target(value int) int { return value + 1 } + +func Apply(fn func(int) int, value int) int { + if fn == nil { + return 0 + } + return fn(value) +} + +func Root() int { return Apply(Target, 41) } +` + ssaPkg, _, files := buildGoSSAPkg(t, source) + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + t.Fatal(err) + } + target := ssaPkg.Func("Target") + apply := ssaPkg.Func("Apply") + dynamicCall := coroPlainDispatchOnlyDynamicCall(t, apply) + hashContext := &context{ + prog: prog, + goProg: ssaPkg.Prog, + goTyps: ssaPkg.Pkg, + goPkg: ssaPkg, + emissionUniverse: universe, + } + targetABI, err := newCoroPlainDispatchABI(hashContext, target.Signature) + if err != nil { + t.Fatal(err) + } + callABI, err := newCoroPlainDispatchABI(hashContext, dynamicCall.Common().Signature()) + if err != nil { + t.Fatal(err) + } + if targetABI.hash != callABI.hash { + t.Fatalf("target ABI hash %x differs from name-less call signature hash %x", targetABI.hash, callABI.hash) + } + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.EntryResolutionABIV0 + functionIDs.SchedulerABI = coro.SchedulerNoneABIV0 + functionIDs.ArchiveReady = true + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: ssaPkg.Func("Root"), Demand: coro.SyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyClosedDynamicCall: func(_ *ssa.Function, call ssa.CallInstruction) (coro.SSAClosedDynamicCallCertificate, bool, error) { + if call != dynamicCall { + return coro.SSAClosedDynamicCallCertificate{}, false, nil + } + return coro.SSAClosedDynamicCallCertificate{Targets: []*ssa.Function{target}, MayBeNil: true}, true, nil + }, + }) + if err != nil { + t.Fatal(err) + } + targetPlan, ok := plan.FunctionPlan(target) + if !ok || targetPlan.FuncRep != coro.Dispatch || targetPlan.Emission != coro.EmitPlain || targetPlan.Primary != coro.PrimaryPlain || targetPlan.Effect != coro.NoSuspend { + t.Fatalf("Target plan = %+v, present=%t; want one descriptor-backed plain body", targetPlan, ok) + } + callPlan, ok := plan.CallPlan(dynamicCall) + if !ok || callPlan.Rep != coro.Dispatch || callPlan.Open || !callPlan.MayBeNil || len(callPlan.Targets) != 1 || callPlan.Targets[0] != targetPlan.ID { + t.Fatalf("Apply dynamic CallPlan = %+v, present=%t; want closed nullable singleton Dispatch", callPlan, ok) + } + + compiled, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: &Compilation{ + CoroPlan: plan, + EmissionUniverse: universe, + EnableCoroEntryResolution: true, + EnableCoroPlainDispatch: true, + CoroABI: coro.EntryResolutionABIV0, + SchedulerABI: coro.SchedulerNoneABIV0, + PanicABI: coro.PanicLegacyABIV0, + FuncRepABI: coro.FuncRepABIV1, + }}, + ) + if err != nil { + t.Fatalf("compile plain dispatch package: %v", err) + } + module := compiled.Module() + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify plain dispatch module: %v\n%s", err, module.String()) + } + ir := module.String() + for _, marker := range []string{ + coroPlainDispatchDescriptorPrefix, + coroPlainDispatchThunkPrefix, + "llvm.trap", + "AssertNilDeref", + "coro.dispatch.result.size.invalid", + "coro.dispatch.result.align.invalid", + } { + if !strings.Contains(ir, marker) { + t.Fatalf("plain dispatch IR is missing %q:\n%s", marker, ir) + } + } + if strings.Contains(ir, coroPrimarySuffix) { + t.Fatalf("plain descriptor unexpectedly emitted a second coroutine body:\n%s", ir) + } + if got := strings.Count(ir, "define i64 @foo.Target("); got != 1 { + t.Fatalf("Target plain body definitions = %d, want exactly one:\n%s", got, ir) + } +} + +func TestCoroPlainDispatchGateAndTargetShapeFailClosed(t *testing.T) { + pkg, plan := buildCoroEntryTestPlan(t) + boxedPlan, ok := plan.FunctionPlan(pkg.Func("Boxed")) + if !ok || boxedPlan.FuncRep != coro.Dispatch { + t.Fatalf("Boxed plan = %+v, present=%t", boxedPlan, ok) + } + entry := plannedFunctionSymbol{function: pkg.Func("Boxed"), plan: boxedPlan, planned: true, coroPlan: plan} + if err := entry.checkSupported(); err == nil || !strings.Contains(err.Error(), "unimplemented dispatch descriptor") { + t.Fatalf("gate-off dispatch error = %v", err) + } + entry.plainDispatch = true + if err := entry.checkSupported(); err != nil { + t.Fatalf("gate-on plain target rejected: %v", err) + } + + badSignatures := []struct { + name string + src string + want string + }{ + {"multiple results", "func Bad() (int, int) { return 1, 2 }", "multiple results"}, + {"aggregate parameter", "func Bad(value string) { _ = value }", "not a supported scalar"}, + {"variadic", "func Bad(values ...int) { _ = values }", "variadic"}, + {"nested function", "func Bad(value func()) { _ = value }", "nested function type"}, + } + for _, test := range badSignatures { + t.Run(test.name, func(t *testing.T) { + ssaPkg, _, _ := buildGoSSAPkg(t, "package foo\n"+test.src) + fn := ssaPkg.Func("Bad") + plan := coro.FunctionPlan{ + ID: "bad", + Effect: coro.NoSuspend, + Emission: coro.EmitPlain, + FuncRep: coro.Dispatch, + External: coro.Defined, + Primary: coro.PrimaryPlain, + } + err := validateCoroPlainDispatchTarget(fn, plan) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("target validation error = %v, want substring %q", err, test.want) + } + }) + } +} + +func TestCoroPlainDispatchCompilesZeroBindingClosure(t *testing.T) { + const source = `package foo + +func Apply(fn func(int) int, value int) int { return fn(value) } + +func Root() int { + fn := func(value int) int { return value + 2 } + return Apply(fn, 40) +} +` + ssaPkg, _, files := buildGoSSAPkg(t, source) + root := ssaPkg.Func("Root") + if len(root.AnonFuncs) != 1 || len(root.AnonFuncs[0].FreeVars) != 0 { + t.Fatalf("Root anonymous functions = %+v, want one zero-binding closure", root.AnonFuncs) + } + target := root.AnonFuncs[0] + apply := ssaPkg.Func("Apply") + dynamicCall := coroPlainDispatchOnlyDynamicCall(t, apply) + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + t.Fatal(err) + } + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.EntryResolutionABIV0 + functionIDs.SchedulerABI = coro.SchedulerNoneABIV0 + functionIDs.ArchiveReady = true + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: root, Demand: coro.SyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyClosedDynamicCall: func(_ *ssa.Function, call ssa.CallInstruction) (coro.SSAClosedDynamicCallCertificate, bool, error) { + if call == dynamicCall { + return coro.SSAClosedDynamicCallCertificate{Targets: []*ssa.Function{target}}, true, nil + } + return coro.SSAClosedDynamicCallCertificate{}, false, nil + }, + }) + if err != nil { + t.Fatal(err) + } + targetPlan, ok := plan.FunctionPlan(target) + if !ok || targetPlan.FuncRep != coro.Dispatch || targetPlan.Emission != coro.EmitPlain { + t.Fatalf("zero-binding target plan = %+v, present=%t", targetPlan, ok) + } + compiled, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: &Compilation{ + CoroPlan: plan, + EmissionUniverse: universe, + EnableCoroEntryResolution: true, + EnableCoroPlainDispatch: true, + CoroABI: coro.EntryResolutionABIV0, + SchedulerABI: coro.SchedulerNoneABIV0, + PanicABI: coro.PanicLegacyABIV0, + FuncRepABI: coro.FuncRepABIV1, + }}, + ) + if err != nil { + t.Fatalf("compile zero-binding descriptor closure: %v", err) + } + if err := llvm.VerifyModule(compiled.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify zero-binding descriptor closure: %v\n%s", err, compiled.Module().String()) + } + ir := compiled.Module().String() + if !strings.Contains(ir, coroPlainDispatchDescriptorPrefix) || !strings.Contains(ir, coroPlainDispatchThunkPrefix) || strings.Contains(ir, coroPrimarySuffix) { + t.Fatalf("zero-binding closure did not use one plain descriptor body:\n%s", ir) + } +} + +func coroPlainDispatchOnlyDynamicCall(t *testing.T, fn *ssa.Function) ssa.CallInstruction { + t.Helper() + var found ssa.CallInstruction + for _, block := range fn.Blocks { + for _, instr := range block.Instrs { + call, ok := instr.(ssa.CallInstruction) + if !ok || call.Common() == nil || call.Common().StaticCallee() != nil { + continue + } + if found != nil { + t.Fatalf("function %q has multiple dynamic calls", fn.Name()) + } + found = call + } + } + if found == nil { + t.Fatalf("function %q has no dynamic call", fn.Name()) + } + return found +} diff --git a/cl/coro_entry.go b/cl/coro_entry.go index b7c396fe96..73bbed8c4f 100644 --- a/cl/coro_entry.go +++ b/cl/coro_entry.go @@ -32,15 +32,16 @@ const coroPrimarySuffix = "$coro" // FuncRep only describes escaped function values and never authorizes a // second body. type plannedFunctionSymbol struct { - function *ssa.Function - pkgTypes *types.Package - name string - ftype int - plan coro.FunctionPlan - planned bool - physical bool - childAwait bool - coroPlan *coro.SSAPlan + function *ssa.Function + pkgTypes *types.Package + name string + ftype int + plan coro.FunctionPlan + planned bool + physical bool + childAwait bool + plainDispatch bool + coroPlan *coro.SSAPlan } // resolveFunctionSymbol is shared by function definitions and declarations so @@ -84,6 +85,7 @@ func (p *context) resolveFunctionSymbol(fn *ssa.Function) (plannedFunctionSymbol entry.planned = true entry.physical = p.compilation.EnableCoroPhysicalABI entry.childAwait = p.compilation.EnableCoroChildAwait + entry.plainDispatch = p.compilation.EnableCoroPlainDispatch entry.coroPlan = p.compilation.CoroPlan if p.compilation.CoroPlan.IgnoresBody(fn) { return entry, fmt.Errorf("coroutine entry resolution: Go-emitted function %q has an ignored SSA body", plan.ID) @@ -163,7 +165,10 @@ func (e plannedFunctionSymbol) checkSupported() error { return fmt.Errorf("coroutine entry resolution: function %q has no emitted entry", e.plan.ID) } if e.plan.FuncRep == coro.Dispatch { - return fmt.Errorf("coroutine entry resolution: function %q requires an unimplemented dispatch descriptor", e.plan.ID) + if !e.plainDispatch { + return fmt.Errorf("coroutine entry resolution: function %q requires an unimplemented dispatch descriptor", e.plan.ID) + } + return validateCoroPlainDispatchTarget(e.function, e.plan) } if e.plan.Emission == coro.EmitCoroutine { if !e.physical { @@ -234,12 +239,13 @@ func (c *Compilation) preflightCoroPlan() error { continue } entry := plannedFunctionSymbol{ - function: function.Function, - plan: function.Plan, - planned: true, - physical: c.EnableCoroPhysicalABI, - childAwait: c.EnableCoroChildAwait, - coroPlan: c.CoroPlan, + function: function.Function, + plan: function.Plan, + planned: true, + physical: c.EnableCoroPhysicalABI, + childAwait: c.EnableCoroChildAwait, + plainDispatch: c.EnableCoroPlainDispatch, + coroPlan: c.CoroPlan, } if err := entry.checkSupported(); err != nil { c.coroPreflightErr = err @@ -258,6 +264,12 @@ func (c *Compilation) preflightCoroPlan() error { } if c.EnableCoroPhysicalABI { c.coroPreflightErr = validateCoroPhysicalConsumers(c.CoroPlan, c.EnableCoroChildAwait) + if c.coroPreflightErr != nil { + return + } + } + if c.EnableCoroPlainDispatch { + c.coroPreflightErr = validateCoroPlainDispatchConsumers(c.CoroPlan) } }) return c.coroPreflightErr From 687723ef1b486184e83c15abd1023a4647dbca99 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 16:43:48 +0800 Subject: [PATCH 05/32] feat(coro): prove TLS destructor dispatch --- internal/build/build.go | 97 ++- internal/build/coro_plan_test.go | 38 +- internal/build/coro_tls_destructor.go | 823 +++++++++++++++++++++ internal/build/coro_tls_destructor_test.go | 379 ++++++++++ 4 files changed, 1298 insertions(+), 39 deletions(-) create mode 100644 internal/build/coro_tls_destructor.go create mode 100644 internal/build/coro_tls_destructor_test.go diff --git a/internal/build/build.go b/internal/build/build.go index daac2c931f..7a33aca9a2 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -140,6 +140,7 @@ type CoroPlanInput struct { requiredRoots coro.Roots requiredPlain map[*ssa.Function]struct{} requiredDirectPlain []requiredCoroDirectPlainCallArgument + requiredClosedDynamic map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate recordAnalysis func(*coro.SSAPlan) } @@ -314,6 +315,31 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. return compilerRequired, nil } } + if len(in.requiredClosedDynamic) != 0 || config.ClassifyClosedDynamicCall != nil { + classifyClosed := config.ClassifyClosedDynamicCall + config.ClassifyClosedDynamicCall = func(caller *ssa.Function, call ssa.CallInstruction) (coro.SSAClosedDynamicCallCertificate, bool, error) { + compilerCertificate, compilerRequired := in.requiredClosedDynamic[call] + if classifyClosed != nil { + requested, classified, err := classifyClosed(caller, call) + if err != nil { + return coro.SSAClosedDynamicCallCertificate{}, false, err + } + if !classified && (requested.MayBeNil || len(requested.Targets) != 0) { + return coro.SSAClosedDynamicCallCertificate{}, false, fmt.Errorf("builder returned closed dynamic call facts without classifying the call in %q", caller.Name()) + } + if classified && !compilerRequired { + return coro.SSAClosedDynamicCallCertificate{}, false, fmt.Errorf("builder cannot close ordinary dynamic call in %q without a frozen compiler field-flow proof", caller.Name()) + } + if classified && !sameCoroClosedDynamicCallCertificate(requested, compilerCertificate) { + return coro.SSAClosedDynamicCallCertificate{}, false, fmt.Errorf("builder closed dynamic call certificate in %q conflicts with the frozen compiler proof", caller.Name()) + } + } + if !compilerRequired { + return coro.SSAClosedDynamicCallCertificate{}, false, nil + } + return cloneCoroClosedDynamicCallCertificate(compilerCertificate), true, nil + } + } if in.augmentFunctionIDs != nil { config.FunctionIDs = in.augmentFunctionIDs(config.FunctionIDs) } @@ -326,6 +352,9 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. if err == nil { err = validateRequiredCoroDirectPlainCallArguments(plan, in.requiredDirectPlain) } + if err == nil { + err = validateRequiredCoroClosedDynamicCalls(plan, in.requiredClosedDynamic) + } if err == nil && in.recordAnalysis != nil { in.recordAnalysis(plan) } @@ -932,15 +961,16 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { } analyzedPlans := make(map[*coro.SSAPlan]struct{}) var analyzedPlansMu sync.Mutex - requiredRoots, requiredPlain, requiredDirectPlain, err := requiredCoroProgramRuntimePlan(ctx) + requiredRoots, requiredPlain, requiredDirectPlain, requiredClosedDynamic, err := requiredCoroProgramRuntimePlan(ctx) if err != nil { return err } input := CoroPlanInput{ - Program: ctx.progSSA, - requiredRoots: requiredRoots, - requiredPlain: requiredPlain, - requiredDirectPlain: requiredDirectPlain, + Program: ctx.progSSA, + requiredRoots: requiredRoots, + requiredPlain: requiredPlain, + requiredDirectPlain: requiredDirectPlain, + requiredClosedDynamic: requiredClosedDynamic, recordAnalysis: func(plan *coro.SSAPlan) { if plan != nil { analyzedPlansMu.Lock() @@ -1064,12 +1094,16 @@ func activeCoroFuncRepABIVersion(conf *Config) string { // coroutine, and exact frozen C leaves receive a temporary compatible-known // summary. Their fallback SSA stubs remain ignored; ordinary C declarations // outside this compiler-owned closure stay unknown foreign. -func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function]struct{}, []requiredCoroDirectPlainCallArgument, error) { +func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function]struct{}, []requiredCoroDirectPlainCallArgument, map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate, error) { if ctx == nil || ctx.buildConf == nil || !ctx.buildConf.EnableCoroProgramBootstrapRun { - return nil, nil, nil, nil + return nil, nil, nil, nil, nil } if ctx.coroSSAEmission == nil || ctx.coroEmission == nil { - return nil, nil, nil, fmt.Errorf("coroutine program bootstrap runtime roots require a frozen emission universe") + return nil, nil, nil, nil, fmt.Errorf("coroutine program bootstrap runtime roots require a frozen emission universe") + } + closedDynamic, err := proveCoroTLSDestructorClosedDynamicCalls(ctx) + if err != nil { + return nil, nil, nil, nil, err } names := []string{ "init", @@ -1094,7 +1128,7 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function continue } if previous := byName[fn.Name()]; previous != nil && previous != fn { - return nil, nil, nil, fmt.Errorf("coroutine program bootstrap runtime ABI %q has multiple canonical SSA bodies", fn.Name()) + return nil, nil, nil, nil, fmt.Errorf("coroutine program bootstrap runtime ABI %q has multiple canonical SSA bodies", fn.Name()) } byName[fn.Name()] = fn } @@ -1102,14 +1136,14 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function for _, name := range names { fn := byName[name] if fn == nil { - return nil, nil, nil, fmt.Errorf("coroutine program bootstrap runtime ABI %q has no emitted Go body in %q", name, llssa.PkgRuntime) + return nil, nil, nil, nil, fmt.Errorf("coroutine program bootstrap runtime ABI %q has no emitted Go body in %q", name, llssa.PkgRuntime) } goBody, err := frozenGoEmittedBody(ctx.coroEmission, fn) if err != nil { - return nil, nil, nil, fmt.Errorf("classify coroutine program bootstrap runtime ABI %q: %w", name, err) + return nil, nil, nil, nil, fmt.Errorf("classify coroutine program bootstrap runtime ABI %q: %w", name, err) } if !goBody { - return nil, nil, nil, fmt.Errorf("coroutine program bootstrap runtime ABI %q has no emitted Go body in %q", name, llssa.PkgRuntime) + return nil, nil, nil, nil, fmt.Errorf("coroutine program bootstrap runtime ABI %q has no emitted Go body in %q", name, llssa.PkgRuntime) } roots = append(roots, coro.Root{Function: fn, Demand: coro.SyncDemand}) } @@ -1128,7 +1162,7 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function plain[fn] = struct{}{} goBody, err := frozenGoEmittedBody(ctx.coroEmission, fn) if err != nil { - return nil, nil, nil, fmt.Errorf("classify compiler runtime ABI function %q: %w", fn.Name(), err) + return nil, nil, nil, nil, fmt.Errorf("classify compiler runtime ABI function %q: %w", fn.Name(), err) } if !goBody { // Exact C declarations remain required plain leaves, but their Go @@ -1144,6 +1178,13 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function } raw := call.Common().StaticCallee() if raw == nil { + if _, certified := closedDynamic[call]; certified { + // The certified descriptor call is part of this exact plain + // callback body, but its target remains outside the trusted + // scheduler-stack island. Fixed-point analysis must prove the + // target NoSuspend/!NeedsPreempt without suppressing either. + continue + } continue } callee, ok := ctx.coroEmission.Resolve(raw) @@ -1152,7 +1193,7 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function } semantics, intrinsic, err := ctx.coroEmission.CoroIntrinsicCallSiteSemantics(call) if err != nil { - return nil, nil, nil, fmt.Errorf("classify compiler runtime ABI intrinsic %q in %q: %w", callee.Name(), fn.Name(), err) + return nil, nil, nil, nil, fmt.Errorf("classify compiler runtime ABI intrinsic %q in %q: %w", callee.Name(), fn.Name(), err) } if intrinsic && semantics == cl.CoroIntrinsicCallInlineNoSuspend { // cl emits the proven no-suspend operation inline in fn; it @@ -1175,9 +1216,9 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function if !ok { continue } - closure, ok, err := provenCoroDirectPlainStaticClosure(ctx, target) + closure, ok, err := provenCoroDirectPlainStaticClosure(ctx, target, closedDynamic) if err != nil { - return nil, nil, nil, fmt.Errorf("prove direct-plain callback target %q in %q: %w", target.Name(), fn.Name(), err) + return nil, nil, nil, nil, fmt.Errorf("prove direct-plain callback target %q in %q: %w", target.Name(), fn.Name(), err) } if !ok { continue @@ -1194,7 +1235,7 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function } } } - return roots, plain, directPlain, nil + return roots, plain, directPlain, closedDynamic, nil } func frozenGoEmittedBody(universe *cl.EmissionUniverse, fn *ssa.Function) (bool, error) { @@ -1274,7 +1315,7 @@ func exactCoroStaticFunctionValue(ctx *context, value ssa.Value) (*ssa.Function, // on the ordinary Dispatch path. Effect and representation are independently // checked after fixed-point analysis; this prefilter only establishes that it // is sound to seed the candidate's bounded scheduler-stack island. -func provenCoroDirectPlainStaticClosure(ctx *context, target *ssa.Function) ([]*ssa.Function, bool, error) { +func provenCoroDirectPlainStaticClosure(ctx *context, target *ssa.Function, closedDynamic map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate) ([]*ssa.Function, bool, error) { if ctx == nil || ctx.coroEmission == nil || target == nil || len(target.FreeVars) != 0 { return nil, false, nil } @@ -1316,6 +1357,13 @@ func provenCoroDirectPlainStaticClosure(ctx *context, target *ssa.Function) ([]* } raw := call.Common().StaticCallee() if raw == nil { + if _, certified := closedDynamic[call]; certified && !call.Common().IsInvoke() { + // The exact descriptor target is deliberately not added to + // closure: unlike the raw C callback it is not trusted to run + // without preemption. Post-plan validation checks its real + // fixed-point Effect/Exec instead. + continue + } return nil, false, nil } callee, ok := ctx.coroEmission.Resolve(raw) @@ -1549,11 +1597,14 @@ type context struct { // coroPlan is compilation-scoped. It remains report-only unless // EnableCoroEntryResolution is set explicitly. - coroPlan *coro.SSAPlan - coroEmission *cl.EmissionUniverse - coroSSAEmission *coro.SSAEmissionUniverse - coroPlanDigest string - coroPlanMetadata coro.PlanDigestMetadata + coroPlan *coro.SSAPlan + coroEmission *cl.EmissionUniverse + coroSSAEmission *coro.SSAEmissionUniverse + // coroTLSDestructorFixturePkg is an internal test-only identity override. + // Production builds leave it empty and accept only runtime/internal/clite/tls. + coroTLSDestructorFixturePkg string + coroPlanDigest string + coroPlanMetadata coro.PlanDigestMetadata // Frozen immediately after whole-program analysis, before package codegen. // linkMainPkg only consumes these exact per-entry-package tables. coroProgramBootstraps map[string]*coroProgramBootstrapV1 diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index 22cc064cf7..f8d765b578 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -306,15 +306,16 @@ func inlineIntrinsic(string) *byte coroEmission: emission, coroSSAEmission: ssaEmission, } - roots, requiredPlain, directPlain, err := requiredCoroProgramRuntimePlan(ctx) + roots, requiredPlain, directPlain, closedDynamic, err := requiredCoroProgramRuntimePlan(ctx) if err != nil { t.Fatal(err) } - rootsAgain, plainAgain, directAgain, err := requiredCoroProgramRuntimePlan(ctx) + rootsAgain, plainAgain, directAgain, closedAgain, err := requiredCoroProgramRuntimePlan(ctx) if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(rootsAgain, roots) || !reflect.DeepEqual(plainAgain, requiredPlain) || !reflect.DeepEqual(directAgain, directPlain) { + if !reflect.DeepEqual(rootsAgain, roots) || !reflect.DeepEqual(plainAgain, requiredPlain) || + !reflect.DeepEqual(directAgain, directPlain) || !reflect.DeepEqual(closedAgain, closedDynamic) { t.Fatal("required runtime roots/plain closure is not deterministic") } if len(directPlain) != 0 { @@ -366,6 +367,7 @@ func inlineIntrinsic(string) *byte requiredRoots: roots, requiredPlain: requiredPlain, requiredDirectPlain: directPlain, + requiredClosedDynamic: closedDynamic, } functionIDs := emission.FunctionIDConfig() functionIDs.CoroABI = coro.PhysicalABIV1 @@ -488,7 +490,7 @@ func inlineIntrinsic(string) *byte coroEmission: emission, coroSSAEmission: ssaEmission, } - _, _, _, err = requiredCoroProgramRuntimePlan(ctx) + _, _, _, _, err = requiredCoroProgramRuntimePlan(ctx) if err == nil || !strings.Contains(err.Error(), "requires exactly one compile-time string constant argument") { t.Fatalf("invalid runtime-closure intrinsic error = %v; want exact call-site rejection", err) } @@ -813,6 +815,7 @@ type requiredCoroRuntimeFixture struct { input CoroPlanInput requiredPlain map[*ssa.Function]struct{} directPlain []requiredCoroDirectPlainCallArgument + closedDynamic map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate functionIDs coro.FunctionIDConfig } @@ -847,12 +850,13 @@ func __llgo_coro_frame_free_v1() {} t.Fatal(err) } ctx := &context{ - prog: prog, - buildConf: &Config{EnableCoroProgramBootstrapRun: true}, - coroEmission: emission, - coroSSAEmission: ssaEmission, + prog: prog, + buildConf: &Config{EnableCoroProgramBootstrapRun: true}, + coroEmission: emission, + coroSSAEmission: ssaEmission, + coroTLSDestructorFixturePkg: llssa.PkgRuntime, } - roots, requiredPlain, directPlain, err := requiredCoroProgramRuntimePlan(ctx) + roots, requiredPlain, directPlain, closedDynamic, err := requiredCoroProgramRuntimePlan(ctx) if err != nil { t.Fatal(err) } @@ -864,16 +868,18 @@ func __llgo_coro_frame_free_v1() {} pkg: ssaPkg, ctx: ctx, input: CoroPlanInput{ - Program: ssaPkg.Prog, - EmissionUniverse: ssaEmission, - resolveFunction: emission.Resolve, - functionBackground: emission.FunctionBackground, - requiredRoots: roots, - requiredPlain: requiredPlain, - requiredDirectPlain: directPlain, + Program: ssaPkg.Prog, + EmissionUniverse: ssaEmission, + resolveFunction: emission.Resolve, + functionBackground: emission.FunctionBackground, + requiredRoots: roots, + requiredPlain: requiredPlain, + requiredDirectPlain: directPlain, + requiredClosedDynamic: closedDynamic, }, requiredPlain: requiredPlain, directPlain: directPlain, + closedDynamic: closedDynamic, functionIDs: functionIDs, } } diff --git a/internal/build/coro_tls_destructor.go b/internal/build/coro_tls_destructor.go new file mode 100644 index 0000000000..79b87c8db3 --- /dev/null +++ b/internal/build/coro_tls_destructor.go @@ -0,0 +1,823 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "fmt" + "go/token" + "go/types" + "strings" + + "golang.org/x/tools/go/ssa" + + "github.com/goplus/llgo/internal/coro" + llssa "github.com/goplus/llgo/ssa" +) + +// coroTLSField identifies one exact field of one concrete SSA struct type. +// Generic instances intentionally remain distinct: a destructor target for +// slot[A] says nothing about slot[B]. +type coroTLSField struct { + container types.Type + index int + typ types.Type +} + +type coroTLSFieldAccesses struct { + loads []*ssa.UnOp + stores []*ssa.Store +} + +// proveCoroTLSDestructorClosedDynamicCalls recognizes only the compiler-owned +// TLS callback shape used by runtime/internal/clite/tls. The proof is derived +// from exact frozen SSA objects; source names are not used to invent targets. +// +// The proof is deliberately object-insensitive but field- and concrete-type- +// sensitive. That is sound for these unexported fields once every normal field +// write and every field-address use in the frozen program has been audited. +// Unsafe writes through a tracked aggregate pointer and interface publication +// fail closed, apart from the runtime's exact opaque-pointer ingress and its +// frozen read-only rootRange helper. +func proveCoroTLSDestructorClosedDynamicCalls(ctx *context) (map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate, error) { + result := make(map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate) + if ctx == nil || ctx.coroEmission == nil || ctx.coroSSAEmission == nil || ctx.prog == nil { + return result, nil + } + functions, err := coroTLSFrozenGoBodies(ctx) + if err != nil { + return nil, err + } + for _, owner := range functions { + if !coroTLSConcreteFunction(owner) { + continue + } + for _, block := range owner.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(ssa.CallInstruction) + if !ok || call.Common() == nil || call.Common().StaticCallee() == nil { + continue + } + for argument, value := range call.Common().Args { + parameter, ok := staticCallArgumentParameterType(call, argument) + if !ok || ctx.prog.TypeBackground(parameter) != llssa.InC { + continue + } + if _, functionType := types.Unalias(parameter).Underlying().(*types.Signature); !functionType { + continue + } + callback, ok := exactCoroStaticFunctionValue(ctx, value) + if !ok || !coroTLSConcreteFunction(callback) { + continue + } + certifiedCall, certificate, candidate, err := proveOneCoroTLSDestructorCallback(ctx, functions, callback) + if err != nil { + return nil, fmt.Errorf("prove TLS direct-plain callback %q in %q: %w", callback.Name(), owner.Name(), err) + } + if !candidate { + continue + } + if previous, exists := result[certifiedCall]; exists && !sameCoroClosedDynamicCallCertificate(previous, certificate) { + return nil, fmt.Errorf("TLS dynamic call in %q has conflicting frozen certificates", callback.Name()) + } + result[certifiedCall] = cloneCoroClosedDynamicCallCertificate(certificate) + } + } + } + } + return result, nil +} + +func coroTLSFrozenGoBodies(ctx *context) ([]*ssa.Function, error) { + functions := make([]*ssa.Function, 0, len(ctx.coroSSAEmission.Functions())) + for _, fn := range ctx.coroSSAEmission.Functions() { + goBody, err := frozenGoEmittedBody(ctx.coroEmission, fn) + if err != nil { + return nil, fmt.Errorf("classify TLS field-flow body %q: %w", fn.Name(), err) + } + if goBody { + functions = append(functions, fn) + } + } + return functions, nil +} + +func coroTLSConcreteFunction(fn *ssa.Function) bool { + if fn == nil || fn.Signature == nil || len(fn.Blocks) == 0 || len(fn.FreeVars) != 0 { + return false + } + return (fn.Signature.TypeParams() == nil || fn.Signature.TypeParams().Len() == 0) && + (fn.Signature.RecvTypeParams() == nil || fn.Signature.RecvTypeParams().Len() == 0) +} + +func proveOneCoroTLSDestructorCallback( + ctx *context, + functions []*ssa.Function, + callback *ssa.Function, +) (ssa.CallInstruction, coro.SSAClosedDynamicCallCertificate, bool, error) { + if !coroTLSFunctionInOwnedPackage(ctx, callback) { + return nil, coro.SSAClosedDynamicCallCertificate{}, false, nil + } + goBody, err := frozenGoEmittedBody(ctx.coroEmission, callback) + if err != nil { + return nil, coro.SSAClosedDynamicCallCertificate{}, false, err + } + if !goBody || len(callback.FreeVars) != 0 { + return nil, coro.SSAClosedDynamicCallCertificate{}, false, nil + } + + var dynamicCalls []ssa.CallInstruction + var fieldCalls []ssa.CallInstruction + var slotField coroTLSField + for _, block := range callback.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(ssa.CallInstruction) + if !ok || call.Common() == nil { + continue + } + if _, builtin := call.Common().Value.(*ssa.Builtin); builtin || call.Common().StaticCallee() != nil { + continue + } + dynamicCalls = append(dynamicCalls, call) + if _, field, ok := coroTLSExactFieldLoad(call.Common().Value); ok { + fieldCalls = append(fieldCalls, call) + slotField = field + } + } + } + if len(fieldCalls) == 0 { + return nil, coro.SSAClosedDynamicCallCertificate{}, false, nil + } + if len(dynamicCalls) != 1 || len(fieldCalls) != 1 { + // A field call alone does not make an arbitrary C callback part of the + // TLS destructor protocol. A real protocol callback must have the exact + // single-dynamic-call shape; otherwise leave it to the ordinary C callback + // closure proof instead of turning unrelated callbacks into TLS errors. + return nil, coro.SSAClosedDynamicCallCertificate{}, false, nil + } + dynamicCall := fieldCalls[0] + if _, ordinary := dynamicCall.(*ssa.Call); !ordinary || dynamicCall.Common().IsInvoke() { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("field-loaded destructor must be an ordinary dynamic *ssa.Call") + } + calleeLoad, _, _ := coroTLSExactFieldLoad(dynamicCall.Common().Value) + + slotAccesses, err := collectCoroTLSFieldAccesses(functions, slotField) + if err != nil { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("audit destination destructor field: %w", err) + } + if err := auditCoroTLSSlotLoads(slotAccesses.loads, calleeLoad, dynamicCall); err != nil { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, err + } + var sourceField coroTLSField + nonnilStores := 0 + nilStores := 0 + for _, store := range slotAccesses.stores { + if coroTLSNilFunctionValue(store.Val) { + nilStores++ + continue + } + _, field, ok := coroTLSExactFieldLoad(store.Val) + if !ok { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("destination destructor field has an unknown non-nil write in %q", store.Parent().Name()) + } + if nonnilStores != 0 && !sameCoroTLSField(sourceField, field) { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("destination destructor field has multiple source fields") + } + sourceField = field + nonnilStores++ + } + if nonnilStores != 1 || nilStores == 0 { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("destination destructor field writes are not the exact source-plus-nil pattern (source=%d nil=%d)", nonnilStores, nilStores) + } + if sameCoroTLSField(slotField, sourceField) { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("destination destructor field feeds itself") + } + + sourceAccesses, err := collectCoroTLSFieldAccesses(functions, sourceField) + if err != nil { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("audit source destructor field: %w", err) + } + if err := auditCoroTLSSourceLoads(sourceAccesses.loads, slotField); err != nil { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, err + } + var formal *ssa.Parameter + formalStores := 0 + for _, store := range sourceAccesses.stores { + if coroTLSNilFunctionValue(store.Val) { + continue + } + parameter, ok := store.Val.(*ssa.Parameter) + if !ok || (formal != nil && formal != parameter) { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("source destructor field has a write not owned by one exact formal parameter") + } + formal = parameter + formalStores++ + } + if formal == nil || formalStores != 1 || formal.Parent() == nil { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("source destructor field is not initialized exactly once from an allocator formal") + } + if err := auditCoroTLSFormalUses(formal, sourceField); err != nil { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, err + } + formalIndex := -1 + for index, parameter := range formal.Parent().Params { + if parameter == formal { + formalIndex = index + break + } + } + if formalIndex < 0 { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("allocator destructor formal is absent from its SSA parameter list") + } + if !coroTLSFunctionTypeMatchesSignature(formal.Type(), dynamicCall.Common().Signature()) || + !coroTLSFunctionTypeMatchesSignature(slotField.typ, dynamicCall.Common().Signature()) || + !types.Identical(types.Unalias(formal.Type()).Underlying(), types.Unalias(sourceField.typ).Underlying()) { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, fmt.Errorf("allocator, source field, destination field, and dynamic call signatures differ") + } + + certificate, err := collectCoroTLSAllocatorTargets(ctx, functions, formal.Parent(), formalIndex, formal.Type()) + if err != nil { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, err + } + if err := auditCoroTLSTrackedEscapes(ctx, functions, slotField, sourceField); err != nil { + return nil, coro.SSAClosedDynamicCallCertificate{}, true, err + } + return dynamicCall, certificate, true, nil +} + +func coroTLSFunctionInOwnedPackage(ctx *context, fn *ssa.Function) bool { + if fn == nil { + return false + } + identity := fn + if origin := fn.Origin(); origin != nil { + identity = origin + } + if identity.Pkg == nil || identity.Pkg.Pkg == nil { + return false + } + expected := strings.TrimSuffix(llssa.PkgRuntime, "/internal/runtime") + "/internal/clite/tls" + if ctx != nil && ctx.coroTLSDestructorFixturePkg != "" { + expected = ctx.coroTLSDestructorFixturePkg + } + return llssa.PathOf(identity.Pkg.Pkg) == expected +} + +func coroTLSExactFieldLoad(value ssa.Value) (*ssa.UnOp, coroTLSField, bool) { + load, ok := value.(*ssa.UnOp) + if !ok || load.Op != token.MUL { + return nil, coroTLSField{}, false + } + field, ok := load.X.(*ssa.FieldAddr) + if !ok { + return nil, coroTLSField{}, false + } + key, ok := coroTLSFieldOf(field) + return load, key, ok +} + +func coroTLSFieldOf(field *ssa.FieldAddr) (coroTLSField, bool) { + if field == nil || field.X == nil || field.X.Type() == nil { + return coroTLSField{}, false + } + pointer, ok := types.Unalias(field.X.Type()).Underlying().(*types.Pointer) + if !ok { + return coroTLSField{}, false + } + container := types.Unalias(pointer.Elem()) + structure, ok := container.Underlying().(*types.Struct) + if !ok || field.Field < 0 || field.Field >= structure.NumFields() { + return coroTLSField{}, false + } + typ := structure.Field(field.Field).Type() + if _, ok := types.Unalias(typ).Underlying().(*types.Signature); !ok { + return coroTLSField{}, false + } + return coroTLSField{container: container, index: field.Field, typ: typ}, true +} + +func sameCoroTLSField(left, right coroTLSField) bool { + return left.index == right.index && left.container != nil && right.container != nil && types.Identical(left.container, right.container) +} + +func collectCoroTLSFieldAccesses(functions []*ssa.Function, field coroTLSField) (coroTLSFieldAccesses, error) { + var result coroTLSFieldAccesses + for _, owner := range functions { + for _, block := range owner.Blocks { + for _, instruction := range block.Instrs { + address, ok := instruction.(*ssa.FieldAddr) + if !ok { + continue + } + candidate, ok := coroTLSFieldOf(address) + if !ok || !sameCoroTLSField(field, candidate) { + continue + } + refs := address.Referrers() + if refs == nil { + return coroTLSFieldAccesses{}, fmt.Errorf("field address in %q has no frozen referrer set", owner.Name()) + } + for _, ref := range *refs { + switch ref := ref.(type) { + case *ssa.DebugRef: + case *ssa.UnOp: + if ref.X != address || ref.Op != token.MUL { + return coroTLSFieldAccesses{}, fmt.Errorf("field address has a non-load unary use in %q", owner.Name()) + } + result.loads = append(result.loads, ref) + case *ssa.Store: + if ref.Addr != address { + return coroTLSFieldAccesses{}, fmt.Errorf("field address escapes as a stored value in %q", owner.Name()) + } + result.stores = append(result.stores, ref) + default: + return coroTLSFieldAccesses{}, fmt.Errorf("field address escapes through %T in %q", ref, owner.Name()) + } + } + } + } + } + return result, nil +} + +func auditCoroTLSSlotLoads(loads []*ssa.UnOp, callee *ssa.UnOp, call ssa.CallInstruction) error { + if len(loads) < 2 || callee == nil { + return fmt.Errorf("destination destructor field lacks an exact nil guard and call load") + } + guarded := false + called := false + for _, load := range loads { + refs := load.Referrers() + if refs == nil || len(*refs) == 0 { + return fmt.Errorf("destination destructor load in %q has no use", load.Parent().Name()) + } + for _, ref := range *refs { + if _, debug := ref.(*ssa.DebugRef); debug { + continue + } + if load == callee && ref == call { + called = true + continue + } + comparison, ok := ref.(*ssa.BinOp) + if !ok || (comparison.Op != token.EQL && comparison.Op != token.NEQ) || !coroTLSComparisonWithNil(comparison, load) { + return fmt.Errorf("destination destructor load escapes through %T in %q", ref, load.Parent().Name()) + } + guarded = guarded || coroTLSComparisonGuardsCall(comparison, call) + } + } + if !guarded || !called { + return fmt.Errorf("destination destructor field is not control-flow nil-guarded before its exact dynamic call") + } + return nil +} + +func coroTLSComparisonGuardsCall(comparison *ssa.BinOp, call ssa.CallInstruction) bool { + if comparison == nil || call == nil || comparison.Block() == nil || call.Block() == nil { + return false + } + refs := comparison.Referrers() + if refs == nil { + return false + } + var branch *ssa.If + for _, ref := range *refs { + if _, debug := ref.(*ssa.DebugRef); debug { + continue + } + candidate, ok := ref.(*ssa.If) + if !ok || candidate.Cond != comparison || branch != nil { + return false + } + branch = candidate + } + if branch == nil || len(branch.Block().Succs) != 2 { + return false + } + nonNilSuccessor := 0 + if comparison.Op == token.EQL { + nonNilSuccessor = 1 + } + return branch.Block().Succs[nonNilSuccessor].Dominates(call.Block()) +} + +func coroTLSComparisonWithNil(comparison *ssa.BinOp, value ssa.Value) bool { + if comparison == nil { + return false + } + other := comparison.X + if other == value { + other = comparison.Y + } else if comparison.Y != value { + return false + } + constant, ok := other.(*ssa.Const) + return ok && constant.IsNil() +} + +func auditCoroTLSSourceLoads(loads []*ssa.UnOp, destination coroTLSField) error { + if len(loads) == 0 { + return fmt.Errorf("source destructor field is never copied to the destination field") + } + for _, load := range loads { + refs := load.Referrers() + if refs == nil || len(*refs) == 0 { + return fmt.Errorf("source destructor load in %q has no use", load.Parent().Name()) + } + for _, ref := range *refs { + if _, debug := ref.(*ssa.DebugRef); debug { + continue + } + store, ok := ref.(*ssa.Store) + if !ok || store.Val != load { + return fmt.Errorf("source destructor load escapes through %T in %q", ref, load.Parent().Name()) + } + address, ok := store.Addr.(*ssa.FieldAddr) + field, fieldOK := coroTLSFieldOf(address) + if !ok || !fieldOK || !sameCoroTLSField(field, destination) { + return fmt.Errorf("source destructor load is stored outside the exact destination field in %q", load.Parent().Name()) + } + } + } + return nil +} + +func auditCoroTLSFormalUses(formal *ssa.Parameter, source coroTLSField) error { + refs := formal.Referrers() + if refs == nil || len(*refs) == 0 { + return fmt.Errorf("allocator destructor formal has no uses") + } + for _, ref := range *refs { + if _, debug := ref.(*ssa.DebugRef); debug { + continue + } + store, ok := ref.(*ssa.Store) + if !ok || store.Val != formal { + return fmt.Errorf("allocator destructor formal escapes through %T in %q", ref, formal.Parent().Name()) + } + address, ok := store.Addr.(*ssa.FieldAddr) + field, fieldOK := coroTLSFieldOf(address) + if !ok || !fieldOK || !sameCoroTLSField(field, source) { + return fmt.Errorf("allocator destructor formal is stored outside the exact source field") + } + } + return nil +} + +func collectCoroTLSAllocatorTargets( + ctx *context, + functions []*ssa.Function, + allocator *ssa.Function, + formalIndex int, + formalType types.Type, +) (coro.SSAClosedDynamicCallCertificate, error) { + if err := auditCoroTLSAllocatorUses(ctx, functions, allocator); err != nil { + return coro.SSAClosedDynamicCallCertificate{}, err + } + certificate := coro.SSAClosedDynamicCallCertificate{MayBeNil: true} + callSites := 0 + var target *ssa.Function + for _, owner := range functions { + for _, block := range owner.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(ssa.CallInstruction) + if !ok || call.Common() == nil || call.Common().StaticCallee() == nil { + continue + } + resolved, ok := ctx.coroEmission.Resolve(call.Common().StaticCallee()) + if !ok || resolved != allocator { + continue + } + if _, ordinary := call.(*ssa.Call); !ordinary || formalIndex >= len(call.Common().Args) { + return coro.SSAClosedDynamicCallCertificate{}, fmt.Errorf("allocator destructor formal is reached through go/defer or a malformed call in %q", owner.Name()) + } + callSites++ + actual := call.Common().Args[formalIndex] + if coroTLSNilFunctionValue(actual) { + continue + } + candidate, ok := exactCoroStaticFunctionValue(ctx, actual) + if !ok || candidate == nil || len(candidate.FreeVars) != 0 { + return coro.SSAClosedDynamicCallCertificate{}, fmt.Errorf("allocator destructor actual in %q is not nil or one exact no-capture function", owner.Name()) + } + goBody, err := frozenGoEmittedBody(ctx.coroEmission, candidate) + if err != nil { + return coro.SSAClosedDynamicCallCertificate{}, err + } + if !goBody || !coroTLSFunctionTypeMatchesSignature(formalType, candidate.Signature) { + return coro.SSAClosedDynamicCallCertificate{}, fmt.Errorf("allocator destructor target %q is not an owned exact-signature Go body", candidate.Name()) + } + if target != nil && target != candidate { + return coro.SSAClosedDynamicCallCertificate{}, fmt.Errorf("allocator destructor field has multiple non-nil targets %q and %q", target.Name(), candidate.Name()) + } + target = candidate + } + } + } + if callSites == 0 { + return coro.SSAClosedDynamicCallCertificate{}, fmt.Errorf("allocator destructor formal has no exact frozen call sites") + } + if target != nil { + certificate.Targets = []*ssa.Function{target} + } + return certificate, nil +} + +func auditCoroTLSAllocatorUses(ctx *context, functions []*ssa.Function, allocator *ssa.Function) error { + if allocator == nil { + return fmt.Errorf("allocator function is nil") + } + operands := make([]*ssa.Value, 0, 8) + for _, owner := range functions { + for _, block := range owner.Blocks { + for _, instruction := range block.Instrs { + operands = instruction.Operands(operands[:0]) + usesAllocator := false + for _, operand := range operands { + if operand != nil && *operand == allocator { + usesAllocator = true + break + } + } + if !usesAllocator { + continue + } + call, ok := instruction.(*ssa.Call) + if !ok || call.Common() == nil || call.Common().Value != allocator || call.Common().StaticCallee() == nil { + return fmt.Errorf("allocator function escapes through %T in %q", instruction, owner.Name()) + } + resolved, ok := ctx.coroEmission.Resolve(call.Common().StaticCallee()) + if !ok || resolved != allocator { + return fmt.Errorf("allocator function has a non-exact static use in %q", owner.Name()) + } + } + } + } + return nil +} + +func coroTLSNilFunctionValue(value ssa.Value) bool { + for value != nil { + switch current := value.(type) { + case *ssa.Const: + return current.IsNil() + case *ssa.ChangeType: + value = current.X + case *ssa.Convert: + value = current.X + default: + return false + } + } + return false +} + +func coroTLSFunctionTypeMatchesSignature(typ types.Type, signature *types.Signature) bool { + if typ == nil || signature == nil { + return false + } + function, ok := types.Unalias(typ).Underlying().(*types.Signature) + return ok && types.Identical(function, signature) +} + +func auditCoroTLSTrackedEscapes( + ctx *context, + functions []*ssa.Function, + slot, source coroTLSField, +) error { + for _, owner := range functions { + for _, block := range owner.Blocks { + for _, instruction := range block.Instrs { + switch instruction := instruction.(type) { + case *ssa.Store: + if coroTLSExactType(instruction.Val.Type(), slot.container) { + return fmt.Errorf("tracked TLS aggregate has a whole-value write in %q", owner.Name()) + } + case *ssa.MakeInterface: + if coroTLSTypeContains(instruction.X.Type(), slot.container) || coroTLSTypeContains(instruction.X.Type(), source.container) { + return fmt.Errorf("tracked TLS aggregate escapes through interface conversion in %q", owner.Name()) + } + case *ssa.TypeAssert: + if coroTLSTypeContains(instruction.AssertedType, slot.container) || coroTLSTypeContains(instruction.AssertedType, source.container) { + return fmt.Errorf("tracked TLS aggregate enters through interface assertion in %q", owner.Name()) + } + case *ssa.Convert: + fromSlot := coroTLSPointerTo(instruction.X.Type(), slot.container) + toSlot := coroTLSPointerTo(instruction.Type(), slot.container) + fromSource := coroTLSPointerTo(instruction.X.Type(), source.container) + toSource := coroTLSPointerTo(instruction.Type(), source.container) + if !fromSlot && !toSlot && !fromSource && !toSource { + continue + } + if toSlot && coroTLSUnsafePointerLike(instruction.X.Type()) && + coroTLSExactOpaqueSlotIngress(ctx, owner, slot.container) { + // Opaque pthread/C allocation pointers enter typed Go code in + // these exact compiler-owned TLS accessors. Every typed + // destructor-field write is still enumerated above. + continue + } + if fromSlot && coroTLSUnsafePointerLike(instruction.Type()) && + coroTLSExactRootRangeHelper(ctx, owner, slot.container) { + // rootRange computes the frozen GC scan interval. It does not + // publish the destructor field address or write through it. + continue + } + return fmt.Errorf("tracked TLS aggregate crosses unsafe conversion in %q", owner.Name()) + case *ssa.ChangeType: + if coroTLSPointerTo(instruction.X.Type(), slot.container) || coroTLSPointerTo(instruction.Type(), slot.container) || + coroTLSPointerTo(instruction.X.Type(), source.container) || coroTLSPointerTo(instruction.Type(), source.container) { + return fmt.Errorf("tracked TLS aggregate crosses named pointer conversion in %q", owner.Name()) + } + case *ssa.MakeClosure: + for _, binding := range instruction.Bindings { + if coroTLSTypeContains(binding.Type(), slot.container) || coroTLSTypeContains(binding.Type(), source.container) { + return fmt.Errorf("tracked TLS aggregate escapes into closure in %q", owner.Name()) + } + } + case ssa.CallInstruction: + if !coroTLSCallCarriesTrackedPointer(instruction, slot.container, source.container) { + continue + } + if builtin, ok := instruction.Common().Value.(*ssa.Builtin); ok && builtin.Name() == "ssa:wrapnilchk" { + // The SSA builder's value-receiver wrapper checks then dereferences + // its receiver; it neither publishes nor mutates the aggregate. + continue + } + call, ordinary := instruction.(*ssa.Call) + if !ordinary || call.Common() == nil || call.Common().StaticCallee() == nil { + return fmt.Errorf("tracked TLS aggregate pointer escapes through a dynamic, go, or defer call %q (%T) in %q", instruction.String(), instruction, owner.Name()) + } + callee, ok := ctx.coroEmission.Resolve(call.Common().StaticCallee()) + if !ok { + return fmt.Errorf("tracked TLS aggregate pointer reaches an unresolved callee in %q", owner.Name()) + } + goBody, err := frozenGoEmittedBody(ctx.coroEmission, callee) + if err != nil { + return err + } + if !goBody { + return fmt.Errorf("tracked TLS aggregate pointer escapes to a non-Go callee in %q", owner.Name()) + } + } + } + } + } + return nil +} + +func coroTLSExactType(typ, tracked types.Type) bool { + return typ != nil && tracked != nil && types.Identical(types.Unalias(typ), tracked) +} + +func coroTLSCallCarriesTrackedPointer(call ssa.CallInstruction, tracked ...types.Type) bool { + if call == nil || call.Common() == nil { + return false + } + for _, argument := range call.Common().Args { + for _, typ := range tracked { + if coroTLSPointerTo(argument.Type(), typ) { + return true + } + } + } + return false +} + +func coroTLSPointerTo(typ, container types.Type) bool { + if typ == nil || container == nil { + return false + } + pointer, ok := types.Unalias(typ).Underlying().(*types.Pointer) + return ok && types.Identical(types.Unalias(pointer.Elem()), container) +} + +func coroTLSUnsafePointerLike(typ types.Type) bool { + if typ == nil { + return false + } + basic, ok := types.Unalias(typ).Underlying().(*types.Basic) + return ok && basic.Kind() == types.UnsafePointer +} + +func coroTLSTypeContains(typ, tracked types.Type) bool { + if typ == nil || tracked == nil { + return false + } + typ = types.Unalias(typ) + if types.Identical(typ, tracked) { + return true + } + if pointer, ok := typ.Underlying().(*types.Pointer); ok { + return types.Identical(types.Unalias(pointer.Elem()), tracked) + } + return false +} + +func coroTLSExactOpaqueSlotIngress(ctx *context, fn *ssa.Function, slot types.Type) bool { + if !coroTLSFunctionInOwnedPackage(ctx, fn) || fn.Signature == nil { + return false + } + identity := fn + if origin := fn.Origin(); origin != nil { + identity = origin + } + switch identity.Name() { + case "Get", "Clear", "ensureSlot", "slotDestructor": + return true + default: + return false + } +} + +func coroTLSExactRootRangeHelper(ctx *context, fn *ssa.Function, slot types.Type) bool { + if !coroTLSFunctionInOwnedPackage(ctx, fn) || fn.Signature == nil { + return false + } + identity := fn + if origin := fn.Origin(); origin != nil { + identity = origin + } + if identity.Name() != "rootRange" { + return false + } + receiver := fn.Signature.Recv() + if receiver == nil || !coroTLSPointerTo(receiver.Type(), slot) { + return false + } + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + switch instruction := instruction.(type) { + case *ssa.Store, *ssa.MapUpdate, *ssa.Send, *ssa.Go, *ssa.Defer: + return false + case ssa.CallInstruction: + if instruction.Common() == nil { + return false + } + if _, builtin := instruction.Common().Value.(*ssa.Builtin); !builtin { + return false + } + } + } + } + return true +} + +func cloneCoroClosedDynamicCallCertificate(certificate coro.SSAClosedDynamicCallCertificate) coro.SSAClosedDynamicCallCertificate { + return coro.SSAClosedDynamicCallCertificate{ + Targets: append([]*ssa.Function(nil), certificate.Targets...), + MayBeNil: certificate.MayBeNil, + } +} + +func sameCoroClosedDynamicCallCertificate(left, right coro.SSAClosedDynamicCallCertificate) bool { + if left.MayBeNil != right.MayBeNil || len(left.Targets) != len(right.Targets) { + return false + } + for index := range left.Targets { + if left.Targets[index] != right.Targets[index] { + return false + } + } + return true +} + +func validateRequiredCoroClosedDynamicCalls(plan *coro.SSAPlan, certificates map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate) error { + if len(certificates) == 0 { + return nil + } + if plan == nil { + return fmt.Errorf("compiler TLS closed dynamic call validation requires a coroutine plan") + } + for call, certificate := range certificates { + callPlan, ok := plan.CallPlan(call) + if !ok || callPlan.Rep != coro.Dispatch || callPlan.Open || callPlan.MayBeNil != certificate.MayBeNil || len(callPlan.Targets) != len(certificate.Targets) { + return fmt.Errorf("compiler TLS destructor call in %q did not retain its exact closed Dispatch plan", call.Parent().Name()) + } + for index, target := range certificate.Targets { + id, ok := plan.FunctionID(target) + if !ok || callPlan.Targets[index] != id { + return fmt.Errorf("compiler TLS destructor call in %q lost target %q", call.Parent().Name(), target.Name()) + } + function, ok := plan.FunctionPlan(target) + if !ok || function.External != coro.Defined || function.Effect != coro.NoSuspend || function.Exec.Contains(coro.NeedsPreempt) || + function.FuncRep != coro.Dispatch || function.Primary != coro.PrimaryPlain || function.Emission != coro.EmitPlain { + return fmt.Errorf("compiler TLS destructor target %q is not a defined non-suspending descriptor-backed plain body (external=%s effect=%s exec=%s representation=%s primary=%s emission=%s)", + target.Name(), function.External, function.Effect, function.Exec, function.FuncRep, function.Primary, function.Emission) + } + } + } + return nil +} diff --git a/internal/build/coro_tls_destructor_test.go b/internal/build/coro_tls_destructor_test.go new file mode 100644 index 0000000000..744f41cd98 --- /dev/null +++ b/internal/build/coro_tls_destructor_test.go @@ -0,0 +1,379 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "strings" + "testing" + + "golang.org/x/tools/go/ssa" + + "github.com/goplus/llgo/cl" + "github.com/goplus/llgo/internal/coro" + llssa "github.com/goplus/llgo/ssa" +) + +func TestCoroTLSDestructorClosedDynamicCallProof(t *testing.T) { + fixture := buildRequiredCoroRuntimeFixture(t, coroTLSRuntimeFixtureSource(`func callback(*int) {}`)) + if len(fixture.closedDynamic) != 1 { + t.Fatalf("TLS closed dynamic certificates = %d, want 1", len(fixture.closedDynamic)) + } + if len(fixture.directPlain) != 1 { + t.Fatalf("TLS direct-plain C callbacks = %d, want 1", len(fixture.directPlain)) + } + callback := fixture.pkg.Func("callback") + slotDestructor := fixture.pkg.Func("slotDestructor") + var dynamicCall ssa.CallInstruction + for call, certificate := range fixture.closedDynamic { + dynamicCall = call + if call.Parent() != slotDestructor || !certificate.MayBeNil || len(certificate.Targets) != 1 || certificate.Targets[0] != callback { + t.Fatalf("TLS certificate = call:%v parent:%v certificate:%+v", call, call.Parent(), certificate) + } + } + if use := fixture.directPlain[0]; use.target != slotDestructor { + t.Fatalf("TLS direct-plain target = %v, want slotDestructor", use.target) + } + if _, required := fixture.requiredPlain[slotDestructor]; !required { + t.Fatal("slotDestructor did not enter the exact scheduler-stack callback island") + } + if _, trusted := fixture.requiredPlain[callback]; trusted { + t.Fatal("descriptor target callback incorrectly entered the trusted no-preempt island") + } + + plan, err := fixture.analyze(coro.SSAConfig{MaxPlainInstructions: -1}) + if err != nil { + t.Fatal(err) + } + callPlan, ok := plan.CallPlan(dynamicCall) + if !ok || callPlan.Rep != coro.Dispatch || callPlan.Open || !callPlan.MayBeNil || len(callPlan.Targets) != 1 { + t.Fatalf("TLS dynamic CallPlan = %+v, present=%t", callPlan, ok) + } + callbackPlan := functionPlanForBuildTest(t, plan, callback) + if callbackPlan.Effect != coro.NoSuspend || callbackPlan.Exec.Contains(coro.NeedsPreempt) || + callbackPlan.FuncRep != coro.Dispatch || callbackPlan.Primary != coro.PrimaryPlain || callbackPlan.Emission != coro.EmitPlain { + t.Fatalf("TLS callback plan = %+v, want descriptor-backed non-suspending plain body", callbackPlan) + } + destructorPlan := functionPlanForBuildTest(t, plan, slotDestructor) + if destructorPlan.Effect != coro.NoSuspend || destructorPlan.Exec.Contains(coro.NeedsPreempt) || + destructorPlan.FuncRep != coro.DirectPlain || destructorPlan.Emission != coro.EmitPlain { + t.Fatalf("slotDestructor plan = %+v, want exact direct-plain C callback", destructorPlan) + } + + _, err = fixture.analyze(coro.SSAConfig{ + MaxPlainInstructions: -1, + ClassifyClosedDynamicCall: func(_ *ssa.Function, call ssa.CallInstruction) (coro.SSAClosedDynamicCallCertificate, bool, error) { + if call != dynamicCall { + return coro.SSAClosedDynamicCallCertificate{}, false, nil + } + return coro.SSAClosedDynamicCallCertificate{MayBeNil: true}, true, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "conflicts with the frozen compiler proof") { + t.Fatalf("builder certificate override error = %v", err) + } + + _, err = fixture.analyze(coro.SSAConfig{ + MaxPlainInstructions: -1, + ClassifyClosedDynamicCall: func(_ *ssa.Function, call ssa.CallInstruction) (coro.SSAClosedDynamicCallCertificate, bool, error) { + if call != dynamicCall { + return coro.SSAClosedDynamicCallCertificate{}, false, nil + } + return coro.SSAClosedDynamicCallCertificate{MayBeNil: true}, false, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "facts without classifying") { + t.Fatalf("builder unclassified certificate error = %v", err) + } +} + +func TestCoroTLSDestructorNilOnlyProof(t *testing.T) { + fixture := buildRequiredCoroRuntimeFixture(t, coroTLSRuntimeFixtureSource(` +func install() { + handle := Alloc(nil) + handle.ensureSlot(new(slot)) +} +`)) + if len(fixture.closedDynamic) != 1 || len(fixture.directPlain) != 1 { + t.Fatalf("nil-only TLS proof = closed:%d direct:%d, want 1/1", len(fixture.closedDynamic), len(fixture.directPlain)) + } + var dynamicCall ssa.CallInstruction + for call, certificate := range fixture.closedDynamic { + dynamicCall = call + if !certificate.MayBeNil || len(certificate.Targets) != 0 { + t.Fatalf("nil-only TLS certificate = %+v", certificate) + } + } + plan, err := fixture.analyze(coro.SSAConfig{MaxPlainInstructions: -1}) + if err != nil { + t.Fatal(err) + } + callPlan, ok := plan.CallPlan(dynamicCall) + if !ok || callPlan.Rep != coro.Dispatch || callPlan.Open || !callPlan.MayBeNil || len(callPlan.Targets) != 0 { + t.Fatalf("nil-only TLS CallPlan = %+v, present=%t", callPlan, ok) + } + if got := functionPlanForBuildTest(t, plan, fixture.pkg.Func("slotDestructor")); got.Effect != coro.NoSuspend || got.FuncRep != coro.DirectPlain { + t.Fatalf("nil-only slotDestructor plan = %+v", got) + } +} + +func TestCoroTLSDestructorProofFailsClosed(t *testing.T) { + tests := []struct { + name string + extra string + want string + }{ + { + name: "unknown write", + extra: ` +func poison(s *slot, destructor func(*int)) { s.destructor = destructor } +func callback(*int) {} +`, + want: "unknown non-nil write", + }, + { + name: "field address escape", + extra: ` +func leak(s *slot) unsafe.Pointer { return unsafe.Pointer(&s.destructor) } +func callback(*int) {} +`, + want: "field address escapes", + }, + { + name: "unsafe aggregate write", + extra: ` +func poison(s *slot, destructor func(*int)) { + ptr := unsafe.Pointer(s) + *(*func(*int))(ptr) = destructor +} +func callback(*int) {} +`, + want: "crosses unsafe conversion", + }, + { + name: "unknown opaque ingress", + extra: ` +func publish(ptr unsafe.Pointer) *slot { return (*slot)(ptr) } +func callback(*int) {} +`, + want: "crosses unsafe conversion", + }, + { + name: "mutating root range helper", + extra: ` +func (s *slot) rootRange() unsafe.Pointer { + ptr := unsafe.Pointer(s) + *(*func(*int))(ptr) = callback + return ptr +} +func callback(*int) {} +`, + want: "crosses unsafe conversion", + }, + { + name: "named pointer escape", + extra: ` +type slotPointer *slot +func leak(s *slot) slotPointer { return slotPointer(s) } +func callback(*int) {} +`, + want: "crosses named pointer conversion", + }, + { + name: "whole aggregate overwrite", + extra: ` +func overwrite(dst *slot, src slot) { *dst = src } +func callback(*int) {} +`, + want: "whole-value write", + }, + { + name: "foreign pointer escape", + extra: ` +func foreign(*slot) +func publish(s *slot) { foreign(s) } +func callback(*int) {} +`, + want: "non-Go callee", + }, + { + name: "interface escape", + extra: ` +func box(handle Handle) any { return handle } +func callback(*int) {} +`, + want: "escapes through interface conversion", + }, + { + name: "multiple targets", + extra: ` +func callback(*int) {} +func other(*int) {} +func install() { + first := Alloc(callback) + first.ensureSlot(new(slot)) + second := Alloc(other) + second.ensureSlot(new(slot)) +} +`, + want: "multiple non-nil targets", + }, + { + name: "captured target", + extra: ` +func install() { + value := 1 + handle := Alloc(func(out *int) { *out = value }) + handle.ensureSlot(new(slot)) +} +`, + want: "not nil or one exact no-capture function", + }, + { + name: "open forwarded target", + extra: ` +func forward(destructor func(*int)) { _ = Alloc(destructor) } +func callback(*int) {} +func install() { go forward(callback) } +`, + want: "not nil or one exact no-capture function", + }, + { + name: "allocator function escape", + extra: ` +var indirectAlloc = Alloc +func forward(destructor func(*int)) { _ = indirectAlloc(destructor) } +func callback(*int) {} +func install() { + handle := Alloc(callback) + handle.ensureSlot(new(slot)) +} +`, + want: "allocator function escapes", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := buildCoroTLSRuntimePlanError(t, coroTLSRuntimeFixtureSource(test.extra)) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("TLS proof error = %v, want substring %q", err, test.want) + } + }) + } +} + +func TestCoroTLSDestructorTargetMustRemainAtomic(t *testing.T) { + for _, test := range []struct { + name string + callback string + }{ + {name: "suspends", callback: `var channel chan int; func callback(*int) { <-channel }`}, + {name: "needs preemption", callback: `func callback(*int) { for {} }`}, + } { + t.Run(test.name, func(t *testing.T) { + fixture := buildRequiredCoroRuntimeFixture(t, coroTLSRuntimeFixtureSource(test.callback)) + _, err := fixture.analyze(coro.SSAConfig{MaxPlainInstructions: -1}) + if err == nil || (!strings.Contains(err.Error(), "non-suspending plain body") && + !strings.Contains(err.Error(), "non-suspending descriptor-backed plain body")) { + t.Fatalf("non-atomic TLS destructor error = %v", err) + } + }) + } +} + +func coroTLSRuntimeFixtureSource(extra string) string { + base := ` +//llgo:type C +type CCallback func(*slot) + +func installC(CCallback) {} + +type Handle struct { destructor func(*int) } +type slot struct { + value int + destructor func(*int) +} + +func Alloc(destructor func(*int)) Handle { + installC(CCallback(slotDestructor)) + var handle Handle + handle.destructor = destructor + return handle +} + +func (handle Handle) ensureSlot(dst *slot) { + dst.destructor = handle.destructor +} + +func slotDestructor(dst *slot) { + if dst.destructor != nil { + dst.destructor(&dst.value) + } + dst.destructor = nil +} +` + if strings.Contains(extra, "func install()") { + return base + extra + } + return base + extra + ` +func install() { + handle := Alloc(callback) + handle.ensureSlot(new(slot)) +} +` +} + +func buildCoroTLSRuntimePlanError(t *testing.T, body string) error { + t.Helper() + source := "package runtime\n" + if strings.Contains(body, "unsafe.") { + source += "import \"unsafe\"\n" + } + source += ` +func __llgo_coro_program_begin_v1() { install() } +func __llgo_coro_program_run_v1() {} +func __llgo_coro_frame_alloc_v1() {} +func __llgo_coro_frame_publish_v1() {} +func __llgo_coro_await_prepare_v1() {} +func __llgo_coro_complete_prepare_v1() {} +func __llgo_coro_frame_free_v1() {} +` + body + ssaPkg, files := buildCoroPlanTestPackage(t, llssa.PkgRuntime, source, nil) + prog := llssa.NewProgram(nil) + t.Cleanup(prog.Dispose) + cl.ParsePkgSyntax(prog, ssaPkg.Pkg, files) + emission, err := cl.PrepareEmissionUniverse(prog, nil, []cl.EmissionPackage{{ + SSA: ssaPkg, Files: files, Identity: llssa.PkgRuntime, + }}) + if err != nil { + t.Fatal(err) + } + ssaEmission, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, emission.Functions()) + if err != nil { + t.Fatal(err) + } + ctx := &context{ + prog: prog, + buildConf: &Config{EnableCoroProgramBootstrapRun: true}, + coroEmission: emission, + coroSSAEmission: ssaEmission, + coroTLSDestructorFixturePkg: llssa.PkgRuntime, + } + _, _, _, _, err = requiredCoroProgramRuntimePlan(ctx) + return err +} From b2e703caed8064b72f0c36c09d8386e81929764b Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 16:44:25 +0800 Subject: [PATCH 06/32] ci(coro): cover plain TLS dispatch --- .github/workflows/coroutine.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coroutine.yml b/.github/workflows/coroutine.yml index f2b6778ac0..0cc8ab4d72 100644 --- a/.github/workflows/coroutine.yml +++ b/.github/workflows/coroutine.yml @@ -97,8 +97,11 @@ jobs: go test -tags='${{ matrix.tags }}' ./internal/build -run '^Test(BuildCoroPlanInstallsArchiveDigest|CoroutinePlanInputsAffectFingerprint|CoroEntryResolutionUsesPlanMatchedPackageCache|CoroPlanDigestMetadataUsesEffectiveLLVMTarget|CoroPhysicalABICacheRegistrationPreservesCollectedFuncInfo)$' -count=1 go test -tags='${{ matrix.tags }}' ./cl -run '^Test(CompilationCoroABIIdentityValidation|CoroEntryResolutionCacheRegistrationWithDigest|CoroPhysicalABICacheRegistrationPreservesPhysicalMetadata)$' -count=1 - - name: Test coroutine physical ABI lowering - run: go test -tags='${{ matrix.tags }}' -v ./cl -run '^TestCoro(LeafPhysicalABI|PhysicalABI|ChildAwaitPhysicalABIV1|ExplicitAsyncRootFactoryV1|ExplicitRootFactoryV1|ExplicitPlain|RootPackageAnchorV1)' -count=1 + - name: Test coroutine physical ABI and function dispatch lowering + run: go test -tags='${{ matrix.tags }}' -v ./cl -run '^TestCoro(LeafPhysicalABI|PhysicalABI|ChildAwaitPhysicalABIV1|ExplicitAsyncRootFactoryV1|ExplicitRootFactoryV1|ExplicitPlain|RootPackageAnchorV1|PlainDispatch)' -count=1 + + - name: Test coroutine TLS function dispatch proof + run: go test -tags='${{ matrix.tags }}' -v ./internal/build -run '^TestCoroTLS' -count=1 - name: Test coroutine registry and control integration run: go test -tags='${{ matrix.tags }}' -v ./internal/build -run '^Test(CollectLinkedCoroRootAnchors|ActiveCoroABIVersions|BuildCoroPlanErrors|CoroProgramManifest.*|CoroProgramBootstrap.*|SelectCoroProgramBootstrap.*|GenMainModule.*Coro.*)$' -count=1 From cecd715351112c1ccd2a38272caf4209c66f4717 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 16:53:18 +0800 Subject: [PATCH 07/32] feat(coro): trust frozen TLS C leaves --- internal/build/build.go | 49 ++++++++-- internal/build/coro_tls_destructor_test.go | 106 +++++++++++++++++++++ 2 files changed, 149 insertions(+), 6 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 7a33aca9a2..199a653635 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1310,11 +1310,16 @@ func exactCoroStaticFunctionValue(ctx *context, value ssa.Value) (*ssa.Function, } // provenCoroDirectPlainStaticClosure accepts only a closed Go body whose calls -// are direct, statically resolved emitted bodies (or builtins). Dynamic calls, -// go/defer, bodyless leaves, captured closures, and unresolved aliases remain -// on the ordinary Dispatch path. Effect and representation are independently -// checked after fixed-point analysis; this prefilter only establishes that it -// is sound to seed the candidate's bounded scheduler-stack island. +// are direct, statically resolved emitted bodies (or builtins). An exact frozen +// C declaration may terminate the closure only for the compiler-owned TLS +// callback whose field-flow proof supplied one of closedDynamic's calls. The +// declaration then enters requiredPlain and is classified through the same +// frozen IgnoreBody/ExternalKnown path as the compiler runtime ABI. Dynamic +// calls, go/defer, other bodyless leaves, captured closures, and unresolved +// aliases remain on the ordinary Dispatch path. Effect and representation are +// independently checked after fixed-point analysis; this prefilter only +// establishes that it is sound to seed the candidate's bounded scheduler-stack +// island. func provenCoroDirectPlainStaticClosure(ctx *context, target *ssa.Function, closedDynamic map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate) ([]*ssa.Function, bool, error) { if ctx == nil || ctx.coroEmission == nil || target == nil || len(target.FreeVars) != 0 { return nil, false, nil @@ -1327,8 +1332,10 @@ func provenCoroDirectPlainStaticClosure(ctx *context, target *ssa.Function, clos return nil, false, nil } seen := make(map[*ssa.Function]struct{}) + seenCLeaves := make(map[*ssa.Function]struct{}) queue := []*ssa.Function{target} closure := make([]*ssa.Function, 0, 4) + tlsCallback := provenCoroTLSDirectPlainClosureRoot(ctx, target, closedDynamic) for head := 0; head < len(queue); head++ { function := queue[head] if _, ok := seen[function]; ok { @@ -1375,7 +1382,18 @@ func provenCoroDirectPlainStaticClosure(ctx *context, target *ssa.Function, clos return nil, false, err } if !calleeGoBody { - return nil, false, nil + background, classified, err := ctx.coroEmission.FunctionBackground(callee) + if err != nil { + return nil, false, err + } + if !tlsCallback || !classified || background != llssa.InC { + return nil, false, nil + } + if _, ok := seenCLeaves[callee]; !ok { + seenCLeaves[callee] = struct{}{} + closure = append(closure, callee) + } + continue } if _, ok := seen[callee]; !ok { queue = append(queue, callee) @@ -1386,6 +1404,25 @@ func provenCoroDirectPlainStaticClosure(ctx *context, target *ssa.Function, clos return closure, true, nil } +// provenCoroTLSDirectPlainClosureRoot binds the frozen-C-leaf exception to the +// exact callback body audited by proveCoroTLSDestructorClosedDynamicCalls. A +// certificate reachable only through a helper is insufficient: otherwise an +// unrelated user callback could call that helper and inherit the exception. +func provenCoroTLSDirectPlainClosureRoot(ctx *context, target *ssa.Function, closedDynamic map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate) bool { + if !coroTLSFunctionInOwnedPackage(ctx, target) { + return false + } + for call := range closedDynamic { + if call == nil || call.Parent() != target || call.Common() == nil || call.Common().IsInvoke() { + continue + } + if _, ordinary := call.(*ssa.Call); ordinary { + return true + } + } + return false +} + func buildCoroPlanDigestMetadata(ctx *context) (coro.PlanDigestMetadata, error) { if ctx == nil || ctx.buildConf == nil { return coro.PlanDigestMetadata{}, fmt.Errorf("missing build context") diff --git a/internal/build/coro_tls_destructor_test.go b/internal/build/coro_tls_destructor_test.go index 744f41cd98..df904bca84 100644 --- a/internal/build/coro_tls_destructor_test.go +++ b/internal/build/coro_tls_destructor_test.go @@ -296,6 +296,112 @@ func TestCoroTLSDestructorTargetMustRemainAtomic(t *testing.T) { } } +func TestCoroTLSDestructorDirectPlainClosureFrozenCLeaf(t *testing.T) { + source := coroTLSRuntimeFixtureSource(` +func hiddenFallback() {} +func callback(*int) {} +func ordinaryCaller() { ordinaryC() } +`) + source = strings.Replace(source, "func slotDestructor(dst *slot) {", ` +//llgo:link tlsCLeaf C.tls_c_leaf +func tlsCLeaf() { hiddenFallback() } + +//llgo:link ordinaryC C.ordinary_c +func ordinaryC() + +func slotDestructor(dst *slot) { + tlsCLeaf() +`, 1) + fixture := buildRequiredCoroRuntimeFixture(t, source) + if len(fixture.directPlain) != 1 { + t.Fatalf("TLS direct-plain callbacks = %d, want 1", len(fixture.directPlain)) + } + slotDestructor := fixture.pkg.Func("slotDestructor") + tlsCLeaf := fixture.pkg.Func("tlsCLeaf") + if use := fixture.directPlain[0]; use.target != slotDestructor { + t.Fatalf("TLS direct-plain target = %v, want slotDestructor", use.target) + } + if _, required := fixture.requiredPlain[tlsCLeaf]; !required { + t.Fatal("exact frozen TLS C leaf did not enter the required plain island") + } + if _, required := fixture.requiredPlain[fixture.pkg.Func("hiddenFallback")]; required { + t.Fatal("ignored C fallback body leaked into the required plain island") + } + if _, required := fixture.requiredPlain[fixture.pkg.Func("ordinaryC")]; required { + t.Fatal("ordinary external C declaration entered the required plain island") + } + + plan, err := fixture.analyze(coro.SSAConfig{MaxPlainInstructions: -1}) + if err != nil { + t.Fatal(err) + } + callbackPlan := functionPlanForBuildTest(t, plan, slotDestructor) + if callbackPlan.External != coro.Defined || callbackPlan.Effect != coro.NoSuspend || callbackPlan.Exec.Contains(coro.NeedsPreempt) || + callbackPlan.FuncRep != coro.DirectPlain || callbackPlan.Primary != coro.PrimaryPlain || callbackPlan.Emission != coro.EmitPlain { + t.Fatalf("TLS callback plan = %+v, want post-plan validated direct plain", callbackPlan) + } + leafPlan := functionPlanForBuildTest(t, plan, tlsCLeaf) + if !plan.IgnoresBody(tlsCLeaf) || leafPlan.External != coro.ExternalKnown || leafPlan.Effect != coro.NoSuspend || + leafPlan.Exec.Contains(coro.BlockForeign|coro.NeedsPreempt) || leafPlan.FuncRep != coro.DirectPlain || leafPlan.Emission != coro.EmitExternal { + t.Fatalf("TLS C leaf plan = %+v, ignored=%t; want exact compatible-known declaration", leafPlan, plan.IgnoresBody(tlsCLeaf)) + } + ordinaryC := functionPlanForBuildTest(t, plan, fixture.pkg.Func("ordinaryC")) + if ordinaryC.External != coro.ExternalUnknownForeign || !ordinaryC.Exec.Contains(coro.BlockForeign|coro.IRQUnsafe) { + t.Fatalf("ordinary C declaration plan = %+v, want unknown foreign", ordinaryC) + } +} + +func TestCoroTLSDestructorDirectPlainClosureCLeafFailsClosed(t *testing.T) { + t.Run("user callback", func(t *testing.T) { + fixture := buildRequiredCoroRuntimeFixture(t, coroTLSRuntimeFixtureSource(` +//llgo:link userCLeaf C.user_c_leaf +func userCLeaf() +func callback(*int) {} +func userCallback(*slot) { userCLeaf() } +func install() { + handle := Alloc(callback) + handle.ensureSlot(new(slot)) + installC(CCallback(userCallback)) +} +`)) + if len(fixture.directPlain) != 1 || fixture.directPlain[0].target != fixture.pkg.Func("slotDestructor") { + t.Fatalf("direct-plain callbacks = %+v, want only compiler-owned slotDestructor", fixture.directPlain) + } + if _, required := fixture.requiredPlain[fixture.pkg.Func("userCallback")]; required { + t.Fatal("user callback inherited the TLS C-leaf exception") + } + if _, required := fixture.requiredPlain[fixture.pkg.Func("userCLeaf")]; required { + t.Fatal("user callback C leaf entered the required plain island") + } + if _, ok, err := provenCoroDirectPlainStaticClosure(fixture.ctx, fixture.pkg.Func("userCallback"), fixture.closedDynamic); err != nil || ok { + t.Fatalf("user callback closure proof = ok:%t err:%v, want false/nil", ok, err) + } + }) + + t.Run("non C declaration", func(t *testing.T) { + source := coroTLSRuntimeFixtureSource(` +func unknownManaged() +func callback(*int) {} +`) + source = strings.Replace(source, "func slotDestructor(dst *slot) {", `func slotDestructor(dst *slot) { + unknownManaged() +`, 1) + fixture := buildRequiredCoroRuntimeFixture(t, source) + if len(fixture.closedDynamic) != 1 { + t.Fatalf("TLS closed dynamic certificates = %d, want 1", len(fixture.closedDynamic)) + } + if len(fixture.directPlain) != 0 { + t.Fatalf("unknown managed leaf produced direct-plain callback uses: %+v", fixture.directPlain) + } + if _, required := fixture.requiredPlain[fixture.pkg.Func("unknownManaged")]; required { + t.Fatal("non-C declaration entered the required plain island") + } + if _, ok, err := provenCoroDirectPlainStaticClosure(fixture.ctx, fixture.pkg.Func("slotDestructor"), fixture.closedDynamic); err != nil || ok { + t.Fatalf("non-C leaf closure proof = ok:%t err:%v, want false/nil", ok, err) + } + }) +} + func coroTLSRuntimeFixtureSource(extra string) string { base := ` //llgo:type C From 8629ea5f09090a0405d4f5d40d17666a0f87702c Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 17:16:31 +0800 Subject: [PATCH 08/32] feat(coro): demand ABI method table entries --- .github/workflows/coroutine.yml | 4 +- cl/emission_abi_demand.go | 6 +- cl/emission_method_link_test.go | 160 +++++++++++++++++++++- cl/emission_universe.go | 227 +++++++++++++++++++++++-------- internal/build/build.go | 46 +++++++ internal/build/coro_plan_test.go | 95 +++++++++++++ internal/coro/ssa_plan.go | 58 ++++++++ internal/coro/ssa_plan_test.go | 65 +++++++++ 8 files changed, 597 insertions(+), 64 deletions(-) diff --git a/.github/workflows/coroutine.yml b/.github/workflows/coroutine.yml index 0cc8ab4d72..8af6cad57d 100644 --- a/.github/workflows/coroutine.yml +++ b/.github/workflows/coroutine.yml @@ -77,7 +77,7 @@ jobs: - name: Test coroutine build integration if: matrix.llvm == 19 - run: go test ./internal/build -run 'Test(CoroPlanBuilderRunsBeforeCodegenWithoutChangingIR|CoroPlanInputCanonicalizesPatchedRoot|CoroPlanInputElidesOnlyFrontendNoInitCalls|RequiredCoroProgramRuntimePlanPlainClosureAndConflicts|ActiveCoroABIVersions|BuildCoroPlanErrors|CoroEntryResolutionUsesPlanMatchedPackageCache|CoroEntryResolutionBuildsPreparedRuntimePackages|CoroRuntimeLinkRequirements|CoroEmissionCoverageStopsBeforeAnyPackageCodegen|CoroUnsupportedEntryResolutionReturnsErrorBeforeCodegen|CoroEmissionUniverseAcceptsModeTestVariants|CoroProgramBootstrapRejectsInvalidRootsBeforePackageCodegen)$' -count=1 + run: go test ./internal/build -run 'Test(CoroPlanBuilderRunsBeforeCodegenWithoutChangingIR|CoroPlanInputCanonicalizesPatchedRoot|CoroPlanInputElidesOnlyFrontendNoInitCalls|CoroPlanInputOwnsFrozenDemandReferences|RequiredCoroProgramRuntimePlanPlainClosureAndConflicts|ActiveCoroABIVersions|BuildCoroPlanErrors|CoroEntryResolutionUsesPlanMatchedPackageCache|CoroEntryResolutionBuildsPreparedRuntimePackages|CoroRuntimeLinkRequirements|CoroEmissionCoverageStopsBeforeAnyPackageCodegen|CoroUnsupportedEntryResolutionReturnsErrorBeforeCodegen|CoroEmissionUniverseAcceptsModeTestVariants|CoroProgramBootstrapRejectsInvalidRootsBeforePackageCodegen)$' -count=1 - name: Test coroutine compiler integration if: matrix.llvm == 19 @@ -98,7 +98,7 @@ jobs: go test -tags='${{ matrix.tags }}' ./cl -run '^Test(CompilationCoroABIIdentityValidation|CoroEntryResolutionCacheRegistrationWithDigest|CoroPhysicalABICacheRegistrationPreservesPhysicalMetadata)$' -count=1 - name: Test coroutine physical ABI and function dispatch lowering - run: go test -tags='${{ matrix.tags }}' -v ./cl -run '^TestCoro(LeafPhysicalABI|PhysicalABI|ChildAwaitPhysicalABIV1|ExplicitAsyncRootFactoryV1|ExplicitRootFactoryV1|ExplicitPlain|RootPackageAnchorV1|PlainDispatch)' -count=1 + run: go test -tags='${{ matrix.tags }}' -v ./cl -run '^Test(Coro(LeafPhysicalABI|PhysicalABI|ChildAwaitPhysicalABIV1|ExplicitAsyncRootFactoryV1|ExplicitRootFactoryV1|ExplicitPlain|RootPackageAnchorV1|PlainDispatch)|EmissionUniverse(ActiveABIMethodTablesUseFrozenWrapperSymbols|ABIMethodDemandReferencesAreExactRecursiveAndOwnerScoped))' -count=1 - name: Test coroutine TLS function dispatch proof run: go test -tags='${{ matrix.tags }}' -v ./internal/build -run '^TestCoroTLS' -count=1 diff --git a/cl/emission_abi_demand.go b/cl/emission_abi_demand.go index 294a1fc631..ee8a34ffd0 100644 --- a/cl/emission_abi_demand.go +++ b/cl/emission_abi_demand.go @@ -253,7 +253,11 @@ func (u *EmissionUniverse) materializeABITypeDemand(fn *ssa.Function, owner *pre if exactState, exactFromPatch, known := u.typeProvenance(owner, typ); known { methodState, methodFromPatch = exactState, exactFromPatch } - return u.selectABITypeMethods(owner, typ, methodState, methodFromPatch) + methods, err := u.selectABITypeMethods(owner, typ, methodState, methodFromPatch) + if err != nil { + return err + } + return u.recordABIMethodReferences(fn, methods) }) } diff --git a/cl/emission_method_link_test.go b/cl/emission_method_link_test.go index 79633f7484..3c7a325cb5 100644 --- a/cl/emission_method_link_test.go +++ b/cl/emission_method_link_test.go @@ -47,20 +47,24 @@ func Value() any { return struct{ Base }{} } if err != nil { t.Fatal(err) } - roots := coro.Roots{{Function: pkg.ssa.Func("Value"), Demand: coro.SyncDemand}} + value := pkg.ssa.Func("Value") + references, err := universe.CoroDemandReferences(value) + if err != nil { + t.Fatal(err) + } foundPromoted := false - for _, fn := range universe.Functions() { + for _, fn := range references { if wrapperKind(fn) == "promoted" && fn.Name() == "M" { - roots = append(roots, coro.Root{Function: fn, Demand: coro.SyncDemand}) foundPromoted = true } } if !foundPromoted { - t.Fatal("prepared universe has no promoted M wrapper to demand") + t.Fatal("Value has no frozen promoted M method-table reference") } - plan, err := coro.AnalyzeSSA(testProg.ssa, roots, coro.SSAConfig{ - EmissionUniverse: ssaUniverse, - FunctionIDs: universe.FunctionIDConfig(), + plan, err := coro.AnalyzeSSA(testProg.ssa, coro.Roots{{Function: value, Demand: coro.SyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: universe.FunctionIDConfig(), + ClassifyDemandReferences: universe.CoroDemandReferences, }) if err != nil { t.Fatal(err) @@ -117,6 +121,148 @@ func Value() any { return struct{ Base }{} } } } +func TestEmissionUniverseABIMethodDemandReferencesAreExactRecursiveAndOwnerScoped(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/methoddemand", `package methoddemand +var channel chan int +type Base struct{} +func (Base) Suspend() { <-channel } +type Leaf struct{} +func (Leaf) Plain() {} +type Outer struct { Base; Child Leaf; Next *Outer } +type Dead struct{} +func (Dead) Method() {} +func Demanded() any { return Outer{} } +func Unreachable() any { return Dead{} } +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{ + SSA: pkg.ssa, Files: []*ast.File{pkg.file}, + }}) + if err != nil { + t.Fatal(err) + } + demanded := pkg.ssa.Func("Demanded") + unreachable := pkg.ssa.Func("Unreachable") + references, err := universe.CoroDemandReferences(demanded) + if err != nil { + t.Fatal(err) + } + if len(references) == 0 { + t.Fatal("Demanded has no frozen ABI method references") + } + for index := 1; index < len(references); index++ { + if universe.functionSortKey(references[index-1]) > universe.functionSortKey(references[index]) { + t.Fatalf("ABI method references are not deterministically sorted at %d", index) + } + } + repeated, err := universe.CoroDemandReferences(demanded) + if err != nil { + t.Fatal(err) + } + if len(repeated) != len(references) { + t.Fatalf("repeated reference count = %d; want %d", len(repeated), len(references)) + } + for index := range references { + if repeated[index] != references[index] { + t.Fatalf("repeated reference %d = %v; want exact %v", index, repeated[index], references[index]) + } + } + references[0] = nil + defensive, err := universe.CoroDemandReferences(demanded) + if err != nil { + t.Fatal(err) + } + if len(defensive) == 0 || defensive[0] == nil { + t.Fatal("caller mutation changed frozen ABI method references") + } + + memberType := func(name string) types.Type { + member, ok := pkg.ssa.Members[name].(*ssa.Type) + if !ok { + t.Fatalf("SSA member %q is not a type", name) + } + return member.Type() + } + exactMethod := func(typ types.Type, name string) *ssa.Function { + selection := emissionABIDemandMethodSelection(t, testProg.ssa, typ, name) + method := testProg.ssa.MethodValue(selection) + if method == nil { + t.Fatalf("method %s.%s has no SSA value", typ, name) + } + canonical, ok := universe.Resolve(method) + if !ok { + t.Fatalf("method %s.%s is outside the frozen universe", typ, name) + } + return canonical + } + hasReference := func(list []*ssa.Function, target *ssa.Function) bool { + for _, candidate := range list { + if candidate == target { + return true + } + } + return false + } + outer := memberType("Outer") + valueTFN := exactMethod(outer, "Suspend") + pointerIFN := exactMethod(types.NewPointer(outer), "Suspend") + if valueTFN == pointerIFN { + t.Fatal("promoted value tfn and pointer ifn unexpectedly share one SSA wrapper") + } + leafPlain := exactMethod(memberType("Leaf"), "Plain") + for label, target := range map[string]*ssa.Function{ + "value tfn": valueTFN, "pointer ifn": pointerIFN, "recursive field method": leafPlain, + } { + if !hasReference(defensive, target) { + t.Fatalf("Demanded references omit exact %s %v", label, target) + } + } + + deadReferences, err := universe.CoroDemandReferences(unreachable) + if err != nil { + t.Fatal(err) + } + deadMethod := exactMethod(memberType("Dead"), "Method") + if !hasReference(deadReferences, deadMethod) { + t.Fatalf("Unreachable references omit exact Dead.Method %v", deadMethod) + } + + ssaUniverse, err := coro.NewSSAEmissionUniverse(testProg.ssa, universe.Functions()) + if err != nil { + t.Fatal(err) + } + plan, err := coro.AnalyzeSSA(testProg.ssa, coro.Roots{{Function: demanded, Demand: coro.SyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: universe.FunctionIDConfig(), + ClassifyDemandReferences: universe.CoroDemandReferences, + }) + if err != nil { + t.Fatal(err) + } + demandedPlan, _ := plan.FunctionPlan(demanded) + if demandedPlan.Effect != coro.NoSuspend || demandedPlan.Emission != coro.EmitPlain { + t.Fatalf("Demanded plan = %+v, method addresses must not propagate effects", demandedPlan) + } + for _, target := range []*ssa.Function{valueTFN, pointerIFN} { + methodPlan, ok := plan.FunctionPlan(target) + if !ok || methodPlan.Demand != coro.AsyncDemand || methodPlan.Emission != coro.EmitCoroutine || methodPlan.Primary != coro.PrimaryCoroutine { + t.Fatalf("suspending method %v plan = %+v, present=%v; want demanded coroutine entry", target, methodPlan, ok) + } + } + deadPlan, ok := plan.FunctionPlan(deadMethod) + if !ok || deadPlan.Demand != coro.NoDemand || deadPlan.Emission != coro.EmitNone { + t.Fatalf("unreachable method plan = %+v, present=%v; want no over-emission", deadPlan, ok) + } + + delete(universe.required, leafPlain) + if _, err := universe.CoroDemandReferences(demanded); err == nil || !strings.Contains(err.Error(), "outside the frozen emission universe") { + t.Fatalf("missing frozen ABI method error = %v", err) + } +} + func TestEmissionUniverseActiveGenericLocalMethodFormsUseFrozenSymbols(t *testing.T) { testProg := newEmissionTestProgram() pkg := testProg.addPackage(t, "example.com/emission/genericmethodlink", `package genericmethodlink diff --git a/cl/emission_universe.go b/cl/emission_universe.go index 77204bd9a5..54b178f37b 100644 --- a/cl/emission_universe.go +++ b/cl/emission_universe.go @@ -78,26 +78,27 @@ type EmissionUniverse struct { byPath map[string]*preparedEmissionPackage pathDup map[string]bool - functions []*ssa.Function - required map[*ssa.Function]none - aliases map[*ssa.Function]*ssa.Function - fnOwners map[*ssa.Function]*preparedEmissionPackage - fnStates map[*ssa.Function]emissionFunctionState - functionKinds map[emissionFunctionOwnerKey]int - intrinsicOps map[emissionFunctionOwnerKey]int - finalKeys map[emissionFunctionOwnerKey]string - physicalNames map[emissionFunctionOwnerKey]string - linkOnceNames map[*ssa.Function]string - callWraps map[intrinsicWrapperKey]*ssa.Function - callWrapInfo map[*ssa.Function]intrinsicWrapperKey - syntheticKeys map[*ssa.Function]string - linkIdentities map[*ssa.Function]string - excluded map[*ssa.Function]none - materialized map[*ssa.Function]none - useOwners map[*ssa.Function]map[*preparedEmissionPackage]none - ownerStates map[*ssa.Function]map[*preparedEmissionPackage]emissionFunctionState - materializedOwners map[*ssa.Function]map[*preparedEmissionPackage]none - ownerStateErr error + functions []*ssa.Function + required map[*ssa.Function]none + aliases map[*ssa.Function]*ssa.Function + fnOwners map[*ssa.Function]*preparedEmissionPackage + fnStates map[*ssa.Function]emissionFunctionState + functionKinds map[emissionFunctionOwnerKey]int + intrinsicOps map[emissionFunctionOwnerKey]int + finalKeys map[emissionFunctionOwnerKey]string + physicalNames map[emissionFunctionOwnerKey]string + linkOnceNames map[*ssa.Function]string + callWraps map[intrinsicWrapperKey]*ssa.Function + callWrapInfo map[*ssa.Function]intrinsicWrapperKey + syntheticKeys map[*ssa.Function]string + linkIdentities map[*ssa.Function]string + excluded map[*ssa.Function]none + materialized map[*ssa.Function]none + useOwners map[*ssa.Function]map[*preparedEmissionPackage]none + ownerStates map[*ssa.Function]map[*preparedEmissionPackage]emissionFunctionState + materializedOwners map[*ssa.Function]map[*preparedEmissionPackage]none + ownerStateErr error + abiMethodReferences map[*ssa.Function]map[*ssa.Function]none localGenericMu sync.Mutex localGenericTypes map[*types.Named]emissionLocalGenericType @@ -153,34 +154,35 @@ func PrepareEmissionUniverse(prog llssa.Program, patches Patches, inputs []Emiss } identities := make(map[string]*ssa.Package, len(inputs)) u := &EmissionUniverse{ - prog: prog, - patches: patches, - packages: make(map[*ssa.Package]*preparedEmissionPackage, len(inputs)), - byTypes: make(map[*types.Package]*preparedEmissionPackage, len(inputs)*3), - typesDup: make(map[*types.Package]bool), - byPath: make(map[string]*preparedEmissionPackage, len(inputs)), - pathDup: make(map[string]bool), - required: make(map[*ssa.Function]none), - aliases: make(map[*ssa.Function]*ssa.Function), - fnOwners: make(map[*ssa.Function]*preparedEmissionPackage), - fnStates: make(map[*ssa.Function]emissionFunctionState), - functionKinds: make(map[emissionFunctionOwnerKey]int), - intrinsicOps: make(map[emissionFunctionOwnerKey]int), - finalKeys: make(map[emissionFunctionOwnerKey]string), - physicalNames: make(map[emissionFunctionOwnerKey]string), - linkOnceNames: make(map[*ssa.Function]string), - callWraps: make(map[intrinsicWrapperKey]*ssa.Function), - callWrapInfo: make(map[*ssa.Function]intrinsicWrapperKey), - syntheticKeys: make(map[*ssa.Function]string), - linkIdentities: make(map[*ssa.Function]string), - excluded: make(map[*ssa.Function]none), - materialized: make(map[*ssa.Function]none), - useOwners: make(map[*ssa.Function]map[*preparedEmissionPackage]none), - ownerStates: make(map[*ssa.Function]map[*preparedEmissionPackage]emissionFunctionState), - materializedOwners: make(map[*ssa.Function]map[*preparedEmissionPackage]none), - localGenericTypes: make(map[*types.Named]emissionLocalGenericType), - localGenericOwners: make(map[*types.Named]*ssa.Function), - genericNamedTypes: make(map[*types.Named]*types.Named), + prog: prog, + patches: patches, + packages: make(map[*ssa.Package]*preparedEmissionPackage, len(inputs)), + byTypes: make(map[*types.Package]*preparedEmissionPackage, len(inputs)*3), + typesDup: make(map[*types.Package]bool), + byPath: make(map[string]*preparedEmissionPackage, len(inputs)), + pathDup: make(map[string]bool), + required: make(map[*ssa.Function]none), + aliases: make(map[*ssa.Function]*ssa.Function), + fnOwners: make(map[*ssa.Function]*preparedEmissionPackage), + fnStates: make(map[*ssa.Function]emissionFunctionState), + functionKinds: make(map[emissionFunctionOwnerKey]int), + intrinsicOps: make(map[emissionFunctionOwnerKey]int), + finalKeys: make(map[emissionFunctionOwnerKey]string), + physicalNames: make(map[emissionFunctionOwnerKey]string), + linkOnceNames: make(map[*ssa.Function]string), + callWraps: make(map[intrinsicWrapperKey]*ssa.Function), + callWrapInfo: make(map[*ssa.Function]intrinsicWrapperKey), + syntheticKeys: make(map[*ssa.Function]string), + abiMethodReferences: make(map[*ssa.Function]map[*ssa.Function]none), + linkIdentities: make(map[*ssa.Function]string), + excluded: make(map[*ssa.Function]none), + materialized: make(map[*ssa.Function]none), + useOwners: make(map[*ssa.Function]map[*preparedEmissionPackage]none), + ownerStates: make(map[*ssa.Function]map[*preparedEmissionPackage]emissionFunctionState), + materializedOwners: make(map[*ssa.Function]map[*preparedEmissionPackage]none), + localGenericTypes: make(map[*types.Named]emissionLocalGenericType), + localGenericOwners: make(map[*types.Named]*ssa.Function), + genericNamedTypes: make(map[*types.Named]*types.Named), } for i, input := range inputs { if input.SSA == nil || input.SSA.Prog == nil || input.SSA.Pkg == nil { @@ -347,6 +349,85 @@ func (u *EmissionUniverse) Functions() []*ssa.Function { return append([]*ssa.Function(nil), u.functions...) } +// CoroDemandReferences returns the exact functions whose addresses are +// embedded in runtime ABI method tables emitted while lowering owner. These +// are demand-only references: a demanded owner must materialize the selected +// tfn/ifn bodies, but taking their addresses does not inherit their effects. +// +// The map is completed together with the emission universe, before coroutine +// analysis or LLVM codegen. Results are sorted by the frozen frontend identity +// and defensively copied so callers cannot change the universe after freezing. +func (u *EmissionUniverse) CoroDemandReferences(owner *ssa.Function) ([]*ssa.Function, error) { + if u == nil { + return nil, fmt.Errorf("coroutine ABI method references require a prepared emission universe") + } + if owner == nil { + return nil, fmt.Errorf("coroutine ABI method references require an exact owner function") + } + canonical := u.canonicalAlias(owner) + if canonical == nil { + return nil, fmt.Errorf("coroutine ABI method reference owner %q has cyclic canonical aliases", owner.Name()) + } + if canonical != owner { + return nil, fmt.Errorf("coroutine ABI method reference owner %q is not the exact canonical function", owner.Name()) + } + if _, frozen := u.required[owner]; !frozen { + return nil, fmt.Errorf("coroutine ABI method reference owner %q is outside the frozen emission universe", owner.Name()) + } + targets := make([]*ssa.Function, 0, len(u.abiMethodReferences[owner])) + for target := range u.abiMethodReferences[owner] { + if target == nil { + return nil, fmt.Errorf("coroutine ABI method reference owner %q has a nil target", owner.Name()) + } + if canonicalTarget := u.canonicalAlias(target); canonicalTarget == nil || canonicalTarget != target { + return nil, fmt.Errorf("coroutine ABI method reference owner %q has a non-canonical target %q", owner.Name(), target.Name()) + } + if _, frozen := u.required[target]; !frozen { + return nil, fmt.Errorf("coroutine ABI method reference owner %q targets method %q outside the frozen emission universe", owner.Name(), target.Name()) + } + targets = append(targets, target) + } + sort.SliceStable(targets, func(i, j int) bool { + return u.functionSortKey(targets[i]) < u.functionSortKey(targets[j]) + }) + return targets, nil +} + +func (u *EmissionUniverse) recordABIMethodReferences(owner *ssa.Function, targets []*ssa.Function) error { + if owner == nil { + return fmt.Errorf("prepare emission universe: ABI method references have no owner") + } + owner = u.canonicalAlias(owner) + if owner == nil { + return fmt.Errorf("prepare emission universe: ABI method reference owner has cyclic canonical aliases") + } + if _, frozen := u.required[owner]; !frozen { + return fmt.Errorf("prepare emission universe: ABI method reference owner %q is outside the emission universe", owner.Name()) + } + if len(targets) == 0 { + return nil + } + references := u.abiMethodReferences[owner] + if references == nil { + references = make(map[*ssa.Function]none) + u.abiMethodReferences[owner] = references + } + for _, target := range targets { + if target == nil { + return fmt.Errorf("prepare emission universe: ABI method reference owner %q has a nil target", owner.Name()) + } + target = u.canonicalAlias(target) + if target == nil { + return fmt.Errorf("prepare emission universe: ABI method reference owner %q reached a cyclic target alias", owner.Name()) + } + if _, frozen := u.required[target]; !frozen { + return fmt.Errorf("prepare emission universe: ABI method reference owner %q targets method %q outside the emission universe", owner.Name(), target.Name()) + } + references[target] = none{} + } + return nil +} + // Contains reports whether fn is an exact canonical required function. func (u *EmissionUniverse) Contains(fn *ssa.Function) bool { if u == nil || fn == nil { @@ -770,7 +851,7 @@ func (u *EmissionUniverse) selectTypeMethods(prepared *preparedEmissionPackage, return nil } -func (u *EmissionUniverse) selectABITypeMethods(prepared *preparedEmissionPackage, typ types.Type, state pkgState, fromPatch bool) error { +func (u *EmissionUniverse) selectABITypeMethods(prepared *preparedEmissionPackage, typ types.Type, state pkgState, fromPatch bool) ([]*ssa.Function, error) { base := types.Unalias(typ) for { pointer, ok := base.(*types.Pointer) @@ -785,16 +866,54 @@ func (u *EmissionUniverse) selectABITypeMethods(prepared *preparedEmissionPackag packageNamed = obj != nil && obj.Pkg() != nil && obj.Parent() == obj.Pkg().Scope() } mset := u.goProg.MethodSets.MethodSet(typ) + methods := make([]*ssa.Function, 0, mset.Len()*2) + selectMethod := func(selection *types.Selection) error { + fn := u.goProg.MethodValue(selection) + if fn == nil { + return fmt.Errorf("prepare emission universe: ABI method table for %v has no SSA implementation for method %q", typ, selection.Obj().Name()) + } + if !packageNamed || functionNeedsLinkOnce(fn) { + if err := u.selectFunction(prepared, fn, state, fromPatch); err != nil { + return err + } + } + fn = u.canonicalAlias(fn) + if fn == nil { + return fmt.Errorf("prepare emission universe: ABI method table for %v reached a cyclic method alias", typ) + } + if _, frozen := u.required[fn]; !frozen { + return fmt.Errorf("prepare emission universe: ABI method table for %v references method %q outside the frozen emission universe", typ, fn.Name()) + } + methods = append(methods, fn) + return nil + } for index := 0; index < mset.Len(); index++ { - fn := u.goProg.MethodValue(mset.At(index)) - if fn == nil || packageNamed && !functionNeedsLinkOnce(fn) { + selection := mset.At(index) + if err := selectMethod(selection); err != nil { + return nil, err + } + + // abiUncommonMethods uses the pointer-receiver method value as ifn for + // every value-receiver selection. Freeze that exact wrapper alongside + // tfn instead of assuming a later pointer descriptor happens to request + // it as an unrelated side effect. + sig, ok := selection.Type().(*types.Signature) + if !ok || sig.Recv() == nil { + return nil, fmt.Errorf("prepare emission universe: ABI method table for %v has a non-method selection %q", typ, selection.Obj().Name()) + } + if _, pointerReceiver := selection.Recv().Underlying().(*types.Pointer); pointerReceiver { continue } - if err := u.selectFunction(prepared, fn, state, fromPatch); err != nil { - return err + pointerReceiver := types.NewPointer(sig.Recv().Type()) + pointerSelection := u.goProg.MethodSets.MethodSet(pointerReceiver).Lookup(selection.Obj().Pkg(), selection.Obj().Name()) + if pointerSelection == nil { + return nil, fmt.Errorf("prepare emission universe: ABI method table for %v cannot resolve pointer ifn for method %q", typ, selection.Obj().Name()) + } + if err := selectMethod(pointerSelection); err != nil { + return nil, err } } - return nil + return stableUniqueFunctions(methods), nil } func (u *EmissionUniverse) functionProvenance(prepared *preparedEmissionPackage, fn *ssa.Function) (pkgState, bool) { diff --git a/internal/build/build.go b/internal/build/build.go index 199a653635..b605628bb4 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -137,6 +137,7 @@ type CoroPlanInput struct { augmentFunctionIDs func(coro.FunctionIDConfig) coro.FunctionIDConfig functionBackground func(*ssa.Function) (llssa.Background, bool, error) intrinsicCallSemantics func(ssa.CallInstruction) (cl.CoroIntrinsicCallSemantics, bool, error) + demandReferences func(*ssa.Function) ([]*ssa.Function, error) requiredRoots coro.Roots requiredPlain map[*ssa.Function]struct{} requiredDirectPlain []requiredCoroDirectPlainCallArgument @@ -340,6 +341,30 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. return cloneCoroClosedDynamicCallCertificate(compilerCertificate), true, nil } } + if in.demandReferences != nil || config.ClassifyDemandReferences != nil { + classifyDemandReferences := config.ClassifyDemandReferences + config.ClassifyDemandReferences = func(owner *ssa.Function) ([]*ssa.Function, error) { + var compilerTargets []*ssa.Function + var err error + if in.demandReferences != nil { + compilerTargets, err = in.demandReferences(owner) + if err != nil { + return nil, fmt.Errorf("classify frozen frontend demand references for %q: %w", owner.Name(), err) + } + } + compilerTargets = append([]*ssa.Function(nil), compilerTargets...) + if classifyDemandReferences != nil { + requested, err := classifyDemandReferences(owner) + if err != nil { + return nil, err + } + if !sameExactCoroFunctionReferences(requested, compilerTargets) { + return nil, fmt.Errorf("builder demand references in %q conflict with the frozen frontend method-table references", owner.Name()) + } + } + return compilerTargets, nil + } + } if in.augmentFunctionIDs != nil { config.FunctionIDs = in.augmentFunctionIDs(config.FunctionIDs) } @@ -361,6 +386,26 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. return plan, err } +func sameExactCoroFunctionReferences(left, right []*ssa.Function) bool { + if len(left) != len(right) { + return false + } + counts := make(map[*ssa.Function]int, len(left)) + for _, fn := range left { + if fn == nil || counts[fn] != 0 { + return false + } + counts[fn] = 1 + } + for _, fn := range right { + if fn == nil || counts[fn] != 1 { + return false + } + counts[fn] = 0 + } + return true +} + // frontendElidesNoInitCall mirrors cl.context.funcKind: the frontend emits no // call for the synthetic zero-argument init of a noinit/decl package. Treating // this as an unresolved managed call would invent an OpaqueSuspend edge that @@ -984,6 +1029,7 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { input.resolveFunction = ctx.coroEmission.Resolve input.functionBackground = ctx.coroEmission.FunctionBackground input.intrinsicCallSemantics = ctx.coroEmission.CoroIntrinsicCallSiteSemantics + input.demandReferences = ctx.coroEmission.CoroDemandReferences input.augmentFunctionIDs = func(config coro.FunctionIDConfig) coro.FunctionIDConfig { if ctx.buildConf.EnableCoroEntryResolution { if config.CoroABI == "" { diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index f8d765b578..d853a24311 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -1198,6 +1198,101 @@ func g() {} } } +func TestCoroPlanInputOwnsFrozenDemandReferences(t *testing.T) { + ssaPkg, _ := buildCoroPlanTestPackage(t, "example.com/demandrefs", `package demandrefs +func owner() {} +func method() {} +func method2() {} +func extra() {} +func alias() {} +`, nil) + owner := ssaPkg.Func("owner") + method := ssaPkg.Func("method") + method2 := ssaPkg.Func("method2") + extra := ssaPkg.Func("extra") + alias := ssaPkg.Func("alias") + universe, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, []*ssa.Function{owner, method, method2, extra}) + if err != nil { + t.Fatal(err) + } + frozen := []*ssa.Function{method, method2} + input := CoroPlanInput{ + Program: ssaPkg.Prog, + EmissionUniverse: universe, + resolveFunction: func(fn *ssa.Function) (*ssa.Function, bool) { + if fn == alias { + return method, true + } + return fn, universe.Contains(fn) + }, + demandReferences: func(fn *ssa.Function) ([]*ssa.Function, error) { + if fn == owner { + return frozen, nil + } + return nil, nil + }, + } + roots := coro.Roots{{Function: owner, Demand: coro.SyncDemand}} + plan, err := input.Analyze(roots, coro.SSAConfig{}) + if err != nil { + t.Fatal(err) + } + for _, target := range []*ssa.Function{method, method2} { + methodPlan, ok := plan.FunctionPlan(target) + if !ok || methodPlan.Demand != coro.SyncDemand || methodPlan.Emission != coro.EmitPlain { + t.Fatalf("frozen method %s plan = %+v, present=%v", target.Name(), methodPlan, ok) + } + } + // A completed exact-pointer plan does not retain the frontend callback's + // backing slice. + frozen[0] = extra + methodPlan, ok := plan.FunctionPlan(method) + if !ok || methodPlan.Demand != coro.SyncDemand { + t.Fatalf("callback slice mutation changed completed method plan = %+v, present=%v", methodPlan, ok) + } + frozen[0] = method + + tests := []struct { + name string + requested []*ssa.Function + }{ + {name: "missing", requested: []*ssa.Function{method}}, + {name: "extra", requested: []*ssa.Function{method, method2, extra}}, + {name: "alias", requested: []*ssa.Function{method, alias}}, + {name: "duplicate", requested: []*ssa.Function{method, method}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := input.Analyze(roots, coro.SSAConfig{ + ClassifyDemandReferences: func(fn *ssa.Function) ([]*ssa.Function, error) { + if fn == owner { + return test.requested, nil + } + return nil, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "conflict with the frozen frontend method-table references") { + t.Fatalf("builder %s demand-reference error = %v", test.name, err) + } + }) + } + + accepted, err := input.Analyze(roots, coro.SSAConfig{ + ClassifyDemandReferences: func(fn *ssa.Function) ([]*ssa.Function, error) { + if fn == owner { + return []*ssa.Function{method2, method}, nil + } + return nil, nil + }, + }) + if err != nil { + t.Fatalf("builder exact frozen reference was rejected: %v", err) + } + if got, ok := accepted.FunctionPlan(method); !ok || got.Demand != coro.SyncDemand { + t.Fatalf("accepted exact reference plan = %+v, present=%v", got, ok) + } +} + func TestActiveCoroABIVersions(t *testing.T) { tests := []struct { name string diff --git a/internal/coro/ssa_plan.go b/internal/coro/ssa_plan.go index 5e36c12fcf..d40796ba77 100644 --- a/internal/coro/ssa_plan.go +++ b/internal/coro/ssa_plan.go @@ -192,6 +192,17 @@ type SSAConfig struct { // callback is trusted to have rejected every unknown physical write or escape // that could reach the exact value loaded at call. ClassifyClosedDynamicCall func(caller *ssa.Function, call ssa.CallInstruction) (SSAClosedDynamicCallCertificate, bool, error) + + // ClassifyDemandReferences supplies exact function addresses that the + // frontend implicitly embeds while lowering one function body, even though + // they are not operands in that body's SSA instructions. Runtime ABI method + // tables are the canonical example. These references propagate entry demand + // only; they do not propagate suspend effects or execution flags. + // + // Every returned target must be a non-nil exact canonical member of the + // effective emission universe. AnalyzeSSA calls the classifier only for + // owned, non-ignored bodies and copies the returned slice before use. + ClassifyDemandReferences func(owner *ssa.Function) ([]*ssa.Function, error) } // SSAFunctionPlan binds an immutable FunctionPlan back to its SSA function. @@ -799,6 +810,9 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err if err := addSSAReferenceEdges(graph, bodyFunctions, includedSet, ids, flow); err != nil { return nil, err } + if err := addSSAClassifiedDemandReferences(graph, bodyFunctions, includedSet, ids, canonicalizer, config); err != nil { + return nil, err + } base, err := graph.Analyze() if err != nil { @@ -835,6 +849,50 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err return result, nil } +func addSSAClassifiedDemandReferences( + graph *Graph, + functions []*ssa.Function, + included map[*ssa.Function]bool, + ids map[*ssa.Function]FunctionID, + canonicalizer *ssaFunctionCanonicalizer, + config SSAConfig, +) error { + if config.ClassifyDemandReferences == nil { + return nil + } + for _, owner := range functions { + targets, err := config.ClassifyDemandReferences(owner) + if err != nil { + return fmt.Errorf("coro: classify demand-only references in %q: %w", owner.Name(), err) + } + // The classifier owns its backing storage. Copy before validation so + // analysis never retains a frontend-owned slice. + targets = append([]*ssa.Function(nil), targets...) + for index, target := range targets { + if target == nil { + return fmt.Errorf("coro: demand-only reference %d in %q has a nil target", index, owner.Name()) + } + if target.Prog != owner.Prog { + return fmt.Errorf("coro: demand-only reference %d in %q targets function %q from another SSA program", index, owner.Name(), target.Name()) + } + canonical, resolved, resolveErr := canonicalizer.resolve(target) + if resolveErr != nil { + return fmt.Errorf("coro: resolve demand-only target %q in %q: %w", target.Name(), owner.Name(), resolveErr) + } + if !resolved || canonical == nil || !included[canonical] { + return fmt.Errorf("coro: demand-only target %q in %q is outside the effective emission universe", target.Name(), owner.Name()) + } + if canonical != target { + return fmt.Errorf("coro: demand-only target %q in %q is not the exact canonical function", target.Name(), owner.Name()) + } + if err := graph.AddReference(ReferenceEdge{Owner: ids[owner], Target: ids[target]}); err != nil { + return fmt.Errorf("coro: add demand-only function reference from %q to %q: %w", owner.Name(), target.Name(), err) + } + } + } + return nil +} + // addSSAReferenceEdges projects known function values used by demanded bodies // into demand-only graph edges. Every CallInstruction callee operand is skipped: // static and dynamic invocation are already represented by CallEdge and must diff --git a/internal/coro/ssa_plan_test.go b/internal/coro/ssa_plan_test.go index e9418b508b..001598652c 100644 --- a/internal/coro/ssa_plan_test.go +++ b/internal/coro/ssa_plan_test.go @@ -358,6 +358,71 @@ func deadOwner() { } } +func TestAnalyzeSSAClassifiedDemandReferencesAreOwnerScopedAndFailClosed(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "implicit_references.go", `package coroid + +var channel chan int + +func owner() {} +func deadOwner() {} +func suspendingMethod() { <-channel } +func deadMethod() {} +func outsideFrozenUniverse() {} +`) + owner := packageFunction(t, pkg, "owner") + deadOwner := packageFunction(t, pkg, "deadOwner") + suspending := packageFunction(t, pkg, "suspendingMethod") + deadMethod := packageFunction(t, pkg, "deadMethod") + outside := packageFunction(t, pkg, "outsideFrozenUniverse") + universe, err := NewSSAEmissionUniverse(prog, []*ssa.Function{owner, deadOwner, suspending, deadMethod}) + if err != nil { + t.Fatal(err) + } + plan, err := AnalyzeSSA(prog, Roots{{Function: owner, Demand: SyncDemand}}, SSAConfig{ + EmissionUniverse: universe, + ClassifyDemandReferences: func(fn *ssa.Function) ([]*ssa.Function, error) { + switch fn { + case owner: + return []*ssa.Function{suspending}, nil + case deadOwner: + return []*ssa.Function{deadMethod}, nil + default: + return nil, nil + } + }, + }) + if err != nil { + t.Fatal(err) + } + ownerPlan := functionPlanFor(t, plan, owner) + if ownerPlan.Effect != NoSuspend || ownerPlan.Emission != EmitPlain { + t.Fatalf("owner plan = %+v, demand-only method address inherited its effect", ownerPlan) + } + suspendingPlan := functionPlanFor(t, plan, suspending) + if suspendingPlan.Demand != AsyncDemand || suspendingPlan.Emission != EmitCoroutine || suspendingPlan.Primary != PrimaryCoroutine { + t.Fatalf("suspending method plan = %+v, want demanded coroutine entry", suspendingPlan) + } + for _, fn := range []*ssa.Function{deadOwner, deadMethod} { + got := functionPlanFor(t, plan, fn) + if got.Demand != NoDemand || got.Emission != EmitNone { + t.Fatalf("unreachable %s plan = %+v, want no demand and no emission", fn.Name(), got) + } + } + + _, err = AnalyzeSSA(prog, Roots{{Function: owner, Demand: SyncDemand}}, SSAConfig{ + EmissionUniverse: universe, + ClassifyDemandReferences: func(fn *ssa.Function) ([]*ssa.Function, error) { + if fn == owner { + return []*ssa.Function{outside}, nil + } + return nil, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "outside the effective emission universe") { + t.Fatalf("missing frozen method error = %v", err) + } +} + func TestAnalyzeSSADynamicOpenAndClosedWorld(t *testing.T) { prog, pkg := buildCoroTestSSA(t, "source.go", `package coroid From c1b5f56a9eeb68c3938cfba3968bb71894530218 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 18:15:32 +0800 Subject: [PATCH 09/32] feat(coro): freeze frontend lowered call edges --- cl/emission_lowered_call_test.go | 84 ++++++++++++++++ cl/emission_universe.go | 105 ++++++++++++++++++++ internal/build/build.go | 49 ++++++++++ internal/build/coro_plan_test.go | 104 ++++++++++++++++++++ internal/coro/plan_digest.go | 63 ++++++++++-- internal/coro/plan_digest_test.go | 67 ++++++++++++- internal/coro/ssa_plan.go | 126 ++++++++++++++++++++++++ internal/coro/ssa_plan_test.go | 157 ++++++++++++++++++++++++++++++ 8 files changed, 745 insertions(+), 10 deletions(-) create mode 100644 cl/emission_lowered_call_test.go diff --git a/cl/emission_lowered_call_test.go b/cl/emission_lowered_call_test.go new file mode 100644 index 0000000000..3bf1d32bc5 --- /dev/null +++ b/cl/emission_lowered_call_test.go @@ -0,0 +1,84 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "go/ast" + "strings" + "testing" +) + +func TestEmissionUniverseCoroLoweredCallsAreExactSortedAndFailClosed(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/loweredcalls", `package loweredcalls +func Owner() {} +func First() {} +func Second() {} +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{ + SSA: pkg.ssa, Files: []*ast.File{pkg.file}, + }}) + if err != nil { + t.Fatal(err) + } + owner := pkg.ssa.Func("Owner") + first := pkg.ssa.Func("First") + second := pkg.ssa.Func("Second") + if err := universe.recordCoroLoweredCall(owner, "runtime.second", second); err != nil { + t.Fatal(err) + } + if err := universe.recordCoroLoweredCall(owner, "runtime.first", first); err != nil { + t.Fatal(err) + } + if err := universe.recordCoroLoweredCall(owner, "runtime.first", first); err != nil { + t.Fatalf("idempotent lowered call: %v", err) + } + + calls, err := universe.CoroLoweredCalls(owner) + if err != nil { + t.Fatal(err) + } + if len(calls) != 2 || calls[0].LogicalName != "runtime.first" || calls[0].Target != first || calls[1].LogicalName != "runtime.second" || calls[1].Target != second { + t.Fatalf("lowered calls = %+v, want sorted exact mappings", calls) + } + calls[0].Target = second + target, ok, err := universe.ResolveCoroLoweredCall(owner, "runtime.first") + if err != nil || !ok || target != first { + t.Fatalf("ResolveCoroLoweredCall(runtime.first) = %v, %v, %v", target, ok, err) + } + if target, ok, err := universe.ResolveCoroLoweredCall(owner, "runtime.missing"); err != nil || ok || target != nil { + t.Fatalf("ResolveCoroLoweredCall(runtime.missing) = %v, %v, %v", target, ok, err) + } + + if err := universe.recordCoroLoweredCall(owner, "runtime.first", second); err == nil || !strings.Contains(err.Error(), "resolves to both") { + t.Fatalf("conflicting logical helper error = %v", err) + } + if err := universe.recordCoroLoweredCall(owner, "", first); err == nil || !strings.Contains(err.Error(), "invalid logical name") { + t.Fatalf("empty logical helper error = %v", err) + } + if err := universe.recordCoroLoweredCall(owner, "runtime.nil", nil); err == nil || !strings.Contains(err.Error(), "nil target") { + t.Fatalf("nil lowered helper error = %v", err) + } + if _, err := universe.CoroLoweredCalls(nil); err == nil || !strings.Contains(err.Error(), "exact owner") { + t.Fatalf("nil lowered-call owner error = %v", err) + } +} diff --git a/cl/emission_universe.go b/cl/emission_universe.go index 54b178f37b..3641640f1a 100644 --- a/cl/emission_universe.go +++ b/cl/emission_universe.go @@ -28,6 +28,7 @@ import ( "strconv" "strings" "sync" + "unicode/utf8" "github.com/goplus/llgo/cl/ssawrap" "github.com/goplus/llgo/internal/coro" @@ -99,6 +100,7 @@ type EmissionUniverse struct { materializedOwners map[*ssa.Function]map[*preparedEmissionPackage]none ownerStateErr error abiMethodReferences map[*ssa.Function]map[*ssa.Function]none + loweredCalls map[*ssa.Function]map[string]*ssa.Function localGenericMu sync.Mutex localGenericTypes map[*types.Named]emissionLocalGenericType @@ -174,6 +176,7 @@ func PrepareEmissionUniverse(prog llssa.Program, patches Patches, inputs []Emiss callWrapInfo: make(map[*ssa.Function]intrinsicWrapperKey), syntheticKeys: make(map[*ssa.Function]string), abiMethodReferences: make(map[*ssa.Function]map[*ssa.Function]none), + loweredCalls: make(map[*ssa.Function]map[string]*ssa.Function), linkIdentities: make(map[*ssa.Function]string), excluded: make(map[*ssa.Function]none), materialized: make(map[*ssa.Function]none), @@ -428,6 +431,108 @@ func (u *EmissionUniverse) recordABIMethodReferences(owner *ssa.Function, target return nil } +// CoroLoweredCalls returns the exact managed helper calls that frontend +// lowering inserts into owner without a corresponding source SSA call. Records +// are sorted by logical helper identity and defensively copied. The mapping is +// frozen together with the emission universe, before coroutine analysis and +// LLVM codegen. +func (u *EmissionUniverse) CoroLoweredCalls(owner *ssa.Function) ([]coro.SSALoweredCall, error) { + if u == nil { + return nil, fmt.Errorf("coroutine lowered calls require a prepared emission universe") + } + if owner == nil { + return nil, fmt.Errorf("coroutine lowered calls require an exact owner function") + } + canonical := u.canonicalAlias(owner) + if canonical == nil { + return nil, fmt.Errorf("coroutine lowered-call owner %q has cyclic canonical aliases", owner.Name()) + } + if canonical != owner { + return nil, fmt.Errorf("coroutine lowered-call owner %q is not the exact canonical function", owner.Name()) + } + if _, frozen := u.required[owner]; !frozen { + return nil, fmt.Errorf("coroutine lowered-call owner %q is outside the frozen emission universe", owner.Name()) + } + byName := u.loweredCalls[owner] + calls := make([]coro.SSALoweredCall, 0, len(byName)) + for logicalName, target := range byName { + if logicalName == "" || !utf8.ValidString(logicalName) || strings.IndexByte(logicalName, 0) >= 0 { + return nil, fmt.Errorf("coroutine lowered-call owner %q has invalid logical name %q", owner.Name(), logicalName) + } + if target == nil { + return nil, fmt.Errorf("coroutine lowered call %q in %q has a nil target", logicalName, owner.Name()) + } + if canonicalTarget := u.canonicalAlias(target); canonicalTarget == nil || canonicalTarget != target { + return nil, fmt.Errorf("coroutine lowered call %q in %q has a non-canonical target %q", logicalName, owner.Name(), target.Name()) + } + if _, frozen := u.required[target]; !frozen { + return nil, fmt.Errorf("coroutine lowered call %q in %q targets helper %q outside the frozen emission universe", logicalName, owner.Name(), target.Name()) + } + calls = append(calls, coro.SSALoweredCall{LogicalName: logicalName, Target: target}) + } + sort.Slice(calls, func(i, j int) bool { + return calls[i].LogicalName < calls[j].LogicalName + }) + return calls, nil +} + +// ResolveCoroLoweredCall resolves one exact frozen helper mapping. It is used +// by codegen to recover the same canonical target that analysis projected as a +// real call edge, without rediscovering it from an LLVM symbol name. +func (u *EmissionUniverse) ResolveCoroLoweredCall(owner *ssa.Function, logicalName string) (*ssa.Function, bool, error) { + calls, err := u.CoroLoweredCalls(owner) + if err != nil { + return nil, false, err + } + index := sort.Search(len(calls), func(index int) bool { + return calls[index].LogicalName >= logicalName + }) + if index == len(calls) || calls[index].LogicalName != logicalName { + return nil, false, nil + } + return calls[index].Target, true, nil +} + +// recordCoroLoweredCall freezes one compiler-inserted helper mapping while the +// emission universe is being materialized. Repeated uses of the same logical +// helper in one owner are idempotent; resolving that identity to two exact +// targets fails closed. +func (u *EmissionUniverse) recordCoroLoweredCall(owner *ssa.Function, logicalName string, target *ssa.Function) error { + if owner == nil { + return fmt.Errorf("prepare emission universe: lowered call has no owner") + } + if logicalName == "" || !utf8.ValidString(logicalName) || strings.IndexByte(logicalName, 0) >= 0 { + return fmt.Errorf("prepare emission universe: lowered call in %q has invalid logical name %q", owner.Name(), logicalName) + } + owner = u.canonicalAlias(owner) + if owner == nil { + return fmt.Errorf("prepare emission universe: lowered-call owner has cyclic canonical aliases") + } + if _, frozen := u.required[owner]; !frozen { + return fmt.Errorf("prepare emission universe: lowered-call owner %q is outside the emission universe", owner.Name()) + } + if target == nil { + return fmt.Errorf("prepare emission universe: lowered call %q in %q has a nil target", logicalName, owner.Name()) + } + target = u.canonicalAlias(target) + if target == nil { + return fmt.Errorf("prepare emission universe: lowered call %q in %q reached a cyclic target alias", logicalName, owner.Name()) + } + if _, frozen := u.required[target]; !frozen { + return fmt.Errorf("prepare emission universe: lowered call %q in %q targets helper %q outside the emission universe", logicalName, owner.Name(), target.Name()) + } + byName := u.loweredCalls[owner] + if byName == nil { + byName = make(map[string]*ssa.Function) + u.loweredCalls[owner] = byName + } + if previous := byName[logicalName]; previous != nil && previous != target { + return fmt.Errorf("prepare emission universe: lowered call %q in %q resolves to both %q and %q", logicalName, owner.Name(), previous.Name(), target.Name()) + } + byName[logicalName] = target + return nil +} + // Contains reports whether fn is an exact canonical required function. func (u *EmissionUniverse) Contains(fn *ssa.Function) bool { if u == nil || fn == nil { diff --git a/internal/build/build.go b/internal/build/build.go index b605628bb4..e26482a627 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -138,6 +138,7 @@ type CoroPlanInput struct { functionBackground func(*ssa.Function) (llssa.Background, bool, error) intrinsicCallSemantics func(ssa.CallInstruction) (cl.CoroIntrinsicCallSemantics, bool, error) demandReferences func(*ssa.Function) ([]*ssa.Function, error) + loweredCalls func(*ssa.Function) ([]coro.SSALoweredCall, error) requiredRoots coro.Roots requiredPlain map[*ssa.Function]struct{} requiredDirectPlain []requiredCoroDirectPlainCallArgument @@ -365,6 +366,30 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. return compilerTargets, nil } } + if in.loweredCalls != nil || config.ClassifyLoweredCalls != nil { + classifyLoweredCalls := config.ClassifyLoweredCalls + config.ClassifyLoweredCalls = func(owner *ssa.Function) ([]coro.SSALoweredCall, error) { + var compilerCalls []coro.SSALoweredCall + var err error + if in.loweredCalls != nil { + compilerCalls, err = in.loweredCalls(owner) + if err != nil { + return nil, fmt.Errorf("classify frozen frontend lowered calls for %q: %w", owner.Name(), err) + } + } + compilerCalls = append([]coro.SSALoweredCall(nil), compilerCalls...) + if classifyLoweredCalls != nil { + requested, err := classifyLoweredCalls(owner) + if err != nil { + return nil, err + } + if !sameExactCoroLoweredCalls(requested, compilerCalls) { + return nil, fmt.Errorf("builder lowered calls in %q conflict with the frozen frontend helper calls", owner.Name()) + } + } + return compilerCalls, nil + } + } if in.augmentFunctionIDs != nil { config.FunctionIDs = in.augmentFunctionIDs(config.FunctionIDs) } @@ -406,6 +431,29 @@ func sameExactCoroFunctionReferences(left, right []*ssa.Function) bool { return true } +func sameExactCoroLoweredCalls(left, right []coro.SSALoweredCall) bool { + if len(left) != len(right) { + return false + } + byName := make(map[string]*ssa.Function, len(left)) + for _, call := range left { + if call.LogicalName == "" || call.Target == nil { + return false + } + if _, duplicate := byName[call.LogicalName]; duplicate { + return false + } + byName[call.LogicalName] = call.Target + } + for _, call := range right { + if call.LogicalName == "" || call.Target == nil || byName[call.LogicalName] != call.Target { + return false + } + delete(byName, call.LogicalName) + } + return len(byName) == 0 +} + // frontendElidesNoInitCall mirrors cl.context.funcKind: the frontend emits no // call for the synthetic zero-argument init of a noinit/decl package. Treating // this as an unresolved managed call would invent an OpaqueSuspend edge that @@ -1030,6 +1078,7 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { input.functionBackground = ctx.coroEmission.FunctionBackground input.intrinsicCallSemantics = ctx.coroEmission.CoroIntrinsicCallSiteSemantics input.demandReferences = ctx.coroEmission.CoroDemandReferences + input.loweredCalls = ctx.coroEmission.CoroLoweredCalls input.augmentFunctionIDs = func(config coro.FunctionIDConfig) coro.FunctionIDConfig { if ctx.buildConf.EnableCoroEntryResolution { if config.CoroABI == "" { diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index d853a24311..2b9753e1db 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -1293,6 +1293,110 @@ func alias() {} } } +func TestCoroPlanInputOwnsFrozenLoweredCalls(t *testing.T) { + ssaPkg, _ := buildCoroPlanTestPackage(t, "example.com/loweredcalls", `package loweredcalls +var channel chan int +func owner() {} +func helper() { <-channel } +func helper2() {} +func extra() {} +func alias() {} +`, nil) + owner := ssaPkg.Func("owner") + helper := ssaPkg.Func("helper") + helper2 := ssaPkg.Func("helper2") + extra := ssaPkg.Func("extra") + alias := ssaPkg.Func("alias") + universe, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, []*ssa.Function{owner, helper, helper2, extra}) + if err != nil { + t.Fatal(err) + } + frozen := []coro.SSALoweredCall{ + {LogicalName: "runtime.helper", Target: helper}, + {LogicalName: "runtime.helper2", Target: helper2}, + } + input := CoroPlanInput{ + Program: ssaPkg.Prog, + EmissionUniverse: universe, + resolveFunction: func(fn *ssa.Function) (*ssa.Function, bool) { + if fn == alias { + return helper, true + } + return fn, universe.Contains(fn) + }, + loweredCalls: func(fn *ssa.Function) ([]coro.SSALoweredCall, error) { + if fn == owner { + return frozen, nil + } + return nil, nil + }, + } + roots := coro.Roots{{Function: owner, Demand: coro.SyncDemand}} + plan, err := input.Analyze(roots, coro.SSAConfig{MaxPlainInstructions: -1}) + if err != nil { + t.Fatal(err) + } + ownerPlan, ok := plan.FunctionPlan(owner) + if !ok || !ownerPlan.Effect.Contains(coro.MayPark) || ownerPlan.Emission != coro.EmitCoroutine { + t.Fatalf("owner plan = %+v, present=%v; frozen lowered call did not propagate effect", ownerPlan, ok) + } + if got, ok := plan.FunctionPlan(helper); !ok || got.Demand != coro.AsyncDemand || got.Emission != coro.EmitCoroutine { + t.Fatalf("suspending helper plan = %+v, present=%v", got, ok) + } + // The completed plan owns both the record slice and its exact mapping. + frozen[0].Target = extra + if target, ok := plan.ResolveLoweredCall(owner, "runtime.helper"); !ok || target != helper { + t.Fatalf("callback slice mutation changed completed lowered call: %v, %v", target, ok) + } + frozen[0].Target = helper + + tests := []struct { + name string + requested []coro.SSALoweredCall + }{ + {name: "missing", requested: []coro.SSALoweredCall{{LogicalName: "runtime.helper", Target: helper}}}, + {name: "extra", requested: []coro.SSALoweredCall{{LogicalName: "runtime.helper", Target: helper}, {LogicalName: "runtime.helper2", Target: helper2}, {LogicalName: "runtime.extra", Target: extra}}}, + {name: "renamed", requested: []coro.SSALoweredCall{{LogicalName: "runtime.renamed", Target: helper}, {LogicalName: "runtime.helper2", Target: helper2}}}, + {name: "retargeted", requested: []coro.SSALoweredCall{{LogicalName: "runtime.helper", Target: helper2}, {LogicalName: "runtime.helper2", Target: helper}}}, + {name: "alias", requested: []coro.SSALoweredCall{{LogicalName: "runtime.helper", Target: alias}, {LogicalName: "runtime.helper2", Target: helper2}}}, + {name: "duplicate", requested: []coro.SSALoweredCall{{LogicalName: "runtime.helper", Target: helper}, {LogicalName: "runtime.helper", Target: helper}}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := input.Analyze(roots, coro.SSAConfig{ + ClassifyLoweredCalls: func(fn *ssa.Function) ([]coro.SSALoweredCall, error) { + if fn == owner { + return test.requested, nil + } + return nil, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "conflict with the frozen frontend helper calls") { + t.Fatalf("builder %s lowered-call error = %v", test.name, err) + } + }) + } + + accepted, err := input.Analyze(roots, coro.SSAConfig{ + MaxPlainInstructions: -1, + ClassifyLoweredCalls: func(fn *ssa.Function) ([]coro.SSALoweredCall, error) { + if fn == owner { + return []coro.SSALoweredCall{ + {LogicalName: "runtime.helper2", Target: helper2}, + {LogicalName: "runtime.helper", Target: helper}, + }, nil + } + return nil, nil + }, + }) + if err != nil { + t.Fatalf("builder exact frozen lowered calls were rejected: %v", err) + } + if got := accepted.LoweredCalls(owner); len(got) != 2 || got[0].LogicalName != "runtime.helper" || got[1].LogicalName != "runtime.helper2" { + t.Fatalf("accepted lowered calls = %+v", got) + } +} + func TestActiveCoroABIVersions(t *testing.T) { tests := []struct { name string diff --git a/internal/coro/plan_digest.go b/internal/coro/plan_digest.go index 5dd064bff8..7f652b49fd 100644 --- a/internal/coro/plan_digest.go +++ b/internal/coro/plan_digest.go @@ -31,7 +31,7 @@ import ( // PlanDigestSchema is the independent canonical schema used for archive cache // identity. It is deliberately separate from SummarySchema: summaries remain // diagnostic snapshots, while this document covers every lowering plan site. -const PlanDigestSchema = "llgo.coro.plan-digest.v4" +const PlanDigestSchema = "llgo.coro.plan-digest.v5" // Current experimental ABI identities. Keeping these in the analysis package // gives build, cache, and lowering code one version source of truth. @@ -77,14 +77,15 @@ type PlanDigestMetadata struct { } type planDigestDocument struct { - Schema string `json:"schema"` - FunctionIDSchema string `json:"function_id_schema"` - Metadata PlanDigestMetadata `json:"metadata"` - Roots []planDigestRoot `json:"roots"` - Functions []planDigestFunction `json:"functions"` - Calls []planDigestCall `json:"calls"` - ElidedCalls []planDigestElidedCall `json:"elided_calls,omitempty"` - Values []planDigestValue `json:"values"` + Schema string `json:"schema"` + FunctionIDSchema string `json:"function_id_schema"` + Metadata PlanDigestMetadata `json:"metadata"` + Roots []planDigestRoot `json:"roots"` + Functions []planDigestFunction `json:"functions"` + Calls []planDigestCall `json:"calls"` + LoweredCalls []planDigestLoweredCall `json:"lowered_calls"` + ElidedCalls []planDigestElidedCall `json:"elided_calls,omitempty"` + Values []planDigestValue `json:"values"` } type planDigestRoot struct { @@ -121,6 +122,12 @@ type planDigestCall struct { MayBeNil bool `json:"may_be_nil"` } +type planDigestLoweredCall struct { + Owner FunctionID `json:"owner"` + LogicalName string `json:"logical_name"` + Target FunctionID `json:"target"` +} + type planDigestElidedCall struct { Function FunctionID `json:"function"` Block int `json:"block"` @@ -207,6 +214,11 @@ func (p *SSAPlan) canonicalPlanDigest(metadata PlanDigestMetadata) (planDigestDo return planDigestDocument{}, err } + loweredCalls, err := p.canonicalDigestLoweredCalls() + if err != nil { + return planDigestDocument{}, err + } + document := planDigestDocument{ Schema: PlanDigestSchema, FunctionIDSchema: FunctionIDSchema, @@ -214,6 +226,7 @@ func (p *SSAPlan) canonicalPlanDigest(metadata PlanDigestMetadata) (planDigestDo Roots: roots, Functions: functions, Calls: make([]planDigestCall, 0, len(p.callPlans)), + LoweredCalls: loweredCalls, ElidedCalls: make([]planDigestElidedCall, 0, len(p.elidedCalls)), Values: make([]planDigestValue, 0, len(p.valuePlans)), } @@ -318,6 +331,38 @@ func (p *SSAPlan) canonicalPlanDigest(metadata PlanDigestMetadata) (planDigestDo return document, nil } +func (p *SSAPlan) canonicalDigestLoweredCalls() ([]planDigestLoweredCall, error) { + ret := make([]planDigestLoweredCall, 0) + for owner, calls := range p.loweredCalls { + ownerID, ok := p.byFunction[owner] + if !ok { + return nil, fmt.Errorf("coro: lowered-call owner %q is absent from the plan", owner.Name()) + } + previous := "" + for index, call := range calls { + if call.LogicalName == "" || !utf8.ValidString(call.LogicalName) || strings.IndexByte(call.LogicalName, 0) >= 0 { + return nil, fmt.Errorf("coro: lowered call %d in %q has invalid logical name %q", index, ownerID, call.LogicalName) + } + if index != 0 && previous >= call.LogicalName { + return nil, fmt.Errorf("coro: lowered calls in %q are not in strict logical-name order", ownerID) + } + previous = call.LogicalName + targetID, ok := p.byFunction[call.Target] + if !ok { + return nil, fmt.Errorf("coro: lowered call %q in %q targets a function outside the plan", call.LogicalName, ownerID) + } + ret = append(ret, planDigestLoweredCall{Owner: ownerID, LogicalName: call.LogicalName, Target: targetID}) + } + } + sort.Slice(ret, func(i, j int) bool { + if ret[i].Owner != ret[j].Owner { + return ret[i].Owner < ret[j].Owner + } + return ret[i].LogicalName < ret[j].LogicalName + }) + return ret, nil +} + func (m PlanDigestMetadata) validate() error { required := []struct { name string diff --git a/internal/coro/plan_digest_test.go b/internal/coro/plan_digest_test.go index e0e3db30d6..16b8c82bee 100644 --- a/internal/coro/plan_digest_test.go +++ b/internal/coro/plan_digest_test.go @@ -623,13 +623,78 @@ func TestCoroPlanDigestCanonicalEmptyArrays(t *testing.T) { t.Fatal(err) } text := string(payload) - for _, field := range []string{`"roots":[]`, `"calls":[]`, `"values":[]`} { + for _, field := range []string{`"roots":[]`, `"calls":[]`, `"lowered_calls":[]`, `"values":[]`} { if !strings.Contains(text, field) { t.Fatalf("canonical document %s does not contain %s", text, field) } } } +func TestCoroPlanDigestIncludesExactLoweredCallMapping(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "lowered_digest.go", `package coroid +func root() {} +func first() {} +func second() {} +`) + root := packageFunction(t, pkg, "root") + first := packageFunction(t, pkg, "first") + second := packageFunction(t, pkg, "second") + build := func(calls []SSALoweredCall) *SSAPlan { + t.Helper() + config := planDigestSSAConfig() + config.MaxPlainInstructions = -1 + config.ClassifyLoweredCalls = func(fn *ssa.Function) ([]SSALoweredCall, error) { + if fn == root { + return calls, nil + } + return nil, nil + } + plan, err := AnalyzeSSA(prog, Roots{{Function: root, Demand: SyncDemand}}, config) + if err != nil { + t.Fatal(err) + } + return plan + } + baseline := build([]SSALoweredCall{ + {LogicalName: "runtime.first", Target: first}, + {LogicalName: "runtime.second", Target: second}, + }) + permuted := build([]SSALoweredCall{ + {LogicalName: "runtime.second", Target: second}, + {LogicalName: "runtime.first", Target: first}, + }) + swapped := build([]SSALoweredCall{ + {LogicalName: "runtime.first", Target: second}, + {LogicalName: "runtime.second", Target: first}, + }) + metadata := validPlanDigestMetadata() + baselineDigest, err := baseline.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + permutedDigest, err := permuted.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if baselineDigest != permutedDigest { + t.Fatalf("classifier order changed lowered-call digest:\n%s\n%s", baselineDigest, permutedDigest) + } + swappedDigest, err := swapped.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if baselineDigest == swappedDigest { + t.Fatal("retargeting logical lowered-call identities did not change digest") + } + document, err := baseline.canonicalPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if len(document.LoweredCalls) != 2 || document.LoweredCalls[0].LogicalName != "runtime.first" || document.LoweredCalls[1].LogicalName != "runtime.second" { + t.Fatalf("canonical lowered calls = %+v", document.LoweredCalls) + } +} + func TestCoroPlanDigestDistinguishesExplicitAndPropagatedRoots(t *testing.T) { prog, pkg := buildCoroTestSSA(t, "roots.go", `package coroid func leaf(ch chan int) { <-ch } diff --git a/internal/coro/ssa_plan.go b/internal/coro/ssa_plan.go index d40796ba77..2be178954c 100644 --- a/internal/coro/ssa_plan.go +++ b/internal/coro/ssa_plan.go @@ -22,6 +22,7 @@ import ( "go/types" "sort" "strings" + "unicode/utf8" "golang.org/x/tools/go/callgraph/cha" "golang.org/x/tools/go/ssa" @@ -115,6 +116,19 @@ type SSAClosedDynamicCallCertificate struct { MayBeNil bool } +// SSALoweredCall records one exact managed call inserted by frontend lowering +// even though no CallInstruction for it exists in the source SSA body. +// LogicalName is a frontend-owned stable identity used to resolve the exact +// helper again during code generation; it is not a symbol-name heuristic. +// +// The first lowering slice projects every record as an ordinary direct call. +// AnalyzeSSA may refine that edge to a foreign boundary from the target's +// frozen function policy, exactly as it does for an explicit static call. +type SSALoweredCall struct { + LogicalName string + Target *ssa.Function +} + // SSAConfig controls the SSA-to-Graph analysis bridge. It deliberately has no // lowering or runtime switches. type SSAConfig struct { @@ -203,6 +217,19 @@ type SSAConfig struct { // effective emission universe. AnalyzeSSA calls the classifier only for // owned, non-ignored bodies and copies the returned slice before use. ClassifyDemandReferences func(owner *ssa.Function) ([]*ssa.Function, error) + + // ClassifyLoweredCalls supplies exact runtime/helper calls that the frontend + // inserts while lowering one function body but which have no corresponding + // source SSA CallInstruction. Unlike ClassifyDemandReferences, these are real + // calls: their effects and inheritable execution constraints propagate into + // owner, and demand reaches their selected plain or coroutine entry only when + // owner itself is demanded. + // + // LogicalName must be nonempty and unique within owner. Every target must be + // a non-nil exact canonical member of the effective emission universe. The + // classifier is called only for owned, non-ignored bodies and its result is + // copied, validated, and sorted before it becomes part of the immutable plan. + ClassifyLoweredCalls func(owner *ssa.Function) ([]SSALoweredCall, error) } // SSAFunctionPlan binds an immutable FunctionPlan back to its SSA function. @@ -231,6 +258,7 @@ type SSAPlan struct { valuePlans map[ssa.Value]SSAValuePlan callPlans map[ssa.CallInstruction]SSACallPlan elidedCalls map[ssa.CallInstruction]struct{} + loweredCalls map[*ssa.Function][]SSALoweredCall functionIDs FunctionIDConfig } @@ -372,6 +400,31 @@ func (p *SSAPlan) IgnoresBody(fn *ssa.Function) bool { return ok } +// LoweredCalls returns the exact compiler-inserted calls frozen for owner in +// LogicalName order. The returned slice is a defensive copy. +func (p *SSAPlan) LoweredCalls(owner *ssa.Function) []SSALoweredCall { + if p == nil || owner == nil { + return nil + } + return append([]SSALoweredCall(nil), p.loweredCalls[owner]...) +} + +// ResolveLoweredCall resolves one frontend logical helper identity for owner. +// ok is false when the exact owner has no call with that identity. +func (p *SSAPlan) ResolveLoweredCall(owner *ssa.Function, logicalName string) (*ssa.Function, bool) { + if p == nil || owner == nil || logicalName == "" { + return nil, false + } + calls := p.loweredCalls[owner] + index := sort.Search(len(calls), func(index int) bool { + return calls[index].LogicalName >= logicalName + }) + if index == len(calls) || calls[index].LogicalName != logicalName { + return nil, false + } + return calls[index].Target, true +} + // Function returns the SSA function assigned to id. func (p *SSAPlan) Function(id FunctionID) (*ssa.Function, bool) { if p == nil { @@ -807,6 +860,10 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err } } } + loweredCalls, err := addSSAClassifiedLoweredCalls(graph, bodyFunctions, includedSet, ids, canonicalizer, policies, config) + if err != nil { + return nil, err + } if err := addSSAReferenceEdges(graph, bodyFunctions, includedSet, ids, flow); err != nil { return nil, err } @@ -838,6 +895,7 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err valuePlans: valuePlans, callPlans: callPlans, elidedCalls: elidedCallSet, + loweredCalls: loweredCalls, functionIDs: config.FunctionIDs, } for _, functionPlan := range base.Functions() { @@ -849,6 +907,74 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err return result, nil } +func addSSAClassifiedLoweredCalls( + graph *Graph, + functions []*ssa.Function, + included map[*ssa.Function]bool, + ids map[*ssa.Function]FunctionID, + canonicalizer *ssaFunctionCanonicalizer, + policies map[*ssa.Function]SSAFunctionPolicy, + config SSAConfig, +) (map[*ssa.Function][]SSALoweredCall, error) { + result := make(map[*ssa.Function][]SSALoweredCall) + if config.ClassifyLoweredCalls == nil { + return result, nil + } + for _, owner := range functions { + calls, err := config.ClassifyLoweredCalls(owner) + if err != nil { + return nil, fmt.Errorf("coro: classify lowered calls in %q: %w", owner.Name(), err) + } + // The classifier owns its backing storage. Copy before validating or + // retaining any record in the immutable plan. + calls = append([]SSALoweredCall(nil), calls...) + seen := make(map[string]struct{}, len(calls)) + for index := range calls { + call := &calls[index] + if call.LogicalName == "" { + return nil, fmt.Errorf("coro: lowered call %d in %q has an empty logical name", index, owner.Name()) + } + if !utf8.ValidString(call.LogicalName) || strings.IndexByte(call.LogicalName, 0) >= 0 { + return nil, fmt.Errorf("coro: lowered call %d in %q has an invalid logical name %q", index, owner.Name(), call.LogicalName) + } + if _, duplicate := seen[call.LogicalName]; duplicate { + return nil, fmt.Errorf("coro: lowered call logical name %q is duplicated in %q", call.LogicalName, owner.Name()) + } + seen[call.LogicalName] = struct{}{} + target := call.Target + if target == nil { + return nil, fmt.Errorf("coro: lowered call %q in %q has a nil target", call.LogicalName, owner.Name()) + } + if target.Prog != owner.Prog { + return nil, fmt.Errorf("coro: lowered call %q in %q targets function %q from another SSA program", call.LogicalName, owner.Name(), target.Name()) + } + canonical, resolved, resolveErr := canonicalizer.resolve(target) + if resolveErr != nil { + return nil, fmt.Errorf("coro: resolve lowered call %q target %q in %q: %w", call.LogicalName, target.Name(), owner.Name(), resolveErr) + } + if !resolved || canonical == nil || !included[canonical] { + return nil, fmt.Errorf("coro: lowered call %q target %q in %q is outside the effective emission universe", call.LogicalName, target.Name(), owner.Name()) + } + if canonical != target { + return nil, fmt.Errorf("coro: lowered call %q target %q in %q is not the exact canonical function", call.LogicalName, target.Name(), owner.Name()) + } + } + sort.Slice(calls, func(i, j int) bool { + return calls[i].LogicalName < calls[j].LogicalName + }) + if len(calls) != 0 { + result[owner] = calls + } + for _, call := range calls { + kind := staticCallKind(CallDirect, policies[call.Target]) + if err := graph.AddCall(CallEdge{Caller: ids[owner], Callee: ids[call.Target], Kind: kind}); err != nil { + return nil, fmt.Errorf("coro: add lowered call %q from %q to %q: %w", call.LogicalName, owner.Name(), call.Target.Name(), err) + } + } + } + return result, nil +} + func addSSAClassifiedDemandReferences( graph *Graph, functions []*ssa.Function, diff --git a/internal/coro/ssa_plan_test.go b/internal/coro/ssa_plan_test.go index 001598652c..f89bc910c7 100644 --- a/internal/coro/ssa_plan_test.go +++ b/internal/coro/ssa_plan_test.go @@ -423,6 +423,163 @@ func outsideFrozenUniverse() {} } } +func TestAnalyzeSSAClassifiedLoweredCallsPropagateEffectAndAreOwnerScoped(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "lowered_calls.go", `package coroid + +var channel chan int + +func owner() {} +func deadOwner() {} +func plainHelper() {} +func suspendingHelper() { <-channel } +func deadHelper() { <-channel } +func outsideFrozenUniverse() {} +`) + owner := packageFunction(t, pkg, "owner") + deadOwner := packageFunction(t, pkg, "deadOwner") + plain := packageFunction(t, pkg, "plainHelper") + suspending := packageFunction(t, pkg, "suspendingHelper") + dead := packageFunction(t, pkg, "deadHelper") + outside := packageFunction(t, pkg, "outsideFrozenUniverse") + universe, err := NewSSAEmissionUniverse(prog, []*ssa.Function{owner, deadOwner, plain, suspending, dead}) + if err != nil { + t.Fatal(err) + } + classify := func(fn *ssa.Function) ([]SSALoweredCall, error) { + switch fn { + case owner: + // Deliberately reverse logical order. The frozen plan must sort it. + return []SSALoweredCall{{LogicalName: "runtime.suspend", Target: suspending}, {LogicalName: "runtime.plain", Target: plain}}, nil + case deadOwner: + return []SSALoweredCall{{LogicalName: "runtime.dead", Target: dead}}, nil + default: + return nil, nil + } + } + plan, err := AnalyzeSSA(prog, Roots{{Function: owner, Demand: SyncDemand}}, SSAConfig{ + EmissionUniverse: universe, + ClassifyLoweredCalls: classify, + MaxPlainInstructions: -1, + }) + if err != nil { + t.Fatal(err) + } + ownerPlan := functionPlanFor(t, plan, owner) + if !ownerPlan.Effect.Contains(MayPark) || ownerPlan.Demand != SyncDemand || ownerPlan.Emission != EmitCoroutine { + t.Fatalf("owner plan = %+v, want lowered helper effect and coroutine emission", ownerPlan) + } + if got := functionPlanFor(t, plan, plain); got.Demand != SyncDemand || got.Emission != EmitPlain { + t.Fatalf("plain helper plan = %+v", got) + } + if got := functionPlanFor(t, plan, suspending); got.Demand != AsyncDemand || got.Emission != EmitCoroutine { + t.Fatalf("suspending helper plan = %+v", got) + } + if got := functionPlanFor(t, plan, deadOwner); !got.Effect.Contains(MayPark) || got.Demand != NoDemand || got.Emission != EmitNone { + t.Fatalf("dead owner plan = %+v, want analyzed effect without entry demand", got) + } + if got := functionPlanFor(t, plan, dead); got.Demand != NoDemand || got.Emission != EmitNone { + t.Fatalf("dead helper plan = %+v, want no demand", got) + } + + calls := plan.LoweredCalls(owner) + if len(calls) != 2 || calls[0].LogicalName != "runtime.plain" || calls[0].Target != plain || calls[1].LogicalName != "runtime.suspend" || calls[1].Target != suspending { + t.Fatalf("owner lowered calls = %+v, want sorted exact mapping", calls) + } + calls[0].Target = dead + if target, ok := plan.ResolveLoweredCall(owner, "runtime.plain"); !ok || target != plain { + t.Fatalf("ResolveLoweredCall(runtime.plain) = %v, %v", target, ok) + } + if _, ok := plan.ResolveLoweredCall(owner, "runtime.missing"); ok { + t.Fatal("missing lowered call unexpectedly resolved") + } + if got := plan.LoweredCalls(outside); got != nil { + t.Fatalf("outside owner lowered calls = %v, want nil", got) + } + + // Permuting classifier order cannot change fixed-point results or the + // immutable logical-name mapping. + permuted, err := AnalyzeSSA(prog, Roots{{Function: owner, Demand: SyncDemand}}, SSAConfig{ + EmissionUniverse: universe, + ClassifyLoweredCalls: func(fn *ssa.Function) ([]SSALoweredCall, error) { + calls, err := classify(fn) + if len(calls) == 2 { + calls[0], calls[1] = calls[1], calls[0] + } + return calls, err + }, + MaxPlainInstructions: -1, + }) + if err != nil { + t.Fatal(err) + } + if got := functionPlanFor(t, permuted, owner); got != ownerPlan { + t.Fatalf("permuted owner plan = %+v, want %+v", got, ownerPlan) + } + if got := permuted.LoweredCalls(owner); len(got) != 2 || got[0].LogicalName != "runtime.plain" || got[1].LogicalName != "runtime.suspend" { + t.Fatalf("permuted lowered calls = %+v", got) + } + + _, err = AnalyzeSSA(prog, Roots{{Function: owner, Demand: SyncDemand}}, SSAConfig{ + EmissionUniverse: universe, + ClassifyLoweredCalls: func(fn *ssa.Function) ([]SSALoweredCall, error) { + if fn == owner { + return []SSALoweredCall{{LogicalName: "runtime.outside", Target: outside}}, nil + } + return nil, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "outside the effective emission universe") { + t.Fatalf("missing frozen lowered target error = %v", err) + } +} + +func TestAnalyzeSSAClassifiedLoweredCallsFailClosed(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "lowered_calls_invalid.go", `package coroid +func owner() {} +func helper() {} +func alias() {} +`) + owner := packageFunction(t, pkg, "owner") + helper := packageFunction(t, pkg, "helper") + alias := packageFunction(t, pkg, "alias") + universe, err := NewSSAEmissionUniverse(prog, []*ssa.Function{owner, helper}) + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + calls []SSALoweredCall + want string + }{ + {name: "empty name", calls: []SSALoweredCall{{Target: helper}}, want: "empty logical name"}, + {name: "nil target", calls: []SSALoweredCall{{LogicalName: "runtime.nil"}}, want: "nil target"}, + {name: "duplicate name", calls: []SSALoweredCall{{LogicalName: "runtime.same", Target: helper}, {LogicalName: "runtime.same", Target: helper}}, want: "duplicated"}, + {name: "alias", calls: []SSALoweredCall{{LogicalName: "runtime.alias", Target: alias}}, want: "not the exact canonical function"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := AnalyzeSSA(prog, Roots{{Function: owner, Demand: SyncDemand}}, SSAConfig{ + EmissionUniverse: universe, + ResolveFunction: func(fn *ssa.Function) (*ssa.Function, bool, error) { + if fn == alias { + return helper, true, nil + } + return fn, universe.Contains(fn), nil + }, + ClassifyLoweredCalls: func(fn *ssa.Function) ([]SSALoweredCall, error) { + if fn == owner { + return test.calls, nil + } + return nil, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("AnalyzeSSA error = %v, want %q", err, test.want) + } + }) + } +} + func TestAnalyzeSSADynamicOpenAndClosedWorld(t *testing.T) { prog, pkg := buildCoroTestSSA(t, "source.go", `package coroid From 8d09749db1959e67b7c669e7f3dbbf075b03da98 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 18:17:56 +0800 Subject: [PATCH 10/32] feat(ssa): resolve compiler-lowered runtime calls --- ssa/decl.go | 9 +++- ssa/expr.go | 22 ++++++++ ssa/package.go | 46 ++++++++++++---- ssa/runtime_call_resolver_test.go | 88 +++++++++++++++++++++++++++++++ ssa/stmt_builder.go | 3 +- 5 files changed, 156 insertions(+), 12 deletions(-) create mode 100644 ssa/runtime_call_resolver_test.go diff --git a/ssa/decl.go b/ssa/decl.go index 26b02f4ed8..d640aca083 100644 --- a/ssa/decl.go +++ b/ssa/decl.go @@ -368,8 +368,13 @@ func (p Function) NewBuilder() Builder { b := prog.ctx.NewBuilder() // TODO(xsw): Finalize may cause panic, so comment it. // b.Finalize() - return &aBuilder{b, nil, p, p.Pkg, prog, - make(map[*types.Scope]DIScope)} + return &aBuilder{ + impl: b, + Func: p, + Pkg: p.Pkg, + Prog: prog, + diScopeCache: make(map[*types.Scope]DIScope), + } } // HasBody reports whether the function has a body. diff --git a/ssa/expr.go b/ssa/expr.go index caee097c06..eec6ffbfdd 100644 --- a/ssa/expr.go +++ b/ssa/expr.go @@ -1224,6 +1224,9 @@ func (b Builder) InlineCall(fn Expr, args ...Expr) (ret Expr) { // t4 = t3() func (b Builder) Call(fn Expr, args ...Expr) (ret Expr) { dbgInstrCall("Call", fn, args) + if ret, resolved := b.resolveRuntimeCall(fn, args); resolved { + return ret + } var kind = fn.kind if kind == vkPyFuncRef { return b.pyCall(fn, args) @@ -1275,6 +1278,25 @@ func (b Builder) Call(fn Expr, args ...Expr) (ret Expr) { return } +func (b Builder) resolveRuntimeCall(fn Expr, args []Expr) (ret Expr, resolved bool) { + resolver := b.Pkg.runtimeCall + if resolver == nil || b.resolvingRuntimeCalls[fn.Type] { + return Nil, false + } + helper, ok := b.Pkg.runtimeFuncs[fn.Type] + if !ok { + return Nil, false + } + if b.resolvingRuntimeCalls == nil { + b.resolvingRuntimeCalls = make(map[Type]bool) + } + b.resolvingRuntimeCalls[fn.Type] = true + defer func() { + delete(b.resolvingRuntimeCalls, fn.Type) + }() + return resolver(b, helper, fn, args) +} + const ( ReflectArrayOf = 1 << iota ReflectChanOf diff --git a/ssa/package.go b/ssa/package.go index 1f67f8197c..ea18ff1d1e 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -541,6 +541,7 @@ func (p Program) NewPackage(name, pkgPath string) Package { preserveSyms: make(map[string]struct{}), llvmUsedValues: make([]llvm.Value, 0, 4), llvmRetainedValues: make([]llvm.Value, 0, 1), + runtimeFuncs: make(map[Type]string), abiTypeFakeUseCache: make(map[llvm.Value][]llvm.Value), } @@ -798,14 +799,16 @@ type aPackage struct { cu CompilationUnit glbDbgVars map[Expr]bool - vars map[string]Global - fns map[string]Function - pyobjs map[string]PyObjRef - pymods map[string]Global - strs map[string]llvm.Value - goStrs map[string]llvm.Value - fnlink func(string) string - methodlink func(string, *types.Func, *types.Signature) string + vars map[string]Global + fns map[string]Function + pyobjs map[string]PyObjRef + pymods map[string]Global + strs map[string]llvm.Value + goStrs map[string]llvm.Value + fnlink func(string) string + methodlink func(string, *types.Func, *types.Signature) string + runtimeCall RuntimeCallResolver + runtimeFuncs map[Type]string iRoutine int @@ -887,7 +890,19 @@ func (p Package) rtFunc(fnName string) Expr { name = p.fnlink(name) } sig := fn.Type().(*types.Signature) - return p.NewFunc(name, sig, InGo).Expr + ret := p.NewFunc(name, sig, InGo).Expr + if p.runtimeCall == nil { + return ret + } + // NewFunc reuses the declaration and its canonical Type. Clone only the + // Type wrapper so Builder.Call can recognize this exact compiler-inserted + // runtime helper expression without confusing an ordinary call to the same + // LLVM declaration, or a helper whose address is merely retained in an ABI + // table. The raw and LLVM function types remain unchanged. + typ := *ret.Type + ret.Type = &typ + p.runtimeFuncs[ret.Type] = fnName + return ret } func (p Package) cFunc(fullName string, sig *types.Signature) Expr { @@ -947,6 +962,19 @@ func (p Package) SetResolveMethodLinkname(fn func(string, *types.Func, *types.Si p.methodlink = fn } +// RuntimeCallResolver may replace a compiler-inserted runtime helper call. +// helper is the logical runtime function name passed to rtFunc; fn already +// contains the resolved physical symbol and args are already lowered. +// Returning ok=false preserves the ordinary direct call. +type RuntimeCallResolver func(b Builder, helper string, fn Expr, args []Expr) (ret Expr, ok bool) + +// SetResolveRuntimeCall installs the resolver for compiler-inserted runtime +// helper calls. Install it before lowering function bodies. A nil resolver +// preserves the legacy lowering, including the canonical function Type. +func (p Package) SetResolveRuntimeCall(fn RuntimeCallResolver) { + p.runtimeCall = fn +} + // ----------------------------------------------------------------------------- // AfterInit is called after the package is initialized (init all packages that depends on). diff --git a/ssa/runtime_call_resolver_test.go b/ssa/runtime_call_resolver_test.go new file mode 100644 index 0000000000..4d7c65c0b7 --- /dev/null +++ b/ssa/runtime_call_resolver_test.go @@ -0,0 +1,88 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import ( + "go/token" + "go/types" + "testing" +) + +func newRuntimeCallResolverTest(t *testing.T) (Program, Package, *types.Signature) { + t.Helper() + prog := NewProgram(nil) + t.Cleanup(prog.Dispose) + runtimePkg := types.NewPackage(PkgRuntime, PkgRuntime) + sig := types.NewSignatureType(nil, nil, nil, nil, nil, false) + if alt := runtimePkg.Scope().Insert(types.NewFunc(token.NoPos, runtimePkg, "Helper", sig)); alt != nil { + t.Fatalf("insert runtime helper returned alternate object %v", alt) + } + prog.SetRuntime(runtimePkg) + return prog, prog.NewPackage("caller", "example.com/caller"), sig +} + +func TestRuntimeCallResolverInterceptsOnlyRtFuncExpression(t *testing.T) { + _, pkg, sig := newRuntimeCallResolverTest(t) + calls := 0 + pkg.SetResolveRuntimeCall(func(b Builder, helper string, fn Expr, args []Expr) (Expr, bool) { + calls++ + if helper != "Helper" { + t.Fatalf("helper = %q, want Helper", helper) + } + // Calling the original expression from inside the resolver must bypass + // the hook, otherwise a plain replacement would recurse forever. + return b.Call(fn, args...), true + }) + + marked := pkg.rtFunc("Helper") + ordinary := pkg.NewFunc(marked.Name(), sig, InGo).Expr + if marked.Type == ordinary.Type { + t.Fatal("rtFunc expression shares the ordinary declaration Type marker") + } + caller := pkg.NewFunc("caller", NoArgsNoRet, InGo) + b := caller.MakeBody(1) + b.Call(ordinary) + b.Call(marked) + b.Return() + b.EndBuild() + if calls != 1 { + t.Fatalf("resolver calls = %d, want 1", calls) + } +} + +func TestRuntimeCallResolverFallbackPreservesDirectCall(t *testing.T) { + _, pkg, _ := newRuntimeCallResolverTest(t) + calls := 0 + pkg.SetResolveRuntimeCall(func(_ Builder, helper string, _ Expr, _ []Expr) (Expr, bool) { + calls++ + if helper != "Helper" { + t.Fatalf("helper = %q, want Helper", helper) + } + return Nil, false + }) + + caller := pkg.NewFunc("caller", NoArgsNoRet, InGo) + b := caller.MakeBody(1) + b.Call(pkg.rtFunc("Helper")) + b.Return() + b.EndBuild() + if calls != 1 { + t.Fatalf("resolver calls = %d, want 1", calls) + } +} diff --git a/ssa/stmt_builder.go b/ssa/stmt_builder.go index fbd3a5242f..0870dc04d7 100644 --- a/ssa/stmt_builder.go +++ b/ssa/stmt_builder.go @@ -64,7 +64,8 @@ type aBuilder struct { Pkg Package Prog Program - diScopeCache map[*types.Scope]DIScope // avoid duplicated DILexicalBlock(s) + diScopeCache map[*types.Scope]DIScope // avoid duplicated DILexicalBlock(s) + resolvingRuntimeCalls map[Type]bool } // Builder represents a builder for creating instructions in a function. From d47b16bbb3c0f051edefedadeb3070ce66c48894 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 22:29:22 +0800 Subject: [PATCH 11/32] build: sync coroutine bindings with upstream LLVM 22 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a26ff98f3e..c102b42564 100644 --- a/go.mod +++ b/go.mod @@ -27,4 +27,4 @@ require ( replace github.com/goplus/llgo/runtime => ./runtime -replace github.com/xgo-dev/llvm => github.com/cpunion/llvm v0.9.4-0.20260715231903-426515db6e7d +replace github.com/xgo-dev/llvm => github.com/cpunion/llvm v0.9.4-0.20260716142756-bf5fb88be315 diff --git a/go.sum b/go.sum index b1e8f43dc1..14f29ac85f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/cpunion/llvm v0.9.4-0.20260715231903-426515db6e7d h1:pzBHogKjOuftDsC+H+pPDU7cykyZUEs9/4W/Cx4Q450= -github.com/cpunion/llvm v0.9.4-0.20260715231903-426515db6e7d/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= +github.com/cpunion/llvm v0.9.4-0.20260716142756-bf5fb88be315 h1:UxPbO92bHJeExHSKhc/77Awun1LpmOCYXI9gNaggVvg= +github.com/cpunion/llvm v0.9.4-0.20260716142756-bf5fb88be315/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= github.com/creack/goselect v0.1.2 h1:2DNy14+JPjRBgPzAd1thbQp4BSIihxcBf0IXhQXDRa0= github.com/creack/goselect v0.1.2/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From e2d62dfd46e39ea60fb97ce8bead680471b43e9c Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 22:31:14 +0800 Subject: [PATCH 12/32] target(wasm): link freestanding allocator without BDWGC --- .../crosscompile/compile/libc/libc_test.go | 123 ++++++++++ .../crosscompile/compile/libc/wasmbuiltins.go | 220 ++++++++++++++++++ internal/crosscompile/crosscompile.go | 90 ++++++- internal/crosscompile/crosscompile_test.go | 138 +++++++++++ internal/crosscompile/libc.go | 15 ++ internal/crosscompile/libc_test.go | 25 ++ .../testdata/wasm_allocator/main.go | 35 +++ .../crosscompile/wasm_target_smoke_test.go | 173 ++++++++++++++ internal/targets/config.go | 6 + internal/targets/loader.go | 3 + internal/targets/resolver.go | 4 + internal/targets/targets_test.go | 25 ++ targets/wasip1.json | 2 +- targets/wasip2.json | 2 +- targets/wasm.json | 2 +- 15 files changed, 854 insertions(+), 9 deletions(-) create mode 100644 internal/crosscompile/compile/libc/wasmbuiltins.go create mode 100644 internal/crosscompile/testdata/wasm_allocator/main.go create mode 100644 internal/crosscompile/wasm_target_smoke_test.go diff --git a/internal/crosscompile/compile/libc/libc_test.go b/internal/crosscompile/compile/libc/libc_test.go index 12a11a41a9..fc99a284ad 100644 --- a/internal/crosscompile/compile/libc/libc_test.go +++ b/internal/crosscompile/compile/libc/libc_test.go @@ -3,12 +3,135 @@ package libc import ( + "os" "path/filepath" "slices" "strings" "testing" ) +func TestWasmBuiltinsCompileConfigIsFreestandingAndTripleScoped(t *testing.T) { + baseDir := "/cache/wasmbuiltins" + includeDir := filepath.Join(baseDir, "llgo-wasmbuiltins-include") + wasip2 := GetWasmBuiltinsCompileConfig(baseDir, includeDir, "wasm32-unknown-wasi") + unknown := GetWasmBuiltinsCompileConfig(baseDir, includeDir, "wasm32-unknown-unknown") + + if len(wasip2.Groups) != 1 || len(unknown.Groups) != 1 { + t.Fatalf("wasmbuiltins groups = %d/%d, want 1/1", len(wasip2.Groups), len(unknown.Groups)) + } + if wasip2.Groups[0].OutputFileName == unknown.Groups[0].OutputFileName { + t.Fatalf("different WebAssembly ABIs share archive %q", wasip2.Groups[0].OutputFileName) + } + if !strings.Contains(wasip2.Groups[0].OutputFileName, "wasm32-unknown-wasi") || + !strings.Contains(unknown.Groups[0].OutputFileName, "wasm32-unknown-unknown") { + t.Fatalf("archive names do not preserve target triples: %q, %q", + wasip2.Groups[0].OutputFileName, unknown.Groups[0].OutputFileName) + } + for _, name := range []string{"dlmalloc.c", "sbrk.c", "errno.c", "errno_state.c", "abort.c", "memcpy.c", "memmove.c", "memset.c", "exp.c", "log.c"} { + if !slices.ContainsFunc(wasip2.Groups[0].Files, func(path string) bool { + return filepath.Base(path) == name + }) { + t.Errorf("wasmbuiltins is missing %s", name) + } + } + for _, forbidden := range []string{"pthread", "socket", "preview1", "wasi_snapshot_preview1"} { + if slices.ContainsFunc(wasip2.Groups[0].Files, func(path string) bool { + return strings.Contains(filepath.Base(path), forbidden) + }) { + t.Errorf("freestanding wasmbuiltins unexpectedly contains %q source", forbidden) + } + } + for _, flag := range []string{ + "-nostdlibinc", + "-D__wasilibc_unmodified_upstream", + "-mno-bulk-memory", + "-I" + includeDir, + "-I" + filepath.Join(baseDir, "dlmalloc", "include"), + "-idirafter" + filepath.Join(baseDir, "libc-top-half", "musl", "include"), + } { + if !slices.Contains(wasip2.Groups[0].CFlags, flag) { + t.Errorf("wasmbuiltins C flags = %v, want %q", wasip2.Groups[0].CFlags, flag) + } + } +} + +func TestPrepareWasmBuiltinsHeadersDoesNotRequireWASISysroot(t *testing.T) { + baseDir := t.TempDir() + archDir := filepath.Join(baseDir, "libc-top-half", "musl", "arch", "wasm32", "bits") + includeSourceDir := filepath.Join(baseDir, "libc-top-half", "musl", "include") + typeHeaderDir := filepath.Join(baseDir, "libc-bottom-half", "headers", "public") + for _, dir := range []string{archDir, includeSourceDir, typeHeaderDir} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } + for _, name := range []string{ + "__typedef_time_t.h", + "__typedef_suseconds_t.h", + "__typedef_clockid_t.h", + "__typedef_sigset_t.h", + "__typedef_clock_t.h", + } { + if err := os.WriteFile(filepath.Join(typeHeaderDir, name), []byte("/* pinned scalar type */\n"), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(archDir, "alltypes.h.in"), []byte(` +#define _Addr long +#if defined(__NEED_sigset_t) && !defined(__DEFINED_sigset_t) +#include <__typedef_sigset_t.h> +#define __DEFINED_sigset_t +#endif +`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(includeSourceDir, "alltypes.h.in"), []byte(` +TYPEDEF unsigned _Addr size_t; +TYPEDEF _Addr intptr_t; +TYPEDEF unsigned wchar_t; +TYPEDEF struct __sigset_t { unsigned long __bits[128/sizeof(long)]; } sigset_t; +`), 0o644); err != nil { + t.Fatal(err) + } + + includeDir, err := PrepareWasmBuiltinsHeaders(baseDir) + if err != nil { + t.Fatal(err) + } + contents, err := os.ReadFile(filepath.Join(includeDir, "bits", "alltypes.h")) + if err != nil { + t.Fatal(err) + } + for _, declaration := range []string{ + "__NEED_size_t", + "__NEED_intptr_t", + "__NEED_wchar_t", + "__NEED_sigset_t", + "__typedef_sigset_t.h", + } { + if !strings.Contains(string(contents), declaration) { + t.Errorf("generated alltypes.h is missing %q", declaration) + } + } + if strings.Contains(string(contents), "TYPEDEF ") { + t.Errorf("generated alltypes.h still contains unexpanded TYPEDEF directives:\n%s", contents) + } + for _, supportFile := range []string{ + "errno.h", + "errno_state.c", + "__macro_PAGESIZE.h", + "__typedef_time_t.h", + "__typedef_suseconds_t.h", + "__typedef_clockid_t.h", + "__typedef_sigset_t.h", + "__typedef_clock_t.h", + } { + if _, err := os.Stat(filepath.Join(includeDir, supportFile)); err != nil { + t.Errorf("generated support file %s: %v", supportFile, err) + } + } +} + func TestGetNewlibESP32Config_LibConfig(t *testing.T) { config := GetNewlibESP32Config() diff --git a/internal/crosscompile/compile/libc/wasmbuiltins.go b/internal/crosscompile/compile/libc/wasmbuiltins.go new file mode 100644 index 0000000000..84296c98fb --- /dev/null +++ b/internal/crosscompile/compile/libc/wasmbuiltins.go @@ -0,0 +1,220 @@ +package libc + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/goplus/llgo/internal/crosscompile/compile" +) + +const wasmBuiltinsRevision = "1dfe5c302d1c5ab621f7abf04620fae92700fd22" +const wasmBuiltinsRecipe = "dlmalloc-v4" + +// GetWasmBuiltinsConfig returns the pinned wasi-libc source used for the +// freestanding WebAssembly builtin library. This is deliberately separate +// from wasi-libc: it provides only the memory and libm helpers that LLVM may +// lower to out-of-line calls, and it must not introduce a WASI Preview 1 ABI. +func GetWasmBuiltinsConfig() compile.LibConfig { + return compile.LibConfig{ + Name: "wasmbuiltins", + Version: wasmBuiltinsRevision + "-" + wasmBuiltinsRecipe, + Url: "https://github.com/WebAssembly/wasi-libc/archive/" + wasmBuiltinsRevision + ".tar.gz", + ResourceSubDir: "wasi-libc-" + wasmBuiltinsRevision, + } +} + +// PrepareWasmBuiltinsHeaders generates musl's bits/alltypes.h from the two +// templates in the pinned wasi-libc source tree. Keeping this generated header +// inside the versioned source cache makes the library independent of a WASI +// sysroot, which is required by wasm32-unknown-unknown. +func PrepareWasmBuiltinsHeaders(baseDir string) (string, error) { + includeDir := filepath.Join(baseDir, "llgo-wasmbuiltins-include") + bitsDir := filepath.Join(includeDir, "bits") + if err := os.MkdirAll(bitsDir, 0o755); err != nil { + return "", fmt.Errorf("create wasmbuiltins headers: %w", err) + } + muslDir := filepath.Join(baseDir, "libc-top-half", "musl") + templates := []string{ + filepath.Join(muslDir, "arch", "wasm32", "bits", "alltypes.h.in"), + filepath.Join(muslDir, "include", "alltypes.h.in"), + } + var generated bytes.Buffer + for _, template := range templates { + contents, err := os.ReadFile(template) + if err != nil { + return "", fmt.Errorf("read wasmbuiltins alltypes template %s: %w", template, err) + } + generateMuslAllTypes(&generated, string(contents)) + } + path := filepath.Join(bitsDir, "alltypes.h") + if err := os.WriteFile(path, generated.Bytes(), 0o644); err != nil { + return "", fmt.Errorf("write wasmbuiltins alltypes.h: %w", err) + } + // The wasm alltypes template refers to these five lower-half headers for + // scalar type definitions. Copy only this closed, declaration-only subset; + // adding the full public directory to the search path would let errno.h pull + // the Preview 1 wasi/api.h ABI into wasm32-unknown-unknown. + typeHeaderDir := filepath.Join(baseDir, "libc-bottom-half", "headers", "public") + for _, name := range []string{ + "__typedef_time_t.h", + "__typedef_suseconds_t.h", + "__typedef_clockid_t.h", + "__typedef_sigset_t.h", + "__typedef_clock_t.h", + } { + source := filepath.Join(typeHeaderDir, name) + contents, err := os.ReadFile(source) + if err != nil { + return "", fmt.Errorf("read wasmbuiltins type header %s: %w", source, err) + } + destination := filepath.Join(includeDir, name) + if err := os.WriteFile(destination, contents, 0o644); err != nil { + return "", fmt.Errorf("write wasmbuiltins type header %s: %w", destination, err) + } + } + generatedFiles := map[string]string{ + // dlmalloc only needs the two errno values below. A target-local errno + // state avoids pulling wasi/api.h (Preview 1) into unknown-unknown and + // remains sufficient for these single-threaded WebAssembly targets. + "errno.h": `#ifndef LLGO_WASMBUILTINS_ERRNO_H +#define LLGO_WASMBUILTINS_ERRNO_H +extern int errno; +#ifndef EINVAL +#define EINVAL 22 +#endif +#ifndef ENOMEM +#define ENOMEM 12 +#endif +#endif +`, + "errno_state.c": `int errno; +`, + // WebAssembly 1.0 fixes the linear-memory page size at 64 KiB. sbrk.c + // consumes only this macro from wasi-libc's public header collection. + "__macro_PAGESIZE.h": `#ifndef LLGO_WASMBUILTINS_PAGESIZE_H +#define LLGO_WASMBUILTINS_PAGESIZE_H +#define PAGESIZE (0x10000) +#endif +`, + } + for name, contents := range generatedFiles { + path := filepath.Join(includeDir, name) + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + return "", fmt.Errorf("write wasmbuiltins support file %s: %w", path, err) + } + } + return includeDir, nil +} + +// generateMuslAllTypes implements wasi-libc's pinned mkalltypes.sed script. +// Keeping this tiny transformation in Go avoids making the crosscompiler rely +// on a host sed implementation while retaining the upstream conditional type +// definitions verbatim. +func generateMuslAllTypes(out *bytes.Buffer, template string) { + for _, line := range strings.SplitAfter(template, "\n") { + hasNewline := strings.HasSuffix(line, "\n") + line = strings.TrimSuffix(line, "\n") + switch { + case strings.HasPrefix(line, "TYPEDEF ") && strings.HasSuffix(line, ";"): + declaration := strings.TrimSuffix(strings.TrimPrefix(line, "TYPEDEF "), ";") + if split := strings.LastIndexByte(declaration, ' '); split >= 0 { + typeName := declaration[split+1:] + fmt.Fprintf(out, "#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s;\n#define __DEFINED_%s\n#endif\n\n", + typeName, typeName, declaration, typeName) + continue + } + case strings.HasPrefix(line, "STRUCT ") && strings.HasSuffix(line, ";"): + declaration := strings.TrimSuffix(strings.TrimPrefix(line, "STRUCT "), ";") + if split := strings.IndexByte(declaration, ' '); split >= 0 { + name := declaration[:split] + body := declaration[split+1:] + fmt.Fprintf(out, "#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n\n", + name, name, name, body, name) + continue + } + case strings.HasPrefix(line, "UNION ") && strings.HasSuffix(line, ";"): + declaration := strings.TrimSuffix(strings.TrimPrefix(line, "UNION "), ";") + if split := strings.IndexByte(declaration, ' '); split >= 0 { + name := declaration[:split] + body := declaration[split+1:] + fmt.Fprintf(out, "#if defined(__NEED_union_%s) && !defined(__DEFINED_union_%s)\nunion %s %s;\n#define __DEFINED_union_%s\n#endif\n\n", + name, name, name, body, name) + continue + } + } + out.WriteString(line) + if hasNewline { + out.WriteByte('\n') + } + } +} + +// GetWasmBuiltinsCompileConfig mirrors the deliberately small freestanding +// library used by TinyGo's wasip2 and wasm-unknown targets. The target triple +// is part of the archive name so the WASI and unknown-unknown ABIs can never +// share cached objects. +func GetWasmBuiltinsCompileConfig(baseDir, includeDir, target string) compile.CompileConfig { + muslDir := filepath.Join(baseDir, "libc-top-half", "musl") + source := func(parts ...string) string { + return filepath.Join(append([]string{muslDir, "src"}, parts...)...) + } + includeFlags := []string{ + "-I" + filepath.Join(baseDir, "dlmalloc", "include"), + "-I" + includeDir, + "-isystem" + filepath.Join(muslDir, "arch", "wasm32"), + "-isystem" + filepath.Join(muslDir, "arch", "generic"), + "-isystem" + filepath.Join(muslDir, "src", "internal"), + "-isystem" + filepath.Join(muslDir, "src", "include"), + // Clang's builtin stddef.h must precede musl's fallback: wasi-libc's + // wasm alltypes template deliberately asks it for wchar_t/max_align_t. + "-idirafter" + filepath.Join(muslDir, "include"), + } + groupFlags := append([]string{ + "-Wall", + "-Wno-unused-but-set-variable", + "-std=gnu11", + "-nostdlibinc", + // Use musl's target-neutral declarations. The patched wasi-libc branch + // includes Preview 1 wasi/api.h for errno values, which is invalid for + // wasm32-unknown-unknown and unnecessary for this builtin subset. + "-D__wasilibc_unmodified_upstream", + "-mnontrapping-fptoint", + // The routines must not recursively lower memcpy/memmove/memset back + // to the very symbols this archive is providing. + "-mno-bulk-memory", + }, includeFlags...) + + return compile.CompileConfig{ + Groups: []compile.CompileGroup{{ + OutputFileName: fmt.Sprintf("libwasmbuiltins-%s.a", target), + Files: []string{ + // wasi-libc's default malloc implementation. Its sbrk backend + // lowers directly to WebAssembly memory.size/memory.grow and + // therefore works for both WASI Preview 2 core modules and the + // unknown-unknown freestanding ABI without Preview 1 imports. + filepath.Join(baseDir, "dlmalloc", "src", "dlmalloc.c"), + filepath.Join(baseDir, "libc-bottom-half", "sources", "sbrk.c"), + filepath.Join(baseDir, "libc-bottom-half", "sources", "errno.c"), + filepath.Join(includeDir, "errno_state.c"), + filepath.Join(baseDir, "libc-bottom-half", "sources", "abort.c"), + source("string", "memcpy.c"), + source("string", "memmove.c"), + source("string", "memset.c"), + source("math", "__math_divzero.c"), + source("math", "__math_invalid.c"), + source("math", "__math_oflow.c"), + source("math", "__math_uflow.c"), + source("math", "__math_xflow.c"), + source("math", "exp.c"), + source("math", "exp_data.c"), + source("math", "exp2.c"), + source("math", "log.c"), + source("math", "log_data.c"), + }, + CFlags: groupFlags, + }}, + } +} diff --git a/internal/crosscompile/crosscompile.go b/internal/crosscompile/crosscompile.go index 4306b34b52..1425f98861 100644 --- a/internal/crosscompile/crosscompile.go +++ b/internal/crosscompile/crosscompile.go @@ -28,6 +28,7 @@ type Export struct { // Additional fields from target configuration BuildTags []string + GC string // Runtime GC capability: precise, conservative, leaking, or none. GOOS string GOARCH string Libc string @@ -186,7 +187,13 @@ func compileWithConfig( compileConfig compile.CompileConfig, outputDir string, options compile.CompileOptions, ) (ldflags []string, err error) { - ldflags = append(ldflags, "-nostdlib", "-L"+outputDir) + // -nostdlib is a compiler-driver option, not part of wasm-ld's interface. + // Named WebAssembly targets invoke wasm-ld directly and already provide + // every archive explicitly. + if filepath.Base(options.Linker) != "wasm-ld" { + ldflags = append(ldflags, "-nostdlib") + } + ldflags = append(ldflags, "-L"+outputDir) for _, group := range compileConfig.Groups { err = group.Compile(outputDir, options) @@ -201,6 +208,36 @@ func compileWithConfig( return } +func linkerSupportsICF(linker string) bool { + if linker == "" { + return false + } + output, err := exec.Command(linker, "--help").CombinedOutput() + return err == nil && linkerHelpSupportsICF(string(output)) +} + +func linkerHelpSupportsICF(help string) bool { + return strings.Contains(help, "--icf=") || strings.Contains(help, "--icf <") +} + +func validateLibcTargetCompatibility(config *targets.Config) error { + if config == nil || config.Libc != "wasmbuiltins" { + return nil + } + if !strings.HasPrefix(config.LLVMTarget, "wasm32-") { + return fmt.Errorf("libc wasmbuiltins requires a wasm32 LLVM target, got %q", config.LLVMTarget) + } + if config.Linker != "wasm-ld" { + return fmt.Errorf("libc wasmbuiltins requires linker wasm-ld, got %q", config.Linker) + } + for _, feature := range strings.Split(config.Features, ",") { + if strings.TrimSpace(feature) == "+atomics" { + return fmt.Errorf("libc wasmbuiltins does not provide a threaded malloc/errno ABI for +atomics targets") + } + } + return nil +} + func use(goos, goarch string, wasiThreads, forceEspClang bool, level optlevel.Level, ltoMode lto.Mode, goGlobalDCE bool) (export Export, err error) { targetSpec := resolvedLLVMTargetSpec(goos, goarch, wasiThreads) targetTriple := targetSpec.Triple @@ -209,6 +246,12 @@ func use(goos, goarch string, wasiThreads, forceEspClang bool, level optlevel.Le export.LLVMTarget = targetSpec.Triple export.CPU = targetSpec.CPU export.Features = targetSpec.Features + if goarch == "wasm" { + // LLGo does not yet have a tracing collector for WebAssembly linear + // memory. Keep the direct GOOS/GOARCH route honest and select the same + // explicit leaking profile as the named wasm targets. + export.GC = "leaking" + } llgoRoot := env.LLGoROOT() // Check for ESP Clang support for target-based builds @@ -478,6 +521,9 @@ func UseTarget(targetName string, level optlevel.Level, ltoMode lto.Mode) (expor if cpu == "" { return export, fmt.Errorf("target '%s' does not have a valid CPU configuration", targetName) } + if err = validateLibcTargetCompatibility(config); err != nil { + return export, fmt.Errorf("target '%s' has incompatible libc/toolchain configuration: %w", targetName, err) + } // Check for ESP Clang support for target-based builds clangRoot, err := getESPClangRoot(true) @@ -491,6 +537,7 @@ func UseTarget(targetName string, level optlevel.Level, ltoMode lto.Mode) (expor // Convert target config to Export - only export necessary fields export.BuildTags = config.BuildTags + export.GC = config.GC export.GOOS = config.GOOS export.GOARCH = config.GOARCH export.ExtraFiles = config.ExtraFiles @@ -525,9 +572,16 @@ func UseTarget(targetName string, level optlevel.Level, ltoMode lto.Mode) (expor // Build environment map for template variable expansion envs := buildEnvMap(env.LLGoROOT()) - // Convert LLVMTarget, CPU, Features to CCFLAGS/LDFLAGS - // ICF off for Go pc-identity semantics (see the non-cross flags above). - ldflags := []string{"-S", "--icf=none"} + // Convert LLVMTarget, CPU, Features to CCFLAGS/LDFLAGS. Some wasm-ld + // distributions expose the lld ICF switch while others (including the ESP + // LLVM 19 build) reject it. Keep the Go pc-identity policy explicit whenever + // the selected linker advertises the option; older wasm-ld defaults to no + // ICF, so omitting the unsupported switch preserves the same semantics. + ldflags := []string{"-S"} + targetLinker := filepath.Join(clangRoot, "bin", config.Linker) + if config.Linker != "wasm-ld" || linkerSupportsICF(targetLinker) { + ldflags = append(ldflags, "--icf=none") + } ccflags := []string{level.Flag()} cflags := []string{"-Wno-override-module", "-Qunused-arguments", "-Wno-unused-command-line-argument"} if config.LLVMTarget != "" { @@ -714,8 +768,32 @@ func UseTarget(targetName string, level optlevel.Level, ltoMode lto.Mode) (expor // Use extends the original Use function to support target-based configuration // If targetName is provided, it takes precedence over goos/goarch func Use(goos, goarch, targetName string, wasiThreads, forceEspClang bool, level optlevel.Level, ltoMode lto.Mode, goGlobalDCE bool) (export Export, err error) { - if targetName != "" && !strings.HasPrefix(targetName, "wasm") && !strings.HasPrefix(targetName, "wasi") { + if targetName == "" { + return use(goos, goarch, wasiThreads, forceEspClang, level, ltoMode, goGlobalDCE) + } + if !strings.HasPrefix(targetName, "wasm") && !strings.HasPrefix(targetName, "wasi") { + return UseTarget(targetName, level, ltoMode) + } + + // The legacy wasm driver route has the complete WASI-SDK/Emscripten setup + // for frontend wasm GOARCH targets. Resolve the named target first so + // -target=wasm/wasip1 cannot accidentally compile a host Mach-O image using + // the caller's default GOOS/GOARCH. Targets such as wasip2 and wasm-unknown + // intentionally use an ARM frontend with a wasm LLVM triple and therefore + // continue through the JSON-driven target pipeline. + config, resolveErr := targets.NewDefaultResolver().Resolve(targetName) + if resolveErr != nil { + return export, fmt.Errorf("failed to resolve target %s: %w", targetName, resolveErr) + } + if config.GOARCH != "wasm" { return UseTarget(targetName, level, ltoMode) } - return use(goos, goarch, wasiThreads, forceEspClang, level, ltoMode, goGlobalDCE) + export, err = use(config.GOOS, config.GOARCH, wasiThreads, forceEspClang, level, ltoMode, goGlobalDCE) + if err != nil { + return export, err + } + export.BuildTags = append([]string(nil), config.BuildTags...) + export.GC = config.GC + export.Emulator = config.Emulator + return export, nil } diff --git a/internal/crosscompile/crosscompile_test.go b/internal/crosscompile/crosscompile_test.go index a1adf042e0..0d7d9bf9c5 100644 --- a/internal/crosscompile/crosscompile_test.go +++ b/internal/crosscompile/crosscompile_test.go @@ -10,11 +10,130 @@ import ( "strings" "testing" + "github.com/goplus/llgo/internal/crosscompile/compile" "github.com/goplus/llgo/internal/lto" "github.com/goplus/llgo/internal/optlevel" + "github.com/goplus/llgo/internal/targets" "github.com/goplus/llgo/internal/xtool/llvm" ) +func TestCompileWithConfigUsesLinkerSpecificNoStdlib(t *testing.T) { + for _, tt := range []struct { + name string + linker string + wantNoStdlib bool + }{ + {name: "elf-lld", linker: "/toolchain/bin/ld.lld", wantNoStdlib: true}, + {name: "wasm-ld", linker: "/toolchain/bin/wasm-ld", wantNoStdlib: false}, + } { + t.Run(tt.name, func(t *testing.T) { + flags, err := compileWithConfig(compile.CompileConfig{}, "/cache/lib", compile.CompileOptions{Linker: tt.linker}) + if err != nil { + t.Fatal(err) + } + if got := slices.Contains(flags, "-nostdlib"); got != tt.wantNoStdlib { + t.Fatalf("compileWithConfig linker %q flags = %v, -nostdlib=%v, want %v", + tt.linker, flags, got, tt.wantNoStdlib) + } + if !slices.Contains(flags, "-L/cache/lib") { + t.Fatalf("compileWithConfig flags = %v, want library search path", flags) + } + }) + } +} + +func TestLinkerHelpSupportsICF(t *testing.T) { + for _, tt := range []struct { + name string + help string + want bool + }{ + {name: "esp-llvm19-no-icf", help: "--import-memory\n--no-entry\n--export=\n", want: false}, + {name: "lld-equals-form", help: "--icf={none,safe,all} Perform identical code folding\n", want: true}, + {name: "lld-separated-form", help: "--icf Perform identical code folding\n", want: true}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := linkerHelpSupportsICF(tt.help); got != tt.want { + t.Fatalf("linkerHelpSupportsICF(%q) = %v, want %v", tt.help, got, tt.want) + } + }) + } +} + +func TestValidateWasmBuiltinsTargetCompatibility(t *testing.T) { + for _, tt := range []struct { + name string + config targets.Config + wantErr string + }{ + { + name: "wasip2-core-module", + config: targets.Config{ + LLVMTarget: "wasm32-unknown-wasi", + Linker: "wasm-ld", + Libc: "wasmbuiltins", + }, + }, + { + name: "unknown-unknown", + config: targets.Config{ + LLVMTarget: "wasm32-unknown-unknown", + Linker: "wasm-ld", + Libc: "wasmbuiltins", + }, + }, + { + name: "native-target", + config: targets.Config{ + LLVMTarget: "aarch64-unknown-linux-gnu", + Linker: "wasm-ld", + Libc: "wasmbuiltins", + }, + wantErr: "requires a wasm32 LLVM target", + }, + { + name: "wrong-linker", + config: targets.Config{ + LLVMTarget: "wasm32-unknown-unknown", + Linker: "ld.lld", + Libc: "wasmbuiltins", + }, + wantErr: "requires linker wasm-ld", + }, + { + name: "unsupported-threads", + config: targets.Config{ + LLVMTarget: "wasm32-unknown-wasi", + Linker: "wasm-ld", + Libc: "wasmbuiltins", + Features: "+bulk-memory,+atomics", + }, + wantErr: "does not provide a threaded malloc/errno ABI", + }, + { + name: "unrelated-libc", + config: targets.Config{ + LLVMTarget: "aarch64-unknown-linux-gnu", + Linker: "ld.lld", + Libc: "picolibc", + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := validateLibcTargetCompatibility(&tt.config) + if tt.wantErr == "" { + if err != nil { + t.Fatal(err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("validate error = %v, want substring %q", err, tt.wantErr) + } + }) + } +} + const ( sysrootPrefix = "--sysroot=" resourceDirPrefix = "-resource-dir=" @@ -357,6 +476,25 @@ func TestUseWithTarget(t *testing.T) { } } +func TestUseNamedWasmTargetResolvesTargetAndGC(t *testing.T) { + export, err := Use(runtime.GOOS, runtime.GOARCH, "wasm", false, false, optlevel.Oz, lto.Off, false) + if err != nil { + t.Fatal(err) + } + if export.GOOS != "js" || export.GOARCH != "wasm" { + t.Fatalf("named wasm Go target = %s/%s, want js/wasm", export.GOOS, export.GOARCH) + } + if !strings.HasPrefix(export.LLVMTarget, "wasm32-") { + t.Fatalf("named wasm LLVM target = %q, want wasm32 triple", export.LLVMTarget) + } + if export.GC != "leaking" { + t.Fatalf("named wasm GC = %q, want leaking", export.GC) + } + if !slices.Contains(export.BuildTags, "tinygo.wasm") { + t.Fatalf("named wasm build tags = %v, want tinygo.wasm", export.BuildTags) + } +} + func TestOptimizationFlagPlacement(t *testing.T) { export, err := UseTarget("rp2040", optlevel.Oz, lto.Off) if err != nil { diff --git a/internal/crosscompile/libc.go b/internal/crosscompile/libc.go index 225a4a56c4..56ee9df59e 100644 --- a/internal/crosscompile/libc.go +++ b/internal/crosscompile/libc.go @@ -32,6 +32,13 @@ func getLibcCompileConfigByName(baseDir, libcName, target, mcpu string) (outputD config = libc.GetNewlibESP32Config() libcDir = filepath.Join(baseDir, config.String()) compileConfig = libc.GetNewlibESP32CompileConfig(libcDir, target, mcpu) + case "wasmbuiltins": + config = libc.GetWasmBuiltinsConfig() + libcDir = filepath.Join(baseDir, config.String()) + // The skipped-download test path only inspects the declarative recipe. + // Use the final deterministic location without touching the filesystem. + includeDir := filepath.Join(libcDir, "llgo-wasmbuiltins-include") + compileConfig = libc.GetWasmBuiltinsCompileConfig(libcDir, includeDir, target) default: err = fmt.Errorf("unsupported libc: %s", libcName) return @@ -43,6 +50,14 @@ func getLibcCompileConfigByName(baseDir, libcName, target, mcpu string) (outputD if err = checkDownloadAndExtractLib(config.Url, libcDir, config.ResourceSubDir); err != nil { return } + if libcName == "wasmbuiltins" { + var includeDir string + includeDir, err = libc.PrepareWasmBuiltinsHeaders(libcDir) + if err != nil { + return + } + compileConfig = libc.GetWasmBuiltinsCompileConfig(libcDir, includeDir, target) + } return libcDir, compileConfig, nil } diff --git a/internal/crosscompile/libc_test.go b/internal/crosscompile/libc_test.go index f03a46467a..f15b04fe0a 100644 --- a/internal/crosscompile/libc_test.go +++ b/internal/crosscompile/libc_test.go @@ -76,6 +76,31 @@ func TestGetLibcCompileConfigByName(t *testing.T) { t.Errorf("Expected flags %v, got: %v", expectedFlags, group.CFlags) } }) + + t.Run("WasmBuiltins", func(t *testing.T) { + wasmTarget := "wasm32-unknown-unknown" + outputDir, cfg, err := getLibcCompileConfigByName(baseDir, "wasmbuiltins", wasmTarget, "generic") + if err != nil { + t.Fatalf("wasmbuiltins setup failed: %v", err) + } + expectedDir := filepath.Join(baseDir, libc.GetWasmBuiltinsConfig().String()) + if outputDir != expectedDir { + t.Fatalf("wasmbuiltins output dir = %q, want %q", outputDir, expectedDir) + } + if len(cfg.Groups) != 1 { + t.Fatalf("wasmbuiltins groups = %d, want 1", len(cfg.Groups)) + } + group := cfg.Groups[0] + if group.OutputFileName != "libwasmbuiltins-"+wasmTarget+".a" { + t.Fatalf("wasmbuiltins archive = %q", group.OutputFileName) + } + if !slices.Contains(group.Files, filepath.Join(expectedDir, "libc-top-half", "musl", "src", "string", "memcpy.c")) { + t.Fatalf("wasmbuiltins files = %v, want pinned wasi-libc memcpy", group.Files) + } + if !slices.Contains(group.CFlags, "-I"+filepath.Join(expectedDir, "llgo-wasmbuiltins-include")) { + t.Fatalf("wasmbuiltins flags = %v, want generated header path", group.CFlags) + } + }) } func TestGetRTCompileConfigByName(t *testing.T) { diff --git a/internal/crosscompile/testdata/wasm_allocator/main.go b/internal/crosscompile/testdata/wasm_allocator/main.go new file mode 100644 index 0000000000..ba526037ec --- /dev/null +++ b/internal/crosscompile/testdata/wasm_allocator/main.go @@ -0,0 +1,35 @@ +//go:build tinygo.wasm + +package main + +import "unsafe" + +// The named freestanding targets provide these symbols through their +// triple-scoped wasmbuiltins archive. Linknames keep this fixture independent +// of a host C sysroot and make the real llgo -target route exercise that ABI. +// +//go:linkname malloc malloc +func malloc(size uintptr) unsafe.Pointer + +//go:linkname free free +func free(ptr unsafe.Pointer) + +//go:linkname abort abort +func abort() + +func main() { + const size = uintptr(64) + ptr := malloc(size) + if ptr == nil { + abort() + } + for offset := uintptr(0); offset < size; offset++ { + *(*byte)(unsafe.Add(ptr, offset)) = byte(offset + 1) + } + for offset := uintptr(0); offset < size; offset++ { + if *(*byte)(unsafe.Add(ptr, offset)) != byte(offset+1) { + abort() + } + } + free(ptr) +} diff --git a/internal/crosscompile/wasm_target_smoke_test.go b/internal/crosscompile/wasm_target_smoke_test.go new file mode 100644 index 0000000000..2b058573ac --- /dev/null +++ b/internal/crosscompile/wasm_target_smoke_test.go @@ -0,0 +1,173 @@ +//go:build !llgo + +package crosscompile + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/goplus/llgo/internal/clang" + "github.com/goplus/llgo/internal/lto" + "github.com/goplus/llgo/internal/optlevel" +) + +// TestFreestandingWasmTargetToolchainSmoke is an opt-in integration test. It +// exercises the exact named-target setup, compiles a real wasm object, links +// it with the target's wasmbuiltins/compiler-rt archives, and audits both link +// configuration and final symbols. CI enables it explicitly; ordinary unit +// tests do not download target toolchains or source archives. +func TestFreestandingWasmTargetToolchainSmoke(t *testing.T) { + if os.Getenv("LLGO_WASM_TARGET_SMOKE") != "1" { + t.Skip("set LLGO_WASM_TARGET_SMOKE=1 to compile and link named wasm targets") + } + + for _, targetName := range []string{"wasip2", "wasm-unknown"} { + t.Run(targetName, func(t *testing.T) { + // Use is the path taken by the llgo driver's -target flag. Host + // GOOS/GOARCH are intentionally supplied here to prove the named + // target, rather than ambient GOOS/tags, owns backend selection. + export, err := Use("host-os", "host-arch", targetName, false, true, optlevel.Oz, lto.Off, false) + if err != nil { + t.Fatalf("setup -target=%s: %v", targetName, err) + } + wantTriple := map[string]string{ + "wasip2": "wasm32-unknown-wasi", + "wasm-unknown": "wasm32-unknown-unknown", + }[targetName] + if export.LLVMTarget != wantTriple { + t.Fatalf("-target=%s LLVM triple = %q, want %q", targetName, export.LLVMTarget, wantTriple) + } + if export.GOOS != "linux" || export.GOARCH != "arm" { + t.Fatalf("-target=%s frontend = %s/%s, want the explicit 32-bit linux/arm frontend", + targetName, export.GOOS, export.GOARCH) + } + if export.Libc != "wasmbuiltins" { + t.Fatalf("-target=%s libc = %q, want freestanding wasmbuiltins", targetName, export.Libc) + } + assertNoConservativeGCLinkInputs(t, export) + + dir := t.TempDir() + source := filepath.Join(dir, "smoke.c") + object := filepath.Join(dir, "smoke.o") + module := filepath.Join(dir, targetName+".wasm") + const smokeSource = ` +typedef __SIZE_TYPE__ size_t; +extern void *memcpy(void *, const void *, size_t); +extern double exp(double); +extern void *malloc(size_t); +extern void free(void *); +__attribute__((visibility("default"))) +int llgo_wasm_target_smoke(void) { + const unsigned char src[4] = {1, 2, 3, 4}; + unsigned char *dst = (unsigned char *)malloc(64); + if (dst == (void *)0) { + return 10; + } + memcpy(dst, src, 4); + double value = exp((double)dst[0]); + int result = dst[3] == 4 && value > 2.0 ? 0 : 20; + free(dst); + return result; +} +` + if err := os.WriteFile(source, []byte(smokeSource), 0o644); err != nil { + t.Fatal(err) + } + + cfg := clang.NewConfig(export.CC, export.CCFLAGS, export.CFLAGS, export.LDFLAGS, export.Linker) + compiler := clang.NewCompiler(cfg) + if err := compiler.Compile("-fno-builtin", "-x", "c", "-c", source, "-o", object); err != nil { + t.Fatalf("compile -target=%s smoke object: %v", targetName, err) + } + linker := clang.NewLinker(cfg) + if err := linker.Link("--export=llgo_wasm_target_smoke", "-o", module, object); err != nil { + t.Fatalf("link -target=%s smoke module: %v", targetName, err) + } + + contents, err := os.ReadFile(module) + if err != nil { + t.Fatal(err) + } + if len(contents) < 8 || !bytes.Equal(contents[:4], []byte{'\x00', 'a', 's', 'm'}) { + t.Fatalf("-target=%s output is not a WebAssembly module", targetName) + } + assertClosedWasmSymbols(t, export, module) + if wasmtime, lookErr := exec.LookPath("wasmtime"); lookErr == nil { + cmd := exec.Command(wasmtime, "run", "--invoke", "llgo_wasm_target_smoke", module) + if output, runErr := cmd.CombinedOutput(); runErr != nil { + t.Fatalf("execute -target=%s allocator smoke: %v\n%s", targetName, runErr, output) + } + t.Logf("executed -target=%s malloc/write/read/free smoke with %s", targetName, wasmtime) + } else { + t.Logf("wasmtime unavailable; compile/link/symbol closure for -target=%s is verified, execution skipped", targetName) + } + }) + } +} + +func assertNoConservativeGCLinkInputs(t *testing.T, export Export) { + t.Helper() + for _, flag := range export.LDFLAGS { + lower := strings.ToLower(flag) + if flag == "-lgc" || strings.Contains(lower, "libgc") || strings.Contains(lower, "bdwgc") || + strings.Contains(lower, "rpath") { + t.Fatalf("WebAssembly link flags contain forbidden conservative-GC/runtime path %q: %v", flag, export.LDFLAGS) + } + } + if !slices.ContainsFunc(export.LDFLAGS, func(flag string) bool { + return strings.Contains(flag, "wasmbuiltins-wasm32") + }) { + t.Fatalf("WebAssembly link flags do not contain a triple-scoped wasmbuiltins archive: %v", export.LDFLAGS) + } +} + +func assertClosedWasmSymbols(t *testing.T, export Export, module string) { + t.Helper() + nm := filepath.Join(filepath.Dir(export.CC), "llvm-nm") + if _, err := os.Stat(nm); err != nil { + if path, lookErr := exec.LookPath("llvm-nm"); lookErr == nil { + nm = path + } else { + t.Fatalf("WebAssembly toolchain capability missing: llvm-nm next to %q and on PATH", export.CC) + } + } + cmd := exec.Command(nm, "--defined-only", "--format=just-symbols", module) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("inspect %s symbols: %v\n%s", module, err, output) + } + definedSymbols := strings.Fields(string(output)) + for _, symbol := range definedSymbols { + if strings.HasPrefix(symbol, "GC_") { + t.Fatalf("final WebAssembly module contains BDWGC symbol %s:\n%s", symbol, output) + } + } + for _, required := range []string{"llgo_wasm_target_smoke", "malloc", "free", "sbrk"} { + if !slices.Contains(definedSymbols, required) { + t.Fatalf("final WebAssembly module is missing required allocator symbol %s:\n%s", required, output) + } + } + undefined := exec.Command(nm, "--undefined-only", "--format=just-symbols", module) + undefinedOutput, err := undefined.CombinedOutput() + if err != nil { + t.Fatalf("inspect %s undefined symbols: %v\n%s", module, err, undefinedOutput) + } + if len(bytes.TrimSpace(undefinedOutput)) != 0 { + t.Fatalf("final WebAssembly module has unresolved symbols:\n%s", undefinedOutput) + } + t.Logf("linked %s with %s (%d bytes)", filepath.Base(module), export.Linker, fileSize(t, module)) +} + +func fileSize(t *testing.T, path string) int64 { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return info.Size() +} diff --git a/internal/targets/config.go b/internal/targets/config.go index 1d56e7d6d4..1850005e0f 100644 --- a/internal/targets/config.go +++ b/internal/targets/config.go @@ -28,6 +28,12 @@ type Config struct { CodeModel string `json:"code-model"` TargetABI string `json:"target-abi"` RelocationModel string `json:"relocation-model"` + // GC is the runtime memory-management capability selected by this target. + // Supported values match the target JSON vocabulary: precise, + // conservative, leaking, and none. A leaking/none profile is consumed by + // the build pipeline as the explicit nogc runtime rather than being treated + // as documentation-only metadata. + GC string `json:"gc"` // Binary and firmware configuration BinaryFormat string `json:"binary-format"` diff --git a/internal/targets/loader.go b/internal/targets/loader.go index 5603ddcd70..daa147e241 100644 --- a/internal/targets/loader.go +++ b/internal/targets/loader.go @@ -155,6 +155,9 @@ func (l *Loader) mergeConfig(dst, src *Config) { if src.RelocationModel != "" { dst.RelocationModel = src.RelocationModel } + if src.GC != "" { + dst.GC = src.GC + } if src.BinaryFormat != "" { dst.BinaryFormat = src.BinaryFormat } diff --git a/internal/targets/resolver.go b/internal/targets/resolver.go index 6d50ffa451..a4ad684335 100644 --- a/internal/targets/resolver.go +++ b/internal/targets/resolver.go @@ -3,6 +3,7 @@ package targets import ( "fmt" "path/filepath" + "slices" "github.com/goplus/llgo/internal/env" ) @@ -57,6 +58,9 @@ func (r *Resolver) validateConfig(config *Config) error { if config.Name == "" { return fmt.Errorf("target name is required") } + if config.GC != "" && !slices.Contains([]string{"precise", "conservative", "leaking", "none"}, config.GC) { + return fmt.Errorf("unsupported gc capability %q", config.GC) + } // For now, we don't require any specific fields since different targets // may have different requirements. This can be extended in the future. diff --git a/internal/targets/targets_test.go b/internal/targets/targets_test.go index bf2407e0d4..34fecded6e 100644 --- a/internal/targets/targets_test.go +++ b/internal/targets/targets_test.go @@ -331,3 +331,28 @@ func TestResolveAllRealTargets(t *testing.T) { t.Logf("GOOS distribution: %v", goosCounts) t.Logf("GOARCH distribution: %v", goarchCounts) } + +func TestWebAssemblyTargetsDeclareLeakingGC(t *testing.T) { + resolver := NewDefaultResolver() + for _, name := range []string{"wasm", "wasip1", "wasip2", "wasm-unknown"} { + t.Run(name, func(t *testing.T) { + config, err := resolver.Resolve(name) + if err != nil { + t.Fatal(err) + } + if config.GC != "leaking" { + t.Fatalf("target GC = %q, want leaking", config.GC) + } + }) + } +} + +func TestResolverRejectsUnknownGC(t *testing.T) { + tempDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tempDir, "bad.json"), []byte(`{"gc":"magic"}`), 0644); err != nil { + t.Fatal(err) + } + if _, err := NewResolver(tempDir).Resolve("bad"); err == nil { + t.Fatal("Resolve accepted an unknown GC capability") + } +} diff --git a/targets/wasip1.json b/targets/wasip1.json index b916de0457..675c5a4509 100644 --- a/targets/wasip1.json +++ b/targets/wasip1.json @@ -8,7 +8,7 @@ "linker": "wasm-ld", "libc": "wasi-libc", "rtlib": "compiler-rt", - "gc": "precise", + "gc": "leaking", "scheduler": "asyncify", "default-stack-size": 65536, "cflags": [ diff --git a/targets/wasip2.json b/targets/wasip2.json index 1b30db93c1..59013953fd 100644 --- a/targets/wasip2.json +++ b/targets/wasip2.json @@ -9,7 +9,7 @@ "linker": "wasm-ld", "libc": "wasmbuiltins", "rtlib": "compiler-rt", - "gc": "precise", + "gc": "leaking", "scheduler": "asyncify", "default-stack-size": 65536, "cflags": [ diff --git a/targets/wasm.json b/targets/wasm.json index c61b607dcc..4d7142c5bf 100644 --- a/targets/wasm.json +++ b/targets/wasm.json @@ -8,7 +8,7 @@ "linker": "wasm-ld", "libc": "wasi-libc", "rtlib": "compiler-rt", - "gc": "precise", + "gc": "leaking", "scheduler": "asyncify", "default-stack-size": 65536, "cflags": [ From d9694a384d21ef11a7a7bee3c8db196a76c65c7f Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 22:31:35 +0800 Subject: [PATCH 13/32] runtime(coro): add preemptive park and wake core --- runtime/gc_target_selection_test.go | 85 +++ runtime/internal/coro/bootstrap.go | 375 +++++++++- runtime/internal/coro/bootstrap_test.go | 247 +++++++ runtime/internal/coro/frame.go | 69 +- runtime/internal/coro/frame_test.go | 35 +- runtime/internal/coro/preempt_atomic_host.go | 33 + runtime/internal/coro/preempt_atomic_llgo.go | 44 ++ runtime/internal/coro/scheduler.go | 430 +++++++++++- .../internal/coro/scheduler_preempt_test.go | 290 ++++++++ runtime/internal/coro/scheduler_wait_test.go | 661 ++++++++++++++++++ runtime/internal/coro/scheduler_yield_test.go | 159 +++++ runtime/internal/coro/wait.go | 172 +++++ runtime/internal/coroalloc/allocator.go | 105 +++ runtime/internal/coroalloc/allocator_test.go | 82 +++ .../internal/coroalloc/backend_baremetal.go | 41 ++ .../coroalloc/backend_baremetal_test.go | 27 + runtime/internal/coroalloc/backend_gc.go | 40 ++ runtime/internal/coroalloc/backend_gc_test.go | 27 + runtime/internal/coroalloc/backend_nogc.go | 39 ++ .../internal/coroalloc/backend_nogc_test.go | 27 + .../backend_target_selection_test.go | 79 +++ .../internal/coroalloc/backend_webassembly.go | 42 ++ .../coroalloc/backend_webassembly_test.go | 49 ++ .../coroalloc/testdata/wasm_backend/main.go | 48 ++ runtime/internal/lib/runtime/mfinal.go | 2 + runtime/internal/lib/runtime/mfinal_nogc.go | 24 + runtime/internal/lib/runtime/runtime_gc.go | 8 +- runtime/internal/lib/runtime/runtime_nogc.go | 13 +- runtime/internal/runtime/coro_allocator.go | 26 + runtime/internal/runtime/coro_frame.go | 43 +- .../internal/runtime/coro_park_intrinsic.go | 32 + runtime/internal/runtime/coro_program.go | 83 +-- runtime/internal/runtime/coro_program_test.go | 121 +++- runtime/internal/runtime/coro_sched.go | 9 +- .../internal/runtime/tinygogc/gc_tinygo.go | 9 + runtime/internal/runtime/z_signal.go | 2 +- 36 files changed, 3461 insertions(+), 117 deletions(-) create mode 100644 runtime/gc_target_selection_test.go create mode 100644 runtime/internal/coro/preempt_atomic_host.go create mode 100644 runtime/internal/coro/preempt_atomic_llgo.go create mode 100644 runtime/internal/coro/scheduler_preempt_test.go create mode 100644 runtime/internal/coro/scheduler_wait_test.go create mode 100644 runtime/internal/coro/scheduler_yield_test.go create mode 100644 runtime/internal/coro/wait.go create mode 100644 runtime/internal/coroalloc/allocator.go create mode 100644 runtime/internal/coroalloc/allocator_test.go create mode 100644 runtime/internal/coroalloc/backend_baremetal.go create mode 100644 runtime/internal/coroalloc/backend_baremetal_test.go create mode 100644 runtime/internal/coroalloc/backend_gc.go create mode 100644 runtime/internal/coroalloc/backend_gc_test.go create mode 100644 runtime/internal/coroalloc/backend_nogc.go create mode 100644 runtime/internal/coroalloc/backend_nogc_test.go create mode 100644 runtime/internal/coroalloc/backend_target_selection_test.go create mode 100644 runtime/internal/coroalloc/backend_webassembly.go create mode 100644 runtime/internal/coroalloc/backend_webassembly_test.go create mode 100644 runtime/internal/coroalloc/testdata/wasm_backend/main.go create mode 100644 runtime/internal/lib/runtime/mfinal_nogc.go create mode 100644 runtime/internal/runtime/coro_allocator.go create mode 100644 runtime/internal/runtime/coro_park_intrinsic.go diff --git a/runtime/gc_target_selection_test.go b/runtime/gc_target_selection_test.go new file mode 100644 index 0000000000..197c0ad387 --- /dev/null +++ b/runtime/gc_target_selection_test.go @@ -0,0 +1,85 @@ +//go:build !llgo + +package runtime + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "testing" +) + +func TestLeakingWebAssemblyProfilesExcludeBDWGC(t *testing.T) { + targets := []struct { + name string + goos string + goarch string + tags string + }{ + {name: "js-wasm", goos: "js", goarch: "wasm", tags: "llgo,tinygo.wasm,nogc"}, + {name: "wasip1", goos: "wasip1", goarch: "wasm", tags: "llgo,tinygo.wasm,nogc"}, + {name: "wasip2", goos: "linux", goarch: "arm", tags: "llgo,tinygo.wasm,wasip2,nogc"}, + {name: "wasm-unknown", goos: "linux", goarch: "arm", tags: "llgo,tinygo.wasm,wasm_unknown,nogc"}, + } + moduleRoot, err := filepath.Abs(".") + if err != nil { + t.Fatal(err) + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + cmd := exec.Command("go", "list", "-deps", "-json", "-tags="+target.tags, + "./internal/runtime", "./internal/lib/runtime", "./internal/clite/pthread", "./internal/clite/tls") + cmd.Dir = moduleRoot + cmd.Env = append(os.Environ(), "GOOS="+target.goos, "GOARCH="+target.goarch, "CGO_ENABLED=0") + output, err := cmd.Output() + if err != nil { + t.Fatalf("go list leaking target packages: %v", err) + } + decoder := json.NewDecoder(bytes.NewReader(output)) + packages := make(map[string]struct { + GoFiles []string + }) + for { + var pkg struct { + ImportPath string + GoFiles []string + } + if err := decoder.Decode(&pkg); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatalf("decode go list stream: %v", err) + } + if pkg.ImportPath == "github.com/goplus/llgo/runtime/internal/clite/bdwgc" { + t.Fatal("leaking target dependency graph retained BDWGC") + } + packages[pkg.ImportPath] = struct{ GoFiles []string }{GoFiles: pkg.GoFiles} + } + assertFiles := func(path string, required, forbidden []string) { + t.Helper() + pkg, ok := packages[path] + if !ok { + t.Fatalf("go list stream is missing %s", path) + } + for _, file := range required { + if !slices.Contains(pkg.GoFiles, file) { + t.Fatalf("%s GoFiles = %v, want %s", path, pkg.GoFiles, file) + } + } + for _, file := range forbidden { + if slices.Contains(pkg.GoFiles, file) { + t.Fatalf("%s GoFiles = %v, unexpectedly selected %s", path, pkg.GoFiles, file) + } + } + } + assertFiles("github.com/goplus/llgo/runtime/internal/runtime", []string{"z_nogc.go"}, []string{"z_gc.go"}) + assertFiles("github.com/goplus/llgo/runtime/internal/lib/runtime", []string{"runtime_nogc.go", "mfinal_nogc.go"}, []string{"runtime_gc.go", "mfinal.go"}) + assertFiles("github.com/goplus/llgo/runtime/internal/clite/pthread", []string{"pthread_nogc.go"}, []string{"pthread_gc.go"}) + assertFiles("github.com/goplus/llgo/runtime/internal/clite/tls", []string{"tls_nogc.go"}, []string{"tls_gc.go"}) + }) + } +} diff --git a/runtime/internal/coro/bootstrap.go b/runtime/internal/coro/bootstrap.go index 11cc462b6a..a598cd7158 100644 --- a/runtime/internal/coro/bootstrap.go +++ b/runtime/internal/coro/bootstrap.go @@ -18,16 +18,17 @@ package coro import "unsafe" -// The v1 bootstrap ABI is deliberately pointer-size neutral. These structures -// mirror compiler-emitted LLVM constants; keep uintptr and pointer fields in -// the same order so the layouts also match wasm32, embedded, and bare-metal -// targets. Non-null pointers come from the linked program image and therefore -// must denote readable constants; structural validation can reject alignment, -// count, and address overflow, but cannot safely probe an arbitrary unmapped -// address supplied by untrusted native memory. +// The bootstrap ABI layouts are deliberately pointer-size neutral. These +// structures mirror compiler-emitted LLVM constants; keep uintptr and pointer +// fields in the same order so the layouts also match wasm32, embedded, and +// bare-metal targets. Non-null pointers come from the linked program image and +// therefore must denote readable constants; structural validation can reject +// alignment, count, and address overflow, but cannot safely probe an arbitrary +// unmapped address supplied by untrusted native memory. const ( ProgramManifestVersionV1 uint32 = 1 ProgramBootstrapVersionV1 uint32 = 1 + ProgramBootstrapVersionV2 uint32 = 2 RootPackageAnchorVersionV1 uint32 = 1 RootFactoryVersionV1 uint32 = 1 ) @@ -41,11 +42,33 @@ const ( ProgramStepCoroRootV1 ProgramStepKindV1 = 2 ) +// Version two deliberately reuses the version-one step representation and +// kind numbers. These aliases let startup-driver code remain version-explicit +// without defining a second physical layout. +type ProgramStepKindV2 = ProgramStepKindV1 + +const ( + ProgramStepDirectPlainV2 ProgramStepKindV2 = ProgramStepDirectPlainV1 + ProgramStepCoroRootV2 ProgramStepKindV2 = ProgramStepCoroRootV1 +) + const ( ProgramStepFlagInitV1 uint32 = 1 << iota ProgramStepFlagMainV1 ) +// Version-two roles describe the complete heterogeneous startup sequence. +// Their bit values are scoped by ProgramBootstrapVersionV2 and therefore may +// overlap the version-one roles. Every table entry must contain exactly the +// role at its canonical position. +const ( + ProgramStepFlagInternalRuntimeInitV2 uint32 = 1 << iota + ProgramStepFlagCompilerABIInitV2 + ProgramStepFlagPublicRuntimeInitV2 + ProgramStepFlagMainPackageInitV2 + ProgramStepFlagMainV2 +) + // ProgramManifestV1 is the runtime view of // __llgo_coro_program_manifest_v1. type ProgramManifestV1 struct { @@ -82,6 +105,12 @@ type ProgramStepV1 struct { Aux uintptr } +// ProgramBootstrapV2 and ProgramStepV2 reuse the pointer-size-neutral v1 +// physical layouts. ProgramBootstrapV2 is distinguished by Version == 2 and +// by its exact five-role step program. +type ProgramBootstrapV2 = ProgramBootstrapV1 +type ProgramStepV2 = ProgramStepV1 + // RootPackageAnchorV1 mirrors the package registry emitted by cl. type RootPackageAnchorV1 struct { Version uint32 @@ -188,6 +217,31 @@ const ( programArrayAddressV1 ) +// checkedProgramSpanV1 performs a full-width uintptr multiplication without +// division. Division-by-zero guards in compiler-owned runtime validation would +// otherwise introduce an async panic helper into the synchronous process-entry +// ABI, even though checkedProgramArrayV1 rejects a zero element size first. +func checkedProgramSpanV1(count, size uintptr) (uintptr, bool) { + const mask32 uint64 = 1<<32 - 1 + x := uint64(count) + y := uint64(size) + x0 := x & mask32 + x1 := x >> 32 + y0 := y & mask32 + y1 := y >> 32 + w0 := x0 * y0 + t := x1*y0 + w0>>32 + w1 := t & mask32 + w2 := t >> 32 + w1 += x0 * y1 + hi := x1*y1 + w2 + w1>>32 + lo := x * y + if hi != 0 || lo > uint64(^uintptr(0)) { + return 0, false + } + return uintptr(lo), true +} + func checkedProgramArrayV1(base unsafe.Pointer, count, size, align uintptr) programArrayStateV1 { if count == 0 { if base != nil { @@ -200,10 +254,13 @@ func checkedProgramArrayV1(base unsafe.Pointer, count, size, align uintptr) prog } address := uintptr(base) if align == 0 || align&(align-1) != 0 || address&(align-1) != 0 || - size == 0 || count > ^uintptr(0)/size { + size == 0 { + return programArrayAddressV1 + } + span, ok := checkedProgramSpanV1(count, size) + if !ok { return programArrayAddressV1 } - span := count * size if address > ^uintptr(0)-(span-1) { return programArrayAddressV1 } @@ -529,3 +586,303 @@ func ResolveProgramStepV1(program ProgramViewV1, index uintptr) (ResolvedProgram return ResolvedProgramStepV1{}, ProgramValidationStepIndexV1 } } + +// ProgramValidationCodeV2 is the allocation-free result of validating the +// version-two heterogeneous startup table. It is deliberately independent of +// ProgramValidationCodeV1 even where both ABIs reject the same physical field. +type ProgramValidationCodeV2 uint32 + +const ( + ProgramValidationOKV2 ProgramValidationCodeV2 = iota + ProgramValidationNilManifestV2 + ProgramValidationManifestAddressV2 + ProgramValidationManifestVersionV2 + ProgramValidationManifestFlagsV2 + ProgramValidationPackageCountPointerV2 + ProgramValidationPackageTableAddressV2 + ProgramValidationNilBootstrapV2 + ProgramValidationBootstrapAddressV2 + ProgramValidationBootstrapVersionV2 + ProgramValidationBootstrapFlagsV2 + ProgramValidationBootstrapHashV2 + ProgramValidationStepCountV2 + ProgramValidationStepCountPointerV2 + ProgramValidationStepTableAddressV2 + ProgramValidationBootstrapFactoryV2 + ProgramValidationNilPackageAnchorV2 + ProgramValidationPackageAnchorAddressV2 + ProgramValidationDuplicatePackageAnchorV2 + ProgramValidationPackageAnchorVersionV2 + ProgramValidationPackageAnchorFlagsV2 + ProgramValidationEmptyPackageAnchorV2 + ProgramValidationDescriptorCountPointerV2 + ProgramValidationDescriptorTableAddressV2 + ProgramValidationNilRootDescriptorV2 + ProgramValidationRootDescriptorAddressV2 + ProgramValidationDuplicateRootDescriptorV2 + ProgramValidationRootDescriptorVersionV2 + ProgramValidationRootDescriptorFlagsV2 + ProgramValidationRootDescriptorFactoryV2 + ProgramValidationRootStartupLayoutV2 + ProgramValidationRootResultLayoutV2 + ProgramValidationStepRoleV2 + ProgramValidationStepKindV2 + ProgramValidationStepTargetV2 + ProgramValidationStepAuxV2 + ProgramValidationStepAnchorV2 + ProgramValidationStepDescriptorIndexV2 + ProgramValidationStepPayloadV2 + ProgramValidationInvalidViewV2 + ProgramValidationStepIndexV2 + ProgramValidationBootstrapFactoryIdentityV2 +) + +// ResolvedProgramStepV2 is one validated heterogeneous startup action. Exactly +// one representation is populated: Plain for DirectPlain, or Descriptor and +// Factory for CoroRoot. Resolving a step never invokes either target. +type ResolvedProgramStepV2 struct { + Kind ProgramStepKindV2 + Flags uint32 + Plain unsafe.Pointer + Descriptor *RootFactoryDescriptorV1 + Factory unsafe.Pointer +} + +const validatedProgramMagicV2 uint32 = 0x42535432 // "BST2" + +// ProgramViewV2 is an immutable, allocation-free snapshot of the five startup +// actions. Its contents are private so only successful validation can produce +// a resolvable value. +type ProgramViewV2 struct { + magic uint32 + factory unsafe.Pointer + internalRuntimeInit ResolvedProgramStepV2 + compilerABIInit ResolvedProgramStepV2 + publicRuntimeInit ResolvedProgramStepV2 + mainPackageInit ResolvedProgramStepV2 + main ResolvedProgramStepV2 +} + +const programStepCountV2 uintptr = 5 + +// programCatalogValidationV2 translates validation of the shared v1 physical +// package/descriptor catalog into the independent v2 result namespace. +func programCatalogValidationV2(code ProgramValidationCodeV1) ProgramValidationCodeV2 { + switch code { + case ProgramValidationOKV1: + return ProgramValidationOKV2 + case ProgramValidationNilPackageAnchorV1: + return ProgramValidationNilPackageAnchorV2 + case ProgramValidationPackageAnchorAddressV1: + return ProgramValidationPackageAnchorAddressV2 + case ProgramValidationDuplicatePackageAnchorV1: + return ProgramValidationDuplicatePackageAnchorV2 + case ProgramValidationPackageAnchorVersionV1: + return ProgramValidationPackageAnchorVersionV2 + case ProgramValidationPackageAnchorFlagsV1: + return ProgramValidationPackageAnchorFlagsV2 + case ProgramValidationEmptyPackageAnchorV1: + return ProgramValidationEmptyPackageAnchorV2 + case ProgramValidationDescriptorCountPointerV1: + return ProgramValidationDescriptorCountPointerV2 + case ProgramValidationDescriptorTableAddressV1: + return ProgramValidationDescriptorTableAddressV2 + case ProgramValidationNilRootDescriptorV1: + return ProgramValidationNilRootDescriptorV2 + case ProgramValidationRootDescriptorAddressV1: + return ProgramValidationRootDescriptorAddressV2 + case ProgramValidationDuplicateRootDescriptorV1: + return ProgramValidationDuplicateRootDescriptorV2 + case ProgramValidationRootDescriptorVersionV1: + return ProgramValidationRootDescriptorVersionV2 + case ProgramValidationRootDescriptorFlagsV1: + return ProgramValidationRootDescriptorFlagsV2 + case ProgramValidationRootDescriptorFactoryV1: + return ProgramValidationRootDescriptorFactoryV2 + case ProgramValidationRootStartupLayoutV1: + return ProgramValidationRootStartupLayoutV2 + case ProgramValidationRootResultLayoutV1: + return ProgramValidationRootResultLayoutV2 + default: + // validateProgramCatalogV1 can only return the cases above. Keep this + // fail closed if that implementation gains a new result. + return ProgramValidationPackageTableAddressV2 + } +} + +func resolveValidatedProgramStepV2( + manifest *ProgramManifestV1, step *ProgramStepV1, expectedRole uint32, +) (ResolvedProgramStepV2, ProgramValidationCodeV2) { + if step.Flags != expectedRole { + return ResolvedProgramStepV2{}, ProgramValidationStepRoleV2 + } + if step.Target == nil { + return ResolvedProgramStepV2{}, ProgramValidationStepTargetV2 + } + switch ProgramStepKindV2(step.Kind) { + case ProgramStepDirectPlainV2: + if step.Aux != 0 { + return ResolvedProgramStepV2{}, ProgramValidationStepAuxV2 + } + return ResolvedProgramStepV2{ + Kind: ProgramStepDirectPlainV2, + Flags: step.Flags, + Plain: step.Target, + }, ProgramValidationOKV2 + case ProgramStepCoroRootV2: + anchor := findProgramPackageV1(manifest, step.Target) + if anchor == nil { + return ResolvedProgramStepV2{}, ProgramValidationStepAnchorV2 + } + if step.Aux >= anchor.Count { + return ResolvedProgramStepV2{}, ProgramValidationStepDescriptorIndexV2 + } + descriptor := rootDescriptorAtV1(anchor, step.Aux) + if descriptor.StartupSize != 0 || descriptor.StartupAlign != 1 || + descriptor.ResultSize != 0 || descriptor.ResultAlign != 1 { + return ResolvedProgramStepV2{}, ProgramValidationStepPayloadV2 + } + return ResolvedProgramStepV2{ + Kind: ProgramStepCoroRootV2, + Flags: step.Flags, + Descriptor: descriptor, + Factory: descriptor.Factory, + }, ProgramValidationOKV2 + default: + return ResolvedProgramStepV2{}, ProgramValidationStepKindV2 + } +} + +// ValidateRunnableProgramV2 validates the shared manifest and package catalog, +// then the exact five-role heterogeneous startup program. It binds the table to +// expectedFactory by pointer identity and snapshots every resolved action. +// Validation performs no allocation and never invokes a target or factory. +func ValidateRunnableProgramV2( + manifest *ProgramManifestV1, expectedFactory unsafe.Pointer, +) (ProgramViewV2, ProgramValidationCodeV2) { + if manifest == nil { + return ProgramViewV2{}, ProgramValidationNilManifestV2 + } + if !checkedProgramObjectV1( + unsafe.Pointer(manifest), unsafe.Sizeof(ProgramManifestV1{}), unsafe.Alignof(ProgramManifestV1{}), + ) { + return ProgramViewV2{}, ProgramValidationManifestAddressV2 + } + if manifest.Version != ProgramManifestVersionV1 { + return ProgramViewV2{}, ProgramValidationManifestVersionV2 + } + if manifest.Flags != 0 { + return ProgramViewV2{}, ProgramValidationManifestFlagsV2 + } + switch checkedProgramArrayV1( + manifest.Packages, + manifest.PackageCount, + unsafe.Sizeof(unsafe.Pointer(nil)), + unsafe.Alignof(unsafe.Pointer(nil)), + ) { + case programArrayCountPointerV1: + return ProgramViewV2{}, ProgramValidationPackageCountPointerV2 + case programArrayAddressV1: + return ProgramViewV2{}, ProgramValidationPackageTableAddressV2 + } + if manifest.Bootstrap == nil { + return ProgramViewV2{}, ProgramValidationNilBootstrapV2 + } + if !checkedProgramObjectV1( + manifest.Bootstrap, unsafe.Sizeof(ProgramBootstrapV1{}), unsafe.Alignof(ProgramBootstrapV1{}), + ) { + return ProgramViewV2{}, ProgramValidationBootstrapAddressV2 + } + bootstrap := (*ProgramBootstrapV1)(manifest.Bootstrap) + if bootstrap.Version != ProgramBootstrapVersionV2 { + return ProgramViewV2{}, ProgramValidationBootstrapVersionV2 + } + if bootstrap.Flags != 0 { + return ProgramViewV2{}, ProgramValidationBootstrapFlagsV2 + } + if bootstrap.HashLo != manifest.HashLo || bootstrap.HashHi != manifest.HashHi { + return ProgramViewV2{}, ProgramValidationBootstrapHashV2 + } + if bootstrap.StepCount != programStepCountV2 { + return ProgramViewV2{}, ProgramValidationStepCountV2 + } + switch checkedProgramArrayV1( + bootstrap.Steps, + bootstrap.StepCount, + unsafe.Sizeof(ProgramStepV1{}), + unsafe.Alignof(ProgramStepV1{}), + ) { + case programArrayCountPointerV1: + return ProgramViewV2{}, ProgramValidationStepCountPointerV2 + case programArrayAddressV1: + return ProgramViewV2{}, ProgramValidationStepTableAddressV2 + } + if catalogCode := programCatalogValidationV2(validateProgramCatalogV1(manifest)); catalogCode != ProgramValidationOKV2 { + return ProgramViewV2{}, catalogCode + } + if bootstrap.Factory == nil { + return ProgramViewV2{}, ProgramValidationBootstrapFactoryV2 + } + if expectedFactory == nil || bootstrap.Factory != expectedFactory { + return ProgramViewV2{}, ProgramValidationBootstrapFactoryIdentityV2 + } + + program := ProgramViewV2{ + magic: validatedProgramMagicV2, + factory: bootstrap.Factory, + } + var code ProgramValidationCodeV2 + program.internalRuntimeInit, code = resolveValidatedProgramStepV2( + manifest, programStepAtV1(bootstrap.Steps, 0), ProgramStepFlagInternalRuntimeInitV2, + ) + if code != ProgramValidationOKV2 { + return ProgramViewV2{}, code + } + program.compilerABIInit, code = resolveValidatedProgramStepV2( + manifest, programStepAtV1(bootstrap.Steps, 1), ProgramStepFlagCompilerABIInitV2, + ) + if code != ProgramValidationOKV2 { + return ProgramViewV2{}, code + } + program.publicRuntimeInit, code = resolveValidatedProgramStepV2( + manifest, programStepAtV1(bootstrap.Steps, 2), ProgramStepFlagPublicRuntimeInitV2, + ) + if code != ProgramValidationOKV2 { + return ProgramViewV2{}, code + } + program.mainPackageInit, code = resolveValidatedProgramStepV2( + manifest, programStepAtV1(bootstrap.Steps, 3), ProgramStepFlagMainPackageInitV2, + ) + if code != ProgramValidationOKV2 { + return ProgramViewV2{}, code + } + program.main, code = resolveValidatedProgramStepV2( + manifest, programStepAtV1(bootstrap.Steps, 4), ProgramStepFlagMainV2, + ) + if code != ProgramValidationOKV2 { + return ProgramViewV2{}, code + } + return program, ProgramValidationOKV2 +} + +// ResolveProgramStepV2 returns one action from an opaque validated view. It +// never calls the plain target or coroutine factory. +func ResolveProgramStepV2(program ProgramViewV2, index uintptr) (ResolvedProgramStepV2, ProgramValidationCodeV2) { + if program.magic != validatedProgramMagicV2 { + return ResolvedProgramStepV2{}, ProgramValidationInvalidViewV2 + } + switch index { + case 0: + return program.internalRuntimeInit, ProgramValidationOKV2 + case 1: + return program.compilerABIInit, ProgramValidationOKV2 + case 2: + return program.publicRuntimeInit, ProgramValidationOKV2 + case 3: + return program.mainPackageInit, ProgramValidationOKV2 + case 4: + return program.main, ProgramValidationOKV2 + default: + return ResolvedProgramStepV2{}, ProgramValidationStepIndexV2 + } +} diff --git a/runtime/internal/coro/bootstrap_test.go b/runtime/internal/coro/bootstrap_test.go index fe10ae87d5..a58a2b214b 100644 --- a/runtime/internal/coro/bootstrap_test.go +++ b/runtime/internal/coro/bootstrap_test.go @@ -537,3 +537,250 @@ func TestValidatedProgramV1ConcurrentRead(t *testing.T) { t.Fatal("concurrent validated-view resolution failed") } } + +type programBootstrapTestFixtureV2 struct { + plainTargets [5]byte + bootstrapFactory byte + rootFactories [5]byte + descriptors [5]RootFactoryDescriptorV1 + anchorEntries [5]unsafe.Pointer + anchor RootPackageAnchorV1 + packages [1]unsafe.Pointer + steps [5]ProgramStepV1 + bootstrap ProgramBootstrapV1 + manifest ProgramManifestV1 +} + +var programStepRolesForTestV2 = [5]uint32{ + ProgramStepFlagInternalRuntimeInitV2, + ProgramStepFlagCompilerABIInitV2, + ProgramStepFlagPublicRuntimeInitV2, + ProgramStepFlagMainPackageInitV2, + ProgramStepFlagMainV2, +} + +func newProgramBootstrapTestFixtureV2(coroMask uint32) *programBootstrapTestFixtureV2 { + f := new(programBootstrapTestFixtureV2) + f.plainTargets = [5]byte{0x31, 0x32, 0x33, 0x34, 0x35} + f.bootstrapFactory = 0x41 + f.rootFactories = [5]byte{0x51, 0x52, 0x53, 0x54, 0x55} + for index := range f.descriptors { + f.descriptors[index] = RootFactoryDescriptorV1{ + Version: RootFactoryVersionV1, + HashLo: uint64(0x100 + index), + HashHi: uint64(0x200 + index), + Factory: unsafe.Pointer(&f.rootFactories[index]), + StartupAlign: 1, + ResultAlign: 1, + } + f.anchorEntries[index] = unsafe.Pointer(&f.descriptors[index]) + } + f.anchor = RootPackageAnchorV1{ + Version: RootPackageAnchorVersionV1, + HashLo: 0xa01, + HashHi: 0xa02, + Count: uintptr(len(f.anchorEntries)), + Entries: unsafe.Pointer(&f.anchorEntries[0]), + } + f.packages[0] = unsafe.Pointer(&f.anchor) + for index, role := range programStepRolesForTestV2 { + if coroMask&(uint32(1)<Ready, so failure + // here means another scheduler consumer or corrupted ownership. + return promoted, false + } + default: + return promoted, false + } + if previous == nil { + p.waitHead = next + } else { + previous.nextWait = next + } + if p.waitTail == g { + p.waitTail = previous + } + g.nextWait = nil + g.waiting = false + g.waitToken = nil + g.waitTicket = 0 + g.state = GRunnable + if !Enqueue(p, g) { + return promoted, false + } + promoted++ + g = next + } + return promoted, true +} + +// PollReady promotes every completed platform wait while the scheduler is +// idle. It never polls or calls platform code; completion producers publish by +// CompleteWait and separately wake the owning executor/event loop. +func PollReady(p *P) (int, bool) { + return pollReady(p) +} + +// HasWaiting reports whether an otherwise idle P owns parked Gs. The runtime +// adapter uses this distinction to wait for a host/platform event instead of +// misreporting an empty ready queue as program completion. +func HasWaiting(p *P) bool { + return p != nil && p.waitHead != nil && p.waitTail != nil +} + // NextRunnable removes the next ready G. It returns ok=false when a scheduler // operation is already in progress; an empty ready queue is (nil, true). func NextRunnable(p *P) (g *G, ok bool) { if p == nil || p.current != nil || p.inResume || p.action.Kind != ActionInvalid { return nil, false } + if preemptLoad(&p.schedule) == scheduleDisabled { + // Preserve the ordinary drain-loop contract after the last G atomically + // sealed the P. A disabled P is not reusable, and any residual queue is + // corruption rather than runnable work. + return nil, validReadyQueue(p) && validWaitQueue(p) && p.readyHead == nil && p.waitHead == nil + } + if _, ok := pollReady(p); !ok { + return nil, false + } return dequeue(p), true } -func dispatchPending(g *G, resumed *Frame) (destroy *Frame, ok bool) { +func dispatchPending(g *G, resumed *Frame) (destroy *Frame, yielded bool, ok bool) { pending := g.pending g.pending = pendingTransition{} if pending.from != resumed { - return nil, false + return nil, false, false } switch pending.kind { case pendingAwait: child := pending.target if child == nil || child.parent != resumed || resumed.header == nil || child.header == nil || + pending.wait != nil || pending.ticket != 0 || resumed.header.Lifecycle != uint16(FrameSuspended) || child.header.Lifecycle != uint16(FrameInitialSuspended) { - return nil, false + return nil, false, false } resumed.state = FrameSuspended g.active = child - return nil, true + return nil, false, true case pendingComplete: - if pending.target != nil || resumed.header == nil || + if pending.target != nil || pending.wait != nil || pending.ticket != 0 || resumed.header == nil || resumed.header.Lifecycle != uint16(FrameFinalSuspended) { - return nil, false + return nil, false, false } g.active = resumed.parent resumed.state = FrameDestroyPending resumed.header.Lifecycle = uint16(FrameDestroyPending) g.destroyTarget = resumed - return resumed, true + return resumed, false, true + case pendingYield: + if pending.target != nil || pending.wait != nil || pending.ticket != 0 || resumed.header == nil || + resumed.header.SuspendReason != uint16(SuspendYield) || + resumed.header.Lifecycle != uint16(FrameSuspended) { + return nil, false, false + } + resumed.state = FrameSuspended + return nil, true, true + case pendingPark: + if pending.target != nil || resumed.header == nil || + resumed.header.SuspendReason != uint16(SuspendPark) || + resumed.header.Lifecycle != uint16(FrameSuspended) || + !validClaimedWait(pending.wait, pending.ticket) || g.waitToken != nil || g.waitTicket != 0 || g.waiting || g.nextWait != nil { + return nil, false, false + } + resumed.state = FrameSuspended + g.waitToken = pending.wait + g.waitTicket = pending.ticket + return nil, false, true default: - return nil, false + return nil, false, false } } @@ -190,7 +505,12 @@ func dispatchPending(g *G, resumed *Frame) (destroy *Frame, ok bool) { func BeginRunG(p *P, g *G) (Action, bool) { if p == nil || p.current != nil || p.inResume || p.action.Kind != ActionInvalid || !ValidG(g) || g.state != GRunnable || g.active == nil || g.root == nil || - g.destroyTarget != nil || g.destroyRoot || g.queued || g.nextReady != nil { + g.destroyTarget != nil || g.destroyRoot || g.queued || g.nextReady != nil || + g.waitToken != nil || g.waitTicket != 0 || g.nextWait != nil || g.waiting || g.runP != nil { + return Action{}, false + } + schedule := preemptLoad(&p.schedule) + if schedule != scheduleIdle && schedule != scheduleRequested { return Action{}, false } frame := g.active @@ -198,8 +518,12 @@ func BeginRunG(p *P, g *G) (Action, bool) { (frame.state != FrameInitialSuspended && frame.state != FrameSuspended) { return Action{}, false } + if p.readyHead != nil && !RequestPreempt(g) { + return Action{}, false + } p.current = g g.state = GRunning + g.runP = p return setAction(p, ActionCheckResume, frame.handle) } @@ -241,10 +565,41 @@ func Resumed(p *P, g *G, action Action) (Action, bool) { p.inResume = false g.state = GDispatching resumed := g.active - destroy, ok := dispatchPending(g, resumed) + destroy, yielded, ok := dispatchPending(g, resumed) if !ok { return Action{}, false } + if yielded { + // BeginRunG guarantees that a running G has no ready-queue link. Check + // the remaining queue invariants before committing any state so a + // corrupted queue cannot leave a half-requeued G behind. + if g.queued || g.nextReady != nil || (p.readyHead == nil) != (p.readyTail == nil) || + (p.readyTail != nil && p.readyTail.nextReady != nil) { + return Action{}, false + } + g.state = GRunnable + g.runP = nil + p.current = nil + p.action = Action{} + if !Enqueue(p, g) { + return Action{}, false + } + return Action{Kind: ActionYield}, true + } + if g.waitToken != nil { + if g.queued || g.nextReady != nil || (p.waitHead == nil) != (p.waitTail == nil) || + (p.waitTail != nil && p.waitTail.nextWait != nil) { + return Action{}, false + } + g.state = GWaiting + g.runP = nil + p.current = nil + p.action = Action{} + if !enqueueWait(p, g) { + return Action{}, false + } + return Action{Kind: ActionPark}, true + } if destroy != nil { // Cache root identity before llvm.coro.destroy synchronously releases // the combined allocation. Destroyed must never dereference it. @@ -266,17 +621,34 @@ func Destroyed(p *P, g *G, action Action) (Action, bool) { return Action{}, false } isRoot := g.destroyRoot - g.destroyRoot = false if isRoot { - if g.active != nil || g.frames != nil { + if g.active != nil || g.frames != nil || !validReadyQueue(p) || !validWaitQueue(p) { + return Action{}, false + } + schedule := preemptLoad(&p.schedule) + if schedule != scheduleIdle && schedule != scheduleRequested { return Action{}, false } + // Disable only when this root is the last G owned by the P. Otherwise + // ready/waiting peers still need the gate. CAS makes terminal success and + // a late asynchronous producer request one exact total order. + if p.readyHead == nil && p.waitHead == nil && + !preemptCompareAndSwap(&p.schedule, scheduleIdle, scheduleDisabled) { + return Action{}, false + } + g.destroyRoot = false g.root = nil + // Disable requests before publishing the terminal scheduler state. A + // requester that observed idle before this store can only CAS against the + // now-disabled gate and fail; an earlier successful CAS is overwritten. + preemptStore(preemptAddress(g), preemptDisabled) g.state = GDead + g.runP = nil p.current = nil p.action = Action{} return Action{Kind: ActionComplete}, true } + g.destroyRoot = false g.state = GRunning if g.active == nil { return Action{}, false @@ -290,8 +662,10 @@ func Destroyed(p *P, g *G, action Action) (Action, bool) { // ready-queue link, destruction bookkeeping, or P operation survived. func TerminalG(p *P, g *G) bool { return p != nil && p.current == nil && p.readyHead == nil && p.readyTail == nil && - !p.inResume && p.action.Kind == ActionInvalid && p.action.Handle == nil && - ValidG(g) && g.state == GDead && g.root == nil && g.active == nil && g.frames == nil && - g.pending.kind == pendingNone && g.pending.from == nil && g.pending.target == nil && - g.destroyTarget == nil && !g.destroyRoot && g.nextReady == nil && !g.queued + p.waitHead == nil && p.waitTail == nil && + preemptLoad(&p.schedule) == scheduleDisabled && !p.inResume && p.action.Kind == ActionInvalid && p.action.Handle == nil && + ValidG(g) && preemptLoad(preemptAddress(g)) == preemptDisabled && g.state == GDead && g.root == nil && g.active == nil && g.frames == nil && + g.pending.kind == pendingNone && g.pending.from == nil && g.pending.target == nil && g.pending.wait == nil && g.pending.ticket == 0 && + g.destroyTarget == nil && !g.destroyRoot && g.nextReady == nil && !g.queued && + g.waitToken == nil && g.waitTicket == 0 && g.nextWait == nil && !g.waiting && g.runP == nil } diff --git a/runtime/internal/coro/scheduler_preempt_test.go b/runtime/internal/coro/scheduler_preempt_test.go new file mode 100644 index 0000000000..2eebb89112 --- /dev/null +++ b/runtime/internal/coro/scheduler_preempt_test.go @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "fmt" + "runtime" + "sync" + "sync/atomic" + "testing" +) + +func activatePreemptTestFrame(t *testing.T, p *P, task *yieldingTestG, action Action) Action { + t.Helper() + if action.Kind != ActionCheckResume { + t.Fatalf("initial action for G %s = %d, want check-resume", task.name, action.Kind) + } + action, ok := Checked(p, task.g, action, false) + if !ok || action.Kind != ActionResume { + t.Fatalf("activate G %s = (%+v, %t), want resume", task.name, action, ok) + } + // LLVM coroutine entry/resume publishes this state before executing a poll. + task.frame.header.SuspendReason = uint16(SuspendNone) + task.frame.header.Lifecycle = uint16(FrameActive) + return action +} + +func TestPreemptPollFailsClosedAndConsumesOnlyActiveRequest(t *testing.T) { + if RequestPreempt(nil) || PollPreempt(nil) || RequestPreempt(new(G)) || PollPreempt(new(G)) { + t.Fatal("nil or uninitialized G accepted a preemption operation") + } + newG := new(G) + if !InitG(newG) { + t.Fatal("initialize validation G") + } + if !RequestPreempt(newG) { + t.Fatal("initialized G did not publish its preemption gate") + } + if PollPreempt(newG) || preemptLoad(preemptAddress(newG)) != preemptRequested { + t.Fatal("new-G poll consumed a request outside an active frame") + } + preemptStore(preemptAddress(newG), preemptDisabled) + if RequestPreempt(newG) { + t.Fatal("preemption requested through a disabled terminal gate") + } + dirtyG := new(G) + preemptStore(preemptAddress(dirtyG), preemptRequested) + if InitG(dirtyG) { + t.Fatal("G initialized with a residual preemption request") + } + + task := newYieldingTestG(t, "poll-validation") + if !RequestPreempt(task.g) { + t.Fatal("request runnable G") + } + if !RequestPreempt(task.g) { + t.Fatal("coalesce duplicate runnable-G request") + } + if PollPreempt(task.g) || preemptLoad(preemptAddress(task.g)) != preemptRequested { + t.Fatal("runnable poll consumed a request outside an active frame") + } + + p := new(P) + action, ok := BeginRunG(p, task.g) + if !ok { + t.Fatal("begin requested G") + } + if PollPreempt(task.g) || preemptLoad(preemptAddress(task.g)) != preemptRequested { + t.Fatal("pre-resume poll consumed a request") + } + action, ok = Checked(p, task.g, action, false) + if !ok || action.Kind != ActionResume { + t.Fatal("enter active resume") + } + if PollPreempt(task.g) || preemptLoad(preemptAddress(task.g)) != preemptRequested { + t.Fatal("poll accepted an active frame before the compiler lifecycle state") + } + task.frame.header.Lifecycle = uint16(FrameActive) + if !PollPreempt(task.g) || preemptLoad(preemptAddress(task.g)) != preemptIdle { + t.Fatal("legal active poll did not consume the request") + } + if PollPreempt(task.g) { + t.Fatal("one preemption request was consumed twice") + } + + if !RequestPreempt(task.g) { + t.Fatal("request running G") + } + task.g.pending = pendingTransition{kind: pendingAwait, from: task.g.active} + if PollPreempt(task.g) || preemptLoad(preemptAddress(task.g)) != preemptRequested { + t.Fatal("transitional poll consumed a request") + } + task.g.pending = pendingTransition{} + if !PollPreempt(task.g) { + t.Fatal("request did not survive the invalid transitional poll") + } + runtime.KeepAlive(task.frame.memory) +} + +func TestBeginRunGDoesNotRequestPreemptWithoutCompetitor(t *testing.T) { + task := newYieldingTestG(t, "single") + p := new(P) + action, ok := BeginRunG(p, task.g) + if !ok { + t.Fatal("begin sole runnable G") + } + activatePreemptTestFrame(t, p, task, action) + if PollPreempt(task.g) { + t.Fatal("sole runnable G received an automatic preemption request") + } + runtime.KeepAlive(task.frame.memory) +} + +// TestSinglePRoundRobinTwoGPreemptPoll models compiler polls driving the +// existing SuspendYield handoff. BeginRunG requests a cut only while another G +// remains ready, and each consumed request moves the current G to the tail. +func TestSinglePRoundRobinTwoGPreemptPoll(t *testing.T) { + p := new(P) + a := newYieldingTestG(t, "a") + b := newYieldingTestG(t, "b") + tasks := map[*G]*yieldingTestG{a.g: a, b.g: b} + if !Enqueue(p, a.g) || !Enqueue(p, b.g) { + t.Fatal("enqueue preemptible Gs") + } + + var events []string + for { + g, ok := NextRunnable(p) + if !ok { + t.Fatal("dequeue preemptible G") + } + if g == nil { + break + } + task := tasks[g] + action, ok := BeginRunG(p, g) + if !ok { + t.Fatalf("begin G %s", task.name) + } + + runSlice: + for { + switch action.Kind { + case ActionCheckResume: + action = activatePreemptTestFrame(t, p, task, action) + case ActionResume: + task.resumes++ + if task.resumes <= 2 { + if !PollPreempt(g) { + t.Fatalf("G %s slice %d missed competitor preemption", task.name, task.resumes) + } + events = append(events, fmt.Sprintf("%s:preempt:%d", task.name, task.resumes)) + task.frame.header.SuspendReason = uint16(SuspendYield) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(g, task.handle, task.frame.header) { + t.Fatalf("prepare preemptive yield for G %s", task.name) + } + } else { + events = append(events, task.name+":complete") + task.frame.header.SuspendReason = uint16(SuspendFrameComplete) + task.frame.header.Lifecycle = uint16(FrameFinalSuspended) + if !PrepareComplete(g, task.handle, task.frame.header) { + t.Fatalf("prepare completion for G %s", task.name) + } + } + action, ok = Resumed(p, g, action) + case ActionCheckDestroy: + action, ok = Checked(p, g, action, true) + case ActionDestroy: + releaseTestFrame(t, g, task.frame) + action, ok = Destroyed(p, g, action) + case ActionYield, ActionComplete: + break runSlice + default: + t.Fatalf("unexpected action %d for G %s", action.Kind, task.name) + } + if !ok { + t.Fatalf("preemptive action protocol failed for G %s", task.name) + } + } + } + + want := []string{ + "a:preempt:1", "b:preempt:1", + "a:preempt:2", "b:preempt:2", + "a:complete", "b:complete", + } + if fmt.Sprint(events) != fmt.Sprint(want) { + t.Fatalf("preemptive round-robin events = %v, want %v", events, want) + } + if !TerminalG(p, a.g) || !TerminalG(p, b.g) { + t.Fatal("preemptive round-robin retained scheduler state") + } + runtime.KeepAlive(a.frame.memory) + runtime.KeepAlive(b.frame.memory) +} + +func TestRequestPreemptConcurrentWithTerminalDisable(t *testing.T) { + task := newYieldingTestG(t, "concurrent-terminal") + p := new(P) + action, ok := BeginRunG(p, task.g) + if !ok { + t.Fatal("begin concurrently requested G") + } + action = activatePreemptTestFrame(t, p, task, action) + + const workers = 8 + start := make(chan struct{}) + stop := make(chan struct{}) + accepted := make(chan struct{}, 1) + var requests atomic.Uint64 + var wg sync.WaitGroup + wg.Add(workers) + for worker := 0; worker < workers; worker++ { + go func() { + defer wg.Done() + <-start + for { + select { + case <-stop: + return + default: + } + if RequestPreempt(task.g) { + requests.Add(1) + select { + case accepted <- struct{}{}: + default: + } + } + } + }() + } + close(start) + <-accepted + + // Complete and destroy the root while requesters are still racing with the + // gate. They may coalesce requests before the terminal store, but none may + // re-enable the gate after destruction disables it. + task.frame.header.SuspendReason = uint16(SuspendFrameComplete) + task.frame.header.Lifecycle = uint16(FrameFinalSuspended) + if !PrepareComplete(task.g, task.handle, task.frame.header) { + t.Fatal("prepare concurrently requested G completion") + } + action, ok = Resumed(p, task.g, action) + if !ok || action.Kind != ActionCheckDestroy { + t.Fatalf("complete concurrently requested G = (%+v, %t), want check-destroy", action, ok) + } + action, ok = Checked(p, task.g, action, true) + if !ok || action.Kind != ActionDestroy { + t.Fatalf("check concurrently requested G destruction = (%+v, %t), want destroy", action, ok) + } + releaseTestFrame(t, task.g, task.frame) + action, ok = Destroyed(p, task.g, action) + if !ok || action.Kind != ActionComplete { + t.Fatalf("destroy concurrently requested G = (%+v, %t), want complete", action, ok) + } + close(stop) + wg.Wait() + + if requests.Load() == 0 { + t.Fatal("concurrent requesters never observed the enabled gate") + } + if gate := preemptLoad(preemptAddress(task.g)); gate != preemptDisabled { + t.Fatalf("terminal preemption gate = %d, want disabled", gate) + } + for attempt := 0; attempt < 1024; attempt++ { + if RequestPreempt(task.g) { + t.Fatal("terminal preemption gate was re-enabled by a late requester") + } + } + if !TerminalG(p, task.g) { + t.Fatal("concurrently requested G retained terminal scheduler state") + } + runtime.KeepAlive(task.frame.memory) +} diff --git a/runtime/internal/coro/scheduler_wait_test.go b/runtime/internal/coro/scheduler_wait_test.go new file mode 100644 index 0000000000..2586162588 --- /dev/null +++ b/runtime/internal/coro/scheduler_wait_test.go @@ -0,0 +1,661 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "runtime" + "sync" + "testing" + "unsafe" +) + +func TestWaitTicketGenerationRejectsDuplicateAndABACompletion(t *testing.T) { + if ticket, ok := ArmWait(nil); ok || ticket != 0 || CompleteWait(nil, 1) { + t.Fatal("nil wait token accepted") + } + token := new(WaitToken) + first, ok := ArmWait(token) + if !ok || first == 0 { + t.Fatal("arm first wait generation") + } + if CompleteWait(token, 0) || !CompleteWait(token, first) || CompleteWait(token, first) { + t.Fatal("first generation did not enforce one exact completion") + } + if ticket, ok := ArmWait(token); ok || ticket != 0 { + t.Fatal("ready wait token rearmed before scheduler consumption") + } + if !claimWait(token, first) || !consumeWait(token, first) { + t.Fatal("consume first ready generation") + } + second, ok := ArmWait(token) + if !ok || second == 0 || second == first { + t.Fatalf("second generation = %d, first = %d", second, first) + } + if CompleteWait(token, first) { + t.Fatal("stale first-generation completion woke second generation") + } + if !claimWait(token, second) || !CompleteWait(token, second) || !consumeWait(token, second) { + t.Fatal("complete and consume second generation") + } + + preemptStore(&token.word, waitWord(waitMaxGen, waitConsumed)) + if ticket, ok := ArmWait(token); ok || ticket != 0 { + t.Fatal("generation counter wrapped and reopened an ABA window") + } +} + +func TestWaitTicketRejectsTruncatingOutOfRangeAlias(t *testing.T) { + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm alias test token") + } + // Before the range check, shifting this value discarded its high bit and + // produced the exact same atomic word as ticket 1. + alias := WaitTicket(uint32(ticket) + waitMaxGen + 1) + if validWaitTicket(alias) || CompleteWait(token, alias) || claimWait(token, alias) || consumeWait(token, alias) { + t.Fatalf("out-of-range alias ticket %d was accepted", alias) + } + if !claimWait(token, ticket) || !CompleteWait(token, ticket) || !consumeWait(token, ticket) { + t.Fatal("rejecting alias damaged the valid generation") + } +} + +func TestWaitClaimAndCompletionRace(t *testing.T) { + const iterations = 1000 + for iteration := 0; iteration < iterations; iteration++ { + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatalf("iteration %d: arm token", iteration) + } + start := make(chan struct{}) + results := make(chan bool, 2) + go func() { + <-start + results <- claimWait(token, ticket) + }() + go func() { + <-start + results <- CompleteWait(token, ticket) + }() + close(start) + if !<-results || !<-results || !consumeWait(token, ticket) { + t.Fatalf("iteration %d: claim/completion race lost transition", iteration) + } + } +} + +func TestWaitClaimAllowsExactlyOneConcurrentWaiter(t *testing.T) { + const iterations = 1000 + for iteration := 0; iteration < iterations; iteration++ { + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatalf("iteration %d: arm token", iteration) + } + start := make(chan struct{}) + results := make(chan bool, 2) + for waiter := 0; waiter < 2; waiter++ { + go func() { + <-start + results <- claimWait(token, ticket) + }() + } + close(start) + first, second := <-results, <-results + if first == second { + t.Fatalf("iteration %d: claim results = %t, %t; want exactly one", iteration, first, second) + } + if !CompleteWait(token, ticket) || !consumeWait(token, ticket) { + t.Fatalf("iteration %d: winning waiter could not consume completion", iteration) + } + } +} + +func TestWaitAtomicFieldsAre32BitAligned(t *testing.T) { + if unsafe.Offsetof(WaitToken{}.word)%4 != 0 || unsafe.Alignof(WaitToken{}) < 4 { + t.Fatalf("WaitToken atomic word is not 32-bit aligned: offset=%d align=%d", unsafe.Offsetof(WaitToken{}.word), unsafe.Alignof(WaitToken{})) + } + if unsafe.Offsetof(G{}.preempt)%4 != 0 || unsafe.Offsetof(P{}.schedule)%4 != 0 { + t.Fatalf("scheduler atomic words are not 32-bit aligned: G.preempt=%d P.schedule=%d", unsafe.Offsetof(G{}.preempt), unsafe.Offsetof(P{}.schedule)) + } +} + +func beginWaitTestResume(t *testing.T, p *P, task *yieldingTestG) Action { + t.Helper() + action, ok := BeginRunG(p, task.g) + if !ok || action.Kind != ActionCheckResume { + t.Fatalf("begin G %s = (%+v, %t)", task.name, action, ok) + } + action, ok = Checked(p, task.g, action, false) + if !ok || action.Kind != ActionResume { + t.Fatalf("activate G %s = (%+v, %t)", task.name, action, ok) + } + task.frame.header.SuspendReason = uint16(SuspendNone) + task.frame.header.Lifecycle = uint16(FrameActive) + return action +} + +func finishWaitTestTask(t *testing.T, p *P, task *yieldingTestG, action Action) { + t.Helper() + task.frame.header.SuspendReason = uint16(SuspendFrameComplete) + task.frame.header.Lifecycle = uint16(FrameFinalSuspended) + if !PrepareComplete(task.g, task.handle, task.frame.header) { + t.Fatalf("prepare completion for G %s", task.name) + } + action, ok := Resumed(p, task.g, action) + if !ok || action.Kind != ActionCheckDestroy { + t.Fatalf("resume completion for G %s = (%+v, %t)", task.name, action, ok) + } + action, ok = Checked(p, task.g, action, true) + if !ok || action.Kind != ActionDestroy { + t.Fatalf("check destroy for G %s = (%+v, %t)", task.name, action, ok) + } + releaseTestFrame(t, task.g, task.frame) + action, ok = Destroyed(p, task.g, action) + if !ok || action.Kind != ActionComplete { + t.Fatalf("destroy G %s = (%+v, %t)", task.name, action, ok) + } +} + +func prepareWaitTestRootDestroy(t *testing.T, p *P, task *yieldingTestG, action Action) Action { + t.Helper() + task.frame.header.SuspendReason = uint16(SuspendFrameComplete) + task.frame.header.Lifecycle = uint16(FrameFinalSuspended) + if !PrepareComplete(task.g, task.handle, task.frame.header) { + t.Fatal("prepare root completion") + } + action, ok := Resumed(p, task.g, action) + if !ok || action.Kind != ActionCheckDestroy { + t.Fatalf("resume root completion = (%+v, %t)", action, ok) + } + action, ok = Checked(p, task.g, action, true) + if !ok || action.Kind != ActionDestroy { + t.Fatalf("check root destroy = (%+v, %t)", action, ok) + } + releaseTestFrame(t, task.g, task.frame) + return action +} + +func TestSinglePParkWakeHandlesEarlyCompletionWithoutLostWake(t *testing.T) { + p := new(P) + parked := newYieldingTestG(t, "parked") + competitor := newYieldingTestG(t, "competitor") + if !Enqueue(p, parked.g) || !Enqueue(p, competitor.g) { + t.Fatal("enqueue park/wake tasks") + } + + g, ok := NextRunnable(p) + if !ok || g != parked.g { + t.Fatalf("first runnable = %p, want parked G %p", g, parked.g) + } + action := beginWaitTestResume(t, p, parked) + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm early-completion token") + } + // Complete before the coroutine publishes its park transition. PreparePark + // must accept the exact ready generation, and NextRunnable must promote it + // behind the already-runnable competitor. + if !CompleteWait(token, ticket) { + t.Fatal("publish early completion") + } + parked.frame.header.SuspendReason = uint16(SuspendPark) + parked.frame.header.Lifecycle = uint16(FrameSuspended) + if !PreparePark(parked.g, parked.handle, parked.frame.header, token, ticket) { + t.Fatal("prepare already-completed park") + } + action, ok = Resumed(p, parked.g, action) + if !ok || action.Kind != ActionPark || action.Handle != nil || parked.g.state != GWaiting || !HasWaiting(p) { + t.Fatalf("park action = (%+v, %t), state=%d waiting=%t", action, ok, parked.g.state, HasWaiting(p)) + } + + g, ok = NextRunnable(p) + if !ok || g != competitor.g { + t.Fatalf("runnable after early completion = %p, want competitor %p", g, competitor.g) + } + finishWaitTestTask(t, p, competitor, beginWaitTestResume(t, p, competitor)) + g, ok = NextRunnable(p) + if !ok || g != parked.g || HasWaiting(p) { + t.Fatalf("promoted parked G = %p, ok=%t waiting=%t", g, ok, HasWaiting(p)) + } + finishWaitTestTask(t, p, parked, beginWaitTestResume(t, p, parked)) + if next, ok := NextRunnable(p); !ok || next != nil { + t.Fatalf("terminal ready queue = (%p, %t)", next, ok) + } + if !TerminalG(p, parked.g) || !TerminalG(p, competitor.g) { + t.Fatal("park/wake run retained scheduler state") + } + runtime.KeepAlive(parked.frame.memory) + runtime.KeepAlive(competitor.frame.memory) +} + +func TestSinglePParkWakeLateConcurrentCompletion(t *testing.T) { + p := new(P) + task := newYieldingTestG(t, "late") + if !Enqueue(p, task.g) { + t.Fatal("enqueue late-completion task") + } + g, ok := NextRunnable(p) + if !ok || g != task.g { + t.Fatal("dequeue late-completion task") + } + action := beginWaitTestResume(t, p, task) + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm late-completion token") + } + task.frame.header.SuspendReason = uint16(SuspendPark) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if !PreparePark(task.g, task.handle, task.frame.header, token, ticket) { + t.Fatal("prepare late park") + } + action, ok = Resumed(p, task.g, action) + if !ok || action.Kind != ActionPark || !HasWaiting(p) { + t.Fatal("commit late park") + } + if next, ok := NextRunnable(p); !ok || next != nil || !HasWaiting(p) { + t.Fatalf("armed wait appeared runnable: (%p, %t), waiting=%t", next, ok, HasWaiting(p)) + } + done := make(chan bool, 1) + go func() { + done <- CompleteWait(token, ticket) + }() + if !<-done { + t.Fatal("concurrent late completion rejected") + } + if count, ok := PollReady(p); !ok || count != 1 || HasWaiting(p) { + t.Fatalf("poll ready = (%d, %t), waiting=%t", count, ok, HasWaiting(p)) + } + g, ok = NextRunnable(p) + if !ok || g != task.g { + t.Fatal("late-completed G not promoted") + } + finishWaitTestTask(t, p, task, beginWaitTestResume(t, p, task)) + if !TerminalG(p, task.g) { + t.Fatal("late park/wake retained scheduler state") + } + runtime.KeepAlive(task.frame.memory) +} + +func TestWaitCompletionPublishesResultAcrossThreads(t *testing.T) { + p := new(P) + task := newYieldingTestG(t, "publication") + if !Enqueue(p, task.g) { + t.Fatal("enqueue publication task") + } + g, ok := NextRunnable(p) + if !ok || g != task.g { + t.Fatal("dequeue publication task") + } + action := beginWaitTestResume(t, p, task) + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm publication token") + } + task.frame.header.SuspendReason = uint16(SuspendPark) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if !PreparePark(task.g, task.handle, task.frame.header, token, ticket) { + t.Fatal("prepare publication park") + } + if action, ok = Resumed(p, task.g, action); !ok || action.Kind != ActionPark { + t.Fatal("commit publication park") + } + + type resultRecord struct { + sequence uint64 + inverse uint64 + } + const sequence = uint64(0x1020304050607080) + result := new(resultRecord) + producerDone := make(chan struct{}) + go func() { + result.sequence = sequence + result.inverse = ^sequence + if !CompleteWait(token, ticket) { + panic("completion publication rejected") + } + if !RequestSchedule(p) { + panic("schedule request rejected") + } + close(producerDone) + }() + + // Do not receive producerDone before reading the result: the only + // happens-before edge publishing these ordinary fields is the wait token's + // atomic completion/consumption transition. This is also a race-detector + // regression test for the runtime ABI contract. + deadline := 100000 + for ; deadline > 0; deadline-- { + count, pollOK := PollReady(p) + if !pollOK { + t.Fatal("poll publication wait") + } + if count == 1 { + break + } + runtime.Gosched() + } + if deadline == 0 { + t.Fatal("publication wait did not become ready") + } + if result.sequence != sequence || result.inverse != ^sequence { + t.Fatalf("published result = (%#x, %#x)", result.sequence, result.inverse) + } + <-producerDone + // The schedule request may race just after the idle poll that promoted the + // G. A second idle observation acknowledges that harmless notification. + if _, ok := PollReady(p); !ok { + t.Fatal("acknowledge publication schedule request") + } + g, ok = NextRunnable(p) + if !ok || g != task.g { + t.Fatal("published waiter not runnable") + } + finishWaitTestTask(t, p, task, beginWaitTestResume(t, p, task)) + runtime.KeepAlive(task.frame.memory) +} + +func TestCompletedWaitRequestsPreemptionWithoutReadingCurrentG(t *testing.T) { + p := new(P) + parked := newYieldingTestG(t, "wake-target") + competitor := newYieldingTestG(t, "competitor") + if !Enqueue(p, parked.g) || !Enqueue(p, competitor.g) { + t.Fatal("enqueue wake/preempt tasks") + } + + g, ok := NextRunnable(p) + if !ok || g != parked.g { + t.Fatal("dequeue wake target") + } + action := beginWaitTestResume(t, p, parked) + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm wake target") + } + parked.frame.header.SuspendReason = uint16(SuspendPark) + parked.frame.header.Lifecycle = uint16(FrameSuspended) + if !PreparePark(parked.g, parked.handle, parked.frame.header, token, ticket) { + t.Fatal("prepare wake target park") + } + if action, ok = Resumed(p, parked.g, action); !ok || action.Kind != ActionPark { + t.Fatal("commit wake target park") + } + + g, ok = NextRunnable(p) + if !ok || g != competitor.g { + t.Fatal("dequeue competitor") + } + action = beginWaitTestResume(t, p, competitor) + // No ready G remains, so the competitor starts with both its G-local gate + // and P's independent scheduling gate idle. + if PollPreempt(competitor.g) { + t.Fatal("competitor started with a residual preemption request") + } + + done := make(chan struct{}) + go func() { + if !CompleteWait(token, ticket) || !RequestSchedule(p) { + panic("complete/request-schedule wake") + } + close(done) + }() + <-done + observed := false + for poll := 0; poll < 64; poll++ { + if PollPreempt(competitor.g) { + observed = true + break + } + } + if !observed || PollPreempt(competitor.g) { + t.Fatal("running competitor did not consume exactly one P-level wake request") + } + competitor.frame.header.SuspendReason = uint16(SuspendYield) + competitor.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(competitor.g, competitor.handle, competitor.frame.header) { + t.Fatal("prepare competitor yield") + } + if action, ok = Resumed(p, competitor.g, action); !ok || action.Kind != ActionYield { + t.Fatal("commit competitor yield") + } + if count, ok := PollReady(p); !ok || count != 1 { + t.Fatalf("promote completed wake target = (%d, %t)", count, ok) + } + if !parked.g.queued || !competitor.g.queued || HasWaiting(p) { + t.Fatal("wake/preempt transition lost a runnable G or retained a waiter") + } + runtime.KeepAlive(parked.frame.memory) + runtime.KeepAlive(competitor.frame.memory) +} + +func TestRequestScheduleConcurrentCoalescing(t *testing.T) { + p := new(P) + const workers = 16 + var wg sync.WaitGroup + wg.Add(workers) + for worker := 0; worker < workers; worker++ { + go func() { + defer wg.Done() + for iteration := 0; iteration < 1000; iteration++ { + if !RequestSchedule(p) { + t.Error("coalesced schedule request rejected") + return + } + } + }() + } + wg.Wait() + if got := preemptLoad(&p.schedule); got != scheduleRequested { + t.Fatalf("coalesced schedule gate = %d, want requested", got) + } + if count, ok := PollReady(p); !ok || count != 0 || preemptLoad(&p.schedule) != scheduleIdle { + t.Fatalf("idle schedule acknowledgement = (%d, %t), gate=%d", count, ok, preemptLoad(&p.schedule)) + } + preemptStore(&p.schedule, scheduleRequested+1) + if RequestSchedule(p) { + t.Fatal("corrupt schedule gate accepted") + } + if count, ok := PollReady(p); ok || count != 0 { + t.Fatal("corrupt schedule gate did not fail closed") + } +} + +func TestTerminalDisableLinearizesWithLateScheduleRequest(t *testing.T) { + const iterations = 250 + for iteration := 0; iteration < iterations; iteration++ { + p := new(P) + task := newYieldingTestG(t, "terminal-race") + if !Enqueue(p, task.g) { + t.Fatalf("iteration %d: enqueue task", iteration) + } + g, ok := NextRunnable(p) + if !ok || g != task.g { + t.Fatalf("iteration %d: dequeue task", iteration) + } + action := prepareWaitTestRootDestroy(t, p, task, beginWaitTestResume(t, p, task)) + + start := make(chan struct{}) + requestResult := make(chan bool, 1) + go func() { + <-start + requestResult <- RequestSchedule(p) + }() + close(start) + terminalAction, terminalOK := Destroyed(p, task.g, action) + requestOK := <-requestResult + if terminalOK { + if terminalAction.Kind != ActionComplete || requestOK || preemptLoad(&p.schedule) != scheduleDisabled || + !TerminalG(p, task.g) { + t.Fatalf("iteration %d: terminal won race inconsistently: action=%+v request=%t gate=%d", iteration, terminalAction, requestOK, preemptLoad(&p.schedule)) + } + } else { + if !requestOK || preemptLoad(&p.schedule) != scheduleRequested || !task.g.destroyRoot || + task.g.state != GDispatching || p.current != task.g || p.action != action { + t.Fatalf("iteration %d: request won race but terminal partially committed: request=%t gate=%d state=%d", iteration, requestOK, preemptLoad(&p.schedule), task.g.state) + } + if !preemptCompareAndSwap(&p.schedule, scheduleRequested, scheduleIdle) { + t.Fatalf("iteration %d: acknowledge winning late request", iteration) + } + terminalAction, terminalOK = Destroyed(p, task.g, action) + if !terminalOK || terminalAction.Kind != ActionComplete || !TerminalG(p, task.g) { + t.Fatalf("iteration %d: terminal retry = (%+v, %t)", iteration, terminalAction, terminalOK) + } + } + for repeat := 0; repeat < 4; repeat++ { + if RequestSchedule(p) || preemptLoad(&p.schedule) != scheduleDisabled { + t.Fatalf("iteration %d: post-terminal request %d reopened gate", iteration, repeat) + } + } + runtime.KeepAlive(task.frame.memory) + } +} + +func TestPollReadyRejectsCorruptQueuesBeforeConsumingWake(t *testing.T) { + p := new(P) + task := newYieldingTestG(t, "queue-validation") + if !Enqueue(p, task.g) { + t.Fatal("enqueue queue-validation task") + } + g, ok := NextRunnable(p) + if !ok || g != task.g { + t.Fatal("dequeue queue-validation task") + } + action := beginWaitTestResume(t, p, task) + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm queue-validation token") + } + task.frame.header.SuspendReason = uint16(SuspendPark) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if !PreparePark(task.g, task.handle, task.frame.header, token, ticket) { + t.Fatal("prepare queue-validation park") + } + if action, ok = Resumed(p, task.g, action); !ok || action.Kind != ActionPark { + t.Fatal("commit queue-validation park") + } + if !CompleteWait(token, ticket) { + t.Fatal("complete queue-validation wait") + } + + // A detached ready tail used to let Enqueue report success while losing the + // promoted G. Validation must reject before consuming the ready ticket. + detached := &G{magic: gMagic, state: GRunnable, queued: true} + p.readyTail = detached + if count, ok := PollReady(p); ok || count != 0 { + t.Fatalf("corrupt ready queue poll = (%d, %t)", count, ok) + } + if word := preemptLoad(&token.word); waitWordState(word) != waitParkedReady || task.g.state != GWaiting { + t.Fatal("failed queue validation partially consumed the waiter") + } + p.readyTail = nil + if count, ok := PollReady(p); !ok || count != 1 { + t.Fatalf("repaired queue poll = (%d, %t)", count, ok) + } + runtime.KeepAlive(task.frame.memory) +} + +func TestWaitQueueCycleFailsClosed(t *testing.T) { + newWaitNode := func() *G { + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok || !claimWait(token, ticket) { + t.Fatal("build claimed wait node") + } + return &G{ + magic: gMagic, + state: GWaiting, + waitToken: token, + waitTicket: ticket, + waiting: true, + } + } + a, b, detachedTail := newWaitNode(), newWaitNode(), newWaitNode() + a.nextWait = b + b.nextWait = a + p := &P{waitHead: a, waitTail: detachedTail} + if count, ok := PollReady(p); ok || count != 0 { + t.Fatalf("cyclic wait queue poll = (%d, %t)", count, ok) + } +} + +func TestPrepareParkFailsClosed(t *testing.T) { + task := newYieldingTestG(t, "park-validation") + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm validation token") + } + if PreparePark(task.g, task.handle, task.frame.header, token, ticket) { + t.Fatal("park accepted outside active resume") + } + task.g.state = GRunning + frame := FrameFromStorage(task.frame.storage) + frame.state = FrameActive + task.frame.header.SuspendReason = uint16(SuspendPark) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if PreparePark(task.g, task.handle, task.frame.header, token, ticket+1) { + t.Fatal("park accepted a stale/unarmed ticket") + } + if !PreparePark(task.g, task.handle, task.frame.header, token, ticket) { + t.Fatal("valid park transition rejected") + } + if PreparePark(task.g, task.handle, task.frame.header, token, ticket) { + t.Fatal("duplicate park transition accepted") + } + runtime.KeepAlive(task.frame.memory) +} + +func TestPrepareParkSameTicketAllowsExactlyOneG(t *testing.T) { + first := newYieldingTestG(t, "first-waiter") + second := newYieldingTestG(t, "second-waiter") + for _, task := range []*yieldingTestG{first, second} { + task.g.state = GRunning + FrameFromStorage(task.frame.storage).state = FrameActive + task.frame.header.SuspendReason = uint16(SuspendPark) + task.frame.header.Lifecycle = uint16(FrameSuspended) + } + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm shared wait ticket") + } + if !PreparePark(first.g, first.handle, first.frame.header, token, ticket) { + t.Fatal("first G did not claim shared ticket") + } + if PreparePark(second.g, second.handle, second.frame.header, token, ticket) { + t.Fatal("second G claimed the same token/ticket") + } + if second.g.pending.kind != pendingNone || second.g.pending.wait != nil || second.g.pending.ticket != 0 { + t.Fatal("rejected second G retained a partial park transition") + } + if first.g.pending.kind != pendingPark || first.g.pending.wait != token || first.g.pending.ticket != ticket || + !validClaimedWait(token, ticket) { + t.Fatal("winning G lost exact claimed wait ownership") + } + if !CompleteWait(token, ticket) || !consumeWait(token, ticket) { + t.Fatal("winning G's claimed ticket could not complete") + } + runtime.KeepAlive(first.frame.memory) + runtime.KeepAlive(second.frame.memory) +} diff --git a/runtime/internal/coro/scheduler_yield_test.go b/runtime/internal/coro/scheduler_yield_test.go new file mode 100644 index 0000000000..f50b54ad38 --- /dev/null +++ b/runtime/internal/coro/scheduler_yield_test.go @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "fmt" + "runtime" + "testing" + "unsafe" +) + +type yieldingTestG struct { + name string + g *G + frame *testFrame + handle unsafe.Pointer + resumes int +} + +func newYieldingTestG(t *testing.T, name string) *yieldingTestG { + t.Helper() + g := new(G) + if !InitG(g) { + t.Fatalf("initialize G %s", name) + } + handle := unsafe.Pointer(new(byte)) + frame := newTestFrame(t, g, handle, nil) + if !AdoptRoot(g, handle) { + t.Fatalf("adopt root for G %s", name) + } + return &yieldingTestG{name: name, g: g, frame: frame, handle: handle} +} + +// TestSinglePRoundRobinTwoGYield models the exact adapter action protocol with +// two independent stackless frame chains. Each task yields twice. Requeueing +// at the tail must let the other runnable task execute before the yielding +// task's retained LLVM handle is resumed again. +func TestSinglePRoundRobinTwoGYield(t *testing.T) { + p := new(P) + a := newYieldingTestG(t, "a") + b := newYieldingTestG(t, "b") + tasks := map[*G]*yieldingTestG{a.g: a, b.g: b} + if !Enqueue(p, a.g) || !Enqueue(p, b.g) { + t.Fatal("enqueue initial runnable Gs") + } + + var events []string + for { + g, ok := NextRunnable(p) + if !ok { + t.Fatal("dequeue rejected without an active scheduler operation") + } + if g == nil { + break + } + task := tasks[g] + if task == nil { + t.Fatalf("dequeued unknown G %p", g) + } + action, ok := BeginRunG(p, g) + if !ok { + t.Fatalf("begin run for G %s", task.name) + } + + runSlice: + for { + switch action.Kind { + case ActionCheckResume: + action, ok = Checked(p, g, action, false) + case ActionResume: + task.resumes++ + if task.resumes <= 2 { + events = append(events, fmt.Sprintf("%s:yield:%d", task.name, task.resumes)) + task.frame.header.SuspendReason = uint16(SuspendYield) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(g, task.handle, task.frame.header) { + t.Fatalf("prepare yield %d for G %s", task.resumes, task.name) + } + } else { + events = append(events, task.name+":complete") + task.frame.header.SuspendReason = uint16(SuspendFrameComplete) + task.frame.header.Lifecycle = uint16(FrameFinalSuspended) + if !PrepareComplete(g, task.handle, task.frame.header) { + t.Fatalf("prepare completion for G %s", task.name) + } + } + action, ok = Resumed(p, g, action) + case ActionCheckDestroy: + action, ok = Checked(p, g, action, true) + case ActionDestroy: + releaseTestFrame(t, g, task.frame) + action, ok = Destroyed(p, g, action) + case ActionYield: + if action.Handle != nil || p.current != nil || g.state != GRunnable || !g.queued || + g.active == nil || g.active.handle != task.handle || g.active.state != FrameSuspended { + t.Fatalf("yielded G %s retained invalid state: action=%+v current=%p state=%d queued=%t active=%p", task.name, action, p.current, g.state, g.queued, g.active) + } + break runSlice + case ActionComplete: + if g.state != GDead || p.current != nil || g.queued || g.active != nil || g.frames != nil { + t.Fatalf("completed G %s retained scheduler state", task.name) + } + break runSlice + default: + t.Fatalf("unexpected action %d for G %s", action.Kind, task.name) + } + if !ok { + t.Fatalf("action protocol failed for G %s at action %d", task.name, action.Kind) + } + } + } + + want := []string{ + "a:yield:1", "b:yield:1", + "a:yield:2", "b:yield:2", + "a:complete", "b:complete", + } + if fmt.Sprint(events) != fmt.Sprint(want) { + t.Fatalf("round-robin events = %v, want %v", events, want) + } + if !TerminalG(p, a.g) || !TerminalG(p, b.g) { + t.Fatal("round-robin run did not consume both Gs") + } + runtime.KeepAlive(a.frame.memory) + runtime.KeepAlive(b.frame.memory) +} + +func TestPrepareYieldFailsClosed(t *testing.T) { + task := newYieldingTestG(t, "yield-validation") + frame := FrameFromStorage(task.frame.storage) + if PrepareYield(task.g, task.handle, task.frame.header) { + t.Fatal("yield accepted outside an active resume") + } + task.g.state = GRunning + frame.state = FrameActive + task.frame.header.SuspendReason = uint16(SuspendYield) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(task.g, task.handle, task.frame.header) { + t.Fatal("valid active yield rejected") + } + if PrepareYield(task.g, task.handle, task.frame.header) { + t.Fatal("duplicate yield transition accepted") + } + runtime.KeepAlive(task.frame.memory) +} diff --git a/runtime/internal/coro/wait.go b/runtime/internal/coro/wait.go new file mode 100644 index 0000000000..1c2b77a630 --- /dev/null +++ b/runtime/internal/coro/wait.go @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +// WaitToken is a target-neutral, allocation-free completion cell. A platform +// worker, host callback, RTOS ISR handoff, or bare-metal event source may only +// call CompleteWait; it never touches G/P state or an LLVM coroutine handle. +// +// The generation and state share one atomic word so a late completion cannot +// wake a later reuse of the same cell (the classic cancellation/ABA race). A +// token is intentionally exhausted after 2^29-1 generations rather than +// wrapping and accepting a stale ticket. The additional states atomically +// claim one exact waiter without storing a target-dependent pointer in the +// completion cell. A WaitToken must not be copied after its first ArmWait. +type WaitToken struct { + word uint32 +} + +// WaitTicket identifies one exact arm of a WaitToken. The zero value is never +// valid. It is safe to copy into a stable foreign-operation argument record. +type WaitTicket uint32 + +const ( + waitStateBits = 3 + waitStateMask = 1<> waitStateBits +) + +type waitState uint32 + +const ( + waitUnused waitState = iota + waitArmed + waitReady + waitParked + waitParkedReady + waitConsumed +) + +func waitWord(generation uint32, state waitState) uint32 { + return generation<> waitStateBits +} + +func waitWordState(word uint32) waitState { + return waitState(word & waitStateMask) +} + +func validWaitTicket(ticket WaitTicket) bool { + return ticket != 0 && uint32(ticket) <= waitMaxGen +} + +// ArmWait starts one new completion generation. Only the scheduler/operation +// submitter may arm a token, and only while it is unused or fully consumed. +func ArmWait(token *WaitToken) (WaitTicket, bool) { + if token == nil { + return 0, false + } + for { + old := preemptLoad(&token.word) + state := waitWordState(old) + if state != waitUnused && state != waitConsumed { + return 0, false + } + generation := waitGeneration(old) + 1 + if generation == 0 || generation > waitMaxGen { + return 0, false + } + armed := waitWord(generation, waitArmed) + if preemptCompareAndSwap(&token.word, old, armed) { + return WaitTicket(generation), true + } + } +} + +// CompleteWait publishes completion of one exact generation. Writes to the +// stable result record must happen before this call. The atomic CAS publishes +// them to the scheduler that consumes the ready ticket. Duplicate, stale, and +// not-yet-armed completions fail closed. This operation deliberately touches +// neither P/G queues nor an LLVM handle. After a successful completion, the +// platform adapter separately calls RequestSchedule on the stable owning P and +// wakes its executor; that producer must quiesce before the P can terminate. +func CompleteWait(token *WaitToken, ticket WaitTicket) bool { + if token == nil || !validWaitTicket(ticket) { + return false + } + generation := uint32(ticket) + for { + old := preemptLoad(&token.word) + if waitGeneration(old) != generation { + return false + } + var ready waitState + switch waitWordState(old) { + case waitArmed: + ready = waitReady + case waitParked: + ready = waitParkedReady + default: + return false + } + if preemptCompareAndSwap(&token.word, old, waitWord(generation, ready)) { + return true + } + } +} + +// claimWait binds one exact generation to one scheduler waiter. Completion is +// permitted to race on either side of this transition; the two claimed states +// preserve whether the result was already published. No second G can claim +// the same token/ticket pair. +func claimWait(token *WaitToken, ticket WaitTicket) bool { + if token == nil || !validWaitTicket(ticket) { + return false + } + generation := uint32(ticket) + for { + old := preemptLoad(&token.word) + if waitGeneration(old) != generation { + return false + } + var claimed waitState + switch waitWordState(old) { + case waitArmed: + claimed = waitParked + case waitReady: + claimed = waitParkedReady + default: + return false + } + if preemptCompareAndSwap(&token.word, old, waitWord(generation, claimed)) { + return true + } + } +} + +func validClaimedWait(token *WaitToken, ticket WaitTicket) bool { + if token == nil || !validWaitTicket(ticket) { + return false + } + word := preemptLoad(&token.word) + if waitGeneration(word) != uint32(ticket) { + return false + } + state := waitWordState(word) + return state == waitParked || state == waitParkedReady +} + +func consumeWait(token *WaitToken, ticket WaitTicket) bool { + if token == nil || !validWaitTicket(ticket) { + return false + } + ready := waitWord(uint32(ticket), waitParkedReady) + return preemptCompareAndSwap(&token.word, ready, waitWord(uint32(ticket), waitConsumed)) +} diff --git a/runtime/internal/coroalloc/allocator.go b/runtime/internal/coroalloc/allocator.go new file mode 100644 index 0000000000..5f2a903f2f --- /dev/null +++ b/runtime/internal/coroalloc/allocator.go @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package coroalloc owns the phase-0 coroutine frame allocator boundary. It +// intentionally has no package initialization, callback, interface, or +// function-value dispatch: the selected target backend is linked statically. +package coroalloc + +import "unsafe" + +type bootstrapState uint8 + +const ( + bootstrapUninitialized bootstrapState = iota + bootstrapInitializing + bootstrapReady + bootstrapFailed +) + +type bootstrapDecision uint8 + +const ( + bootstrapReject bootstrapDecision = iota + bootstrapStart + bootstrapAlreadyReady +) + +var state bootstrapState + +func beginBootstrap(current bootstrapState) (bootstrapState, bootstrapDecision) { + switch current { + case bootstrapUninitialized: + return bootstrapInitializing, bootstrapStart + case bootstrapReady: + return bootstrapReady, bootstrapAlreadyReady + case bootstrapInitializing, bootstrapFailed: + return bootstrapFailed, bootstrapReject + default: + return bootstrapFailed, bootstrapReject + } +} + +func finishBootstrap(current bootstrapState, success bool) (bootstrapState, bool) { + if current != bootstrapInitializing || !success { + return bootstrapFailed, false + } + return bootstrapReady, true +} + +// Bootstrap initializes the statically selected frame allocator backend. The +// process-entry path is single-threaded until this function returns; after a +// successful transition state is immutable and may be read by scheduler +// workers. A recursive or failed initialization permanently fails closed. +func Bootstrap() bool { + next, decision := beginBootstrap(state) + state = next + switch decision { + case bootstrapAlreadyReady: + return true + case bootstrapStart: + next, success := finishBootstrap(state, backendBootstrap()) + state = next + return success + default: + return false + } +} + +// Ready reports whether Bootstrap completed successfully. +func Ready() bool { + return state == bootstrapReady +} + +// AllocFrame allocates one explicitly owned, GC-visible coroutine frame +// range. A caller cannot accidentally rely on a backend's implicit lazy init. +func AllocFrame(size uintptr) unsafe.Pointer { + if !Ready() || size == 0 { + return nil + } + return backendAllocFrame(size) +} + +// FreeFrame releases a range previously returned by AllocFrame. Backends that +// reclaim through a tracing collector may deliberately implement physical +// free as a no-op, but still validate allocator readiness through this API. +func FreeFrame(ptr unsafe.Pointer) bool { + if !Ready() || ptr == nil { + return false + } + backendFreeFrame(ptr) + return true +} diff --git a/runtime/internal/coroalloc/allocator_test.go b/runtime/internal/coroalloc/allocator_test.go new file mode 100644 index 0000000000..279603dc35 --- /dev/null +++ b/runtime/internal/coroalloc/allocator_test.go @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import "testing" + +func TestBootstrapStateSuccessAndIdempotence(t *testing.T) { + current := bootstrapUninitialized + current, got := beginBootstrap(current) + if got != bootstrapStart || current != bootstrapInitializing { + t.Fatalf("begin = %d, state=%d; want start/initializing", got, current) + } + current, success := finishBootstrap(current, true) + if !success || current != bootstrapReady { + t.Fatalf("finish success state=%d, want ready", current) + } + current, got = beginBootstrap(current) + if got != bootstrapAlreadyReady || current != bootstrapReady { + t.Fatalf("repeat begin = %d, state=%d; want already-ready/ready", got, current) + } +} + +func TestBootstrapStateFailsClosed(t *testing.T) { + tests := []struct { + name string + initial bootstrapState + finish bool + }{ + {name: "backend failure", initial: bootstrapUninitialized, finish: true}, + {name: "recursive bootstrap", initial: bootstrapInitializing}, + {name: "prior failure", initial: bootstrapFailed}, + {name: "invalid state", initial: bootstrapState(0xff)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + current := test.initial + current, decision := beginBootstrap(current) + if test.finish { + if decision != bootstrapStart { + t.Fatalf("begin = %d, want start", decision) + } + var success bool + current, success = finishBootstrap(current, false) + if success { + t.Fatal("failed backend committed ready") + } + } else if decision != bootstrapReject { + t.Fatalf("begin = %d, want reject", decision) + } + if current != bootstrapFailed { + t.Fatalf("state=%d, want permanently failed", current) + } + current, decision = beginBootstrap(current) + current, success := finishBootstrap(current, true) + if decision != bootstrapReject || success || current != bootstrapFailed { + t.Fatal("failed state was recoverable") + } + }) + } +} + +func TestSelectedBackendKindIsKnown(t *testing.T) { + switch backendKind { + case "bdwgc", "malloc", "tinygogc": + default: + t.Fatalf("unknown statically selected backend %q", backendKind) + } +} diff --git a/runtime/internal/coroalloc/backend_baremetal.go b/runtime/internal/coroalloc/backend_baremetal.go new file mode 100644 index 0000000000..806c42aac5 --- /dev/null +++ b/runtime/internal/coroalloc/backend_baremetal.go @@ -0,0 +1,41 @@ +//go:build !nogc && baremetal + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/internal/runtime/tinygogc" +) + +const backendKind = "tinygogc" + +func backendBootstrap() bool { + tinygogc.Init() + return true +} + +func backendAllocFrame(size uintptr) unsafe.Pointer { + return tinygogc.Alloc(size) +} + +func backendFreeFrame(ptr unsafe.Pointer) { + // tinygogc currently reclaims unreachable frames during tracing GC. + _ = ptr +} diff --git a/runtime/internal/coroalloc/backend_baremetal_test.go b/runtime/internal/coroalloc/backend_baremetal_test.go new file mode 100644 index 0000000000..ca7f260316 --- /dev/null +++ b/runtime/internal/coroalloc/backend_baremetal_test.go @@ -0,0 +1,27 @@ +//go:build !nogc && baremetal + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import "testing" + +func TestBaremetalBackendBuildSelection(t *testing.T) { + if backendKind != "tinygogc" { + t.Fatalf("baremetal frame allocator backend = %q, want tinygogc", backendKind) + } +} diff --git a/runtime/internal/coroalloc/backend_gc.go b/runtime/internal/coroalloc/backend_gc.go new file mode 100644 index 0000000000..e2139cc338 --- /dev/null +++ b/runtime/internal/coroalloc/backend_gc.go @@ -0,0 +1,40 @@ +//go:build !nogc && !baremetal && !wasm && !tinygo.wasm + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/internal/clite/bdwgc" +) + +const backendKind = "bdwgc" + +func backendBootstrap() bool { + bdwgc.Init() + return true +} + +func backendAllocFrame(size uintptr) unsafe.Pointer { + return bdwgc.MallocUncollectable(size) +} + +func backendFreeFrame(ptr unsafe.Pointer) { + bdwgc.Free(ptr) +} diff --git a/runtime/internal/coroalloc/backend_gc_test.go b/runtime/internal/coroalloc/backend_gc_test.go new file mode 100644 index 0000000000..72383883c2 --- /dev/null +++ b/runtime/internal/coroalloc/backend_gc_test.go @@ -0,0 +1,27 @@ +//go:build !nogc && !baremetal && !wasm && !tinygo.wasm + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import "testing" + +func TestGCBackendBuildSelection(t *testing.T) { + if backendKind != "bdwgc" { + t.Fatalf("GC frame allocator backend = %q, want bdwgc", backendKind) + } +} diff --git a/runtime/internal/coroalloc/backend_nogc.go b/runtime/internal/coroalloc/backend_nogc.go new file mode 100644 index 0000000000..3550521973 --- /dev/null +++ b/runtime/internal/coroalloc/backend_nogc.go @@ -0,0 +1,39 @@ +//go:build nogc + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import ( + "unsafe" + + c "github.com/goplus/llgo/runtime/internal/clite" +) + +const backendKind = "malloc" + +func backendBootstrap() bool { + return true +} + +func backendAllocFrame(size uintptr) unsafe.Pointer { + return c.Malloc(size) +} + +func backendFreeFrame(ptr unsafe.Pointer) { + c.Free(ptr) +} diff --git a/runtime/internal/coroalloc/backend_nogc_test.go b/runtime/internal/coroalloc/backend_nogc_test.go new file mode 100644 index 0000000000..657e311038 --- /dev/null +++ b/runtime/internal/coroalloc/backend_nogc_test.go @@ -0,0 +1,27 @@ +//go:build nogc + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import "testing" + +func TestNoGCBackendBuildSelection(t *testing.T) { + if backendKind != "malloc" { + t.Fatalf("nogc frame allocator backend = %q, want malloc", backendKind) + } +} diff --git a/runtime/internal/coroalloc/backend_target_selection_test.go b/runtime/internal/coroalloc/backend_target_selection_test.go new file mode 100644 index 0000000000..4988830452 --- /dev/null +++ b/runtime/internal/coroalloc/backend_target_selection_test.go @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import ( + "encoding/json" + "os" + "os/exec" + "slices" + "testing" +) + +func TestWebAssemblyTargetsSelectMallocBackend(t *testing.T) { + targets := []struct { + name string + goos string + goarch string + tags string + }{ + {name: "js-wasm", goos: "js", goarch: "wasm", tags: "llgo,tinygo.wasm"}, + {name: "wasip1", goos: "wasip1", goarch: "wasm", tags: "llgo,tinygo.wasm"}, + {name: "wasip2", goos: "linux", goarch: "arm", tags: "llgo,tinygo.wasm,wasip2"}, + {name: "wasm-unknown", goos: "linux", goarch: "arm", tags: "llgo,tinygo.wasm,wasm_unknown"}, + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + t.Parallel() + + cmd := exec.Command("go", "list", "-json", "-tags="+target.tags, ".") + cmd.Env = append(os.Environ(), + "GOOS="+target.goos, + "GOARCH="+target.goarch, + "CGO_ENABLED=0", + ) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("go list target package: %v\n%s", err, output) + } + + var pkg struct { + GoFiles []string + TestGoFiles []string + Imports []string + } + if err := json.Unmarshal(output, &pkg); err != nil { + t.Fatalf("decode go list output: %v\n%s", err, output) + } + if !slices.Contains(pkg.GoFiles, "backend_webassembly.go") { + t.Fatalf("GoFiles = %v, want backend_webassembly.go", pkg.GoFiles) + } + if slices.Contains(pkg.GoFiles, "backend_gc.go") { + t.Fatalf("GoFiles = %v, unexpectedly selected BDWGC backend", pkg.GoFiles) + } + if !slices.Contains(pkg.TestGoFiles, "backend_webassembly_test.go") { + t.Fatalf("TestGoFiles = %v, want backend_webassembly_test.go", pkg.TestGoFiles) + } + if slices.Contains(pkg.TestGoFiles, "backend_gc_test.go") { + t.Fatalf("TestGoFiles = %v, unexpectedly selected BDWGC backend test", pkg.TestGoFiles) + } + if slices.Contains(pkg.Imports, "github.com/goplus/llgo/runtime/internal/clite/bdwgc") { + t.Fatalf("Imports = %v, unexpectedly retained BDWGC", pkg.Imports) + } + }) + } +} diff --git a/runtime/internal/coroalloc/backend_webassembly.go b/runtime/internal/coroalloc/backend_webassembly.go new file mode 100644 index 0000000000..51a14d3a3a --- /dev/null +++ b/runtime/internal/coroalloc/backend_webassembly.go @@ -0,0 +1,42 @@ +//go:build !nogc && !baremetal && (wasm || tinygo.wasm) + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import ( + "unsafe" + + c "github.com/goplus/llgo/runtime/internal/clite" +) + +// tinygo.wasm is the common target tag carried by wasm, wasip1, wasip2, and +// wasm-unknown configurations. Keep the built-in wasm alternative so direct +// GOARCH=wasm package builds select the same backend. +const backendKind = "malloc" + +func backendBootstrap() bool { + return true +} + +func backendAllocFrame(size uintptr) unsafe.Pointer { + return c.Malloc(size) +} + +func backendFreeFrame(ptr unsafe.Pointer) { + c.Free(ptr) +} diff --git a/runtime/internal/coroalloc/backend_webassembly_test.go b/runtime/internal/coroalloc/backend_webassembly_test.go new file mode 100644 index 0000000000..13a7e310e8 --- /dev/null +++ b/runtime/internal/coroalloc/backend_webassembly_test.go @@ -0,0 +1,49 @@ +//go:build !nogc && !baremetal && (wasm || tinygo.wasm) + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coroalloc + +import ( + "testing" + "unsafe" +) + +func TestWasmBackendAllocatesAndFreesWithLibc(t *testing.T) { + if backendKind != "malloc" { + t.Fatalf("wasm frame allocator backend = %q, want malloc", backendKind) + } + if !Bootstrap() || !Ready() { + t.Fatal("bootstrap wasm malloc frame allocator") + } + const size = uintptr(64) + ptr := AllocFrame(size) + if ptr == nil { + t.Fatal("wasm malloc frame allocation returned nil") + } + for offset := uintptr(0); offset < size; offset++ { + *(*byte)(unsafe.Add(ptr, offset)) = byte(offset + 1) + } + for offset := uintptr(0); offset < size; offset++ { + if got, want := *(*byte)(unsafe.Add(ptr, offset)), byte(offset+1); got != want { + t.Fatalf("wasm malloc frame byte %d = %d, want %d", offset, got, want) + } + } + if !FreeFrame(ptr) { + t.Fatal("wasm free frame rejected allocated range") + } +} diff --git a/runtime/internal/coroalloc/testdata/wasm_backend/main.go b/runtime/internal/coroalloc/testdata/wasm_backend/main.go new file mode 100644 index 0000000000..938554c338 --- /dev/null +++ b/runtime/internal/coroalloc/testdata/wasm_backend/main.go @@ -0,0 +1,48 @@ +//go:build wasm || tinygo.wasm + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/internal/coroalloc" +) + +func main() { + if !coroalloc.Bootstrap() || !coroalloc.Ready() { + panic("bootstrap wasm coroutine allocator") + } + + const size = uintptr(64) + ptr := coroalloc.AllocFrame(size) + if ptr == nil { + panic("allocate wasm coroutine frame") + } + for offset := uintptr(0); offset < size; offset++ { + *(*byte)(unsafe.Add(ptr, offset)) = byte(offset + 1) + } + for offset := uintptr(0); offset < size; offset++ { + if *(*byte)(unsafe.Add(ptr, offset)) != byte(offset+1) { + panic("corrupt wasm coroutine frame") + } + } + if !coroalloc.FreeFrame(ptr) { + panic("free wasm coroutine frame") + } +} diff --git a/runtime/internal/lib/runtime/mfinal.go b/runtime/internal/lib/runtime/mfinal.go index 7ed607e65f..5d54fe6b3c 100644 --- a/runtime/internal/lib/runtime/mfinal.go +++ b/runtime/internal/lib/runtime/mfinal.go @@ -1,3 +1,5 @@ +//go:build !nogc && !baremetal + // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/runtime/internal/lib/runtime/mfinal_nogc.go b/runtime/internal/lib/runtime/mfinal_nogc.go new file mode 100644 index 0000000000..435dd127f2 --- /dev/null +++ b/runtime/internal/lib/runtime/mfinal_nogc.go @@ -0,0 +1,24 @@ +//go:build nogc || baremetal + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +// SetFinalizer is deliberately inert when no finalizer-capable collector is +// present. Objects are not reclaimed by the leaking/nogc profile, and tinygogc +// does not implement finalizer queues, so the callback can never run. +func SetFinalizer(obj any, finalizer any) {} diff --git a/runtime/internal/lib/runtime/runtime_gc.go b/runtime/internal/lib/runtime/runtime_gc.go index d8656f93a4..3287e2173f 100644 --- a/runtime/internal/lib/runtime/runtime_gc.go +++ b/runtime/internal/lib/runtime/runtime_gc.go @@ -5,11 +5,17 @@ package runtime import ( "runtime" + c "github.com/goplus/llgo/runtime/internal/clite" "github.com/goplus/llgo/runtime/internal/clite/bdwgc" + "github.com/goplus/llgo/runtime/internal/coroalloc" ) func init() { - bdwgc.Init() + // Legacy entry paths initialize the same allocator here. Coroutine entry + // performs this phase explicitly before any Go/runtime initialization. + if !coroalloc.Bootstrap() { + c.Exit(2) + } } func ReadMemStats(m *runtime.MemStats) { diff --git a/runtime/internal/lib/runtime/runtime_nogc.go b/runtime/internal/lib/runtime/runtime_nogc.go index 3f11426023..9aadb73786 100644 --- a/runtime/internal/lib/runtime/runtime_nogc.go +++ b/runtime/internal/lib/runtime/runtime_nogc.go @@ -2,6 +2,17 @@ package runtime -func GC() { +import "runtime" + +// ReadMemStats reports an empty managed heap for the explicit leaking/nogc +// profile. Allocations are owned by libc malloc and are not traced or reclaimed, +// so reporting them as a Go GC heap would falsely advertise collector state. +func ReadMemStats(m *runtime.MemStats) { + if m != nil { + *m = runtime.MemStats{} + } +} +func GC() { + // The leaking/nogc profile has no tracing collector. } diff --git a/runtime/internal/runtime/coro_allocator.go b/runtime/internal/runtime/coro_allocator.go new file mode 100644 index 0000000000..371884ea58 --- /dev/null +++ b/runtime/internal/runtime/coro_allocator.go @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import "github.com/goplus/llgo/runtime/internal/coroalloc" + +//export __llgo_coro_frame_allocator_bootstrap_v1 +func __llgo_coro_frame_allocator_bootstrap_v1() { + if !coroalloc.Bootstrap() { + coroRuntimeAbort("coroutine frame allocator bootstrap failed") + } +} diff --git a/runtime/internal/runtime/coro_frame.go b/runtime/internal/runtime/coro_frame.go index 2c8579e92d..49e087a8d2 100644 --- a/runtime/internal/runtime/coro_frame.go +++ b/runtime/internal/runtime/coro_frame.go @@ -21,10 +21,16 @@ import ( c "github.com/goplus/llgo/runtime/internal/clite" "github.com/goplus/llgo/runtime/internal/coro" + "github.com/goplus/llgo/runtime/internal/coroalloc" ) func coroRuntimeAbort(message string) { - fatal(message) + // Scheduler/runtime ABI failures happen on the executor stack and cannot + // enter the general formatting or panic machinery: either path may require a + // managed coroutine continuation. Keep this terminal path bounded and + // allocation-free; detailed diagnostics belong in the caller-side verifier. + _ = message + c.Fputs(c.Str("fatal error: invalid coroutine runtime state\n"), c.Stderr) c.Exit(2) } @@ -34,13 +40,15 @@ func __llgo_coro_frame_alloc_v1(g unsafe.Pointer, size, align uintptr, descripto if !ok { coroRuntimeAbort("invalid coroutine frame allocation size") } - raw := AllocRoot(total) + raw := coroalloc.AllocFrame(total) if raw == nil { coroRuntimeAbort("coroutine frame allocation failed") } storage, ok := coro.RegisterFrame((*coro.G)(g), raw, total, size, align, descriptor) if !ok { - FreeRoot(raw) + if !coroalloc.FreeFrame(raw) { + coroRuntimeAbort("coroutine frame allocation rollback failed") + } coroRuntimeAbort("invalid coroutine frame allocation") } return storage @@ -60,6 +68,31 @@ func __llgo_coro_await_prepare_v1(g, parent, child unsafe.Pointer) { } } +//export __llgo_coro_preempt_poll_v1 +func __llgo_coro_preempt_poll_v1(g unsafe.Pointer) bool { + return coro.PollPreempt((*coro.G)(g)) +} + +//export __llgo_coro_yield_prepare_v1 +func __llgo_coro_yield_prepare_v1(g, handle, header unsafe.Pointer) { + if !coro.PrepareYield((*coro.G)(g), handle, (*coro.HeaderV1)(header)) { + coroRuntimeAbort("invalid coroutine yield handoff") + } +} + +//export __llgo_coro_park_prepare_v1 +func __llgo_coro_park_prepare_v1(g, handle, header, token unsafe.Pointer, ticket uint32) { + if !coro.PreparePark( + (*coro.G)(g), + handle, + (*coro.HeaderV1)(header), + (*coro.WaitToken)(token), + coro.WaitTicket(ticket), + ) { + coroRuntimeAbort("invalid coroutine park handoff") + } +} + //export __llgo_coro_complete_prepare_v1 func __llgo_coro_complete_prepare_v1(g, handle, header unsafe.Pointer) { if !coro.PrepareComplete((*coro.G)(g), handle, (*coro.HeaderV1)(header)) { @@ -74,5 +107,7 @@ func __llgo_coro_frame_free_v1(g, storage unsafe.Pointer, size, align uintptr, d coroRuntimeAbort("invalid coroutine frame destruction") } coro.Zero(raw, total) - FreeRoot(raw) + if !coroalloc.FreeFrame(raw) { + coroRuntimeAbort("coroutine frame release failed") + } } diff --git a/runtime/internal/runtime/coro_park_intrinsic.go b/runtime/internal/runtime/coro_park_intrinsic.go new file mode 100644 index 0000000000..ece62e06e9 --- /dev/null +++ b/runtime/internal/runtime/coro_park_intrinsic.go @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + _ "unsafe" + + "github.com/goplus/llgo/runtime/internal/coro" +) + +// coroPark is the compiler-owned source spelling for an exact current-frame +// park. It intentionally has no ordinary Go body: cl lowers a direct call in +// the caller's physical coroutine to publish/prepare/suspend/activate. Future +// channel, timer, syscall, and platform adapters may call this declaration +// while preserving their synchronous Go source signatures. +// +//go:linkname coroPark llgo.coroPark +func coroPark(token *coro.WaitToken, ticket coro.WaitTicket) diff --git a/runtime/internal/runtime/coro_program.go b/runtime/internal/runtime/coro_program.go index 51db859ed0..b17d2e7cbc 100644 --- a/runtime/internal/runtime/coro_program.go +++ b/runtime/internal/runtime/coro_program.go @@ -20,6 +20,7 @@ import ( "unsafe" "github.com/goplus/llgo/runtime/internal/coro" + "github.com/goplus/llgo/runtime/internal/coroalloc" ) type coroProgramLifecycleV1 uint8 @@ -32,70 +33,74 @@ const ( coroProgramFailedV1 ) -// coroProgramV1 is the allocation-free, single-start scheduler state used by +// The coroutine program globals form the allocation-free, single-start state used by // the process entry coroutine. Keeping G and P in static storage avoids a -// pthread, TLS, or event-library dependency for scheduler state. The LLVM -// coroutine frame is still allocated through the target's AllocRoot backend; -// native currently uses BDWGC or C malloc, while allocator-independent -// wasm/embedded/bare-metal profiles require their planned linear-memory or -// static/slab backend. +// pthread, TLS, or event-library dependency for scheduler state. LLVM frames +// use the explicitly bootstrapped, statically selected coroalloc backend: +// native GC builds use BDWGC uncollectable ranges, nogc/wasm profiles use C +// malloc/free, and bare-metal builds use tinygogc. // // The entry path is intentionally single-use. No failure path resets this // object: exported ABI failures terminate the process, and successful startup // transitions from unused to complete or permanently failed. -type coroProgramStateV1 struct { - lifecycle coroProgramLifecycleV1 - manifest *coro.ProgramManifestV1 - factory unsafe.Pointer - g coroG - p coroP -} - -var coroProgramV1 coroProgramStateV1 +// Keep phase-0 fields as separate globals. Besides making ownership explicit, +// this avoids a synthetic nil-dereference helper on field access through the +// address of one aggregate global; the process-entry ABI must remain a plain, +// non-suspending call island. +var ( + coroProgramLifecycleV1State coroProgramLifecycleV1 + coroProgramManifestV1State *coro.ProgramManifestV1 + coroProgramFactoryV1State unsafe.Pointer + coroProgramGV1State coroG + coroProgramPV1State coroP +) func coroProgramBeginV1(manifest, expectedFactory unsafe.Pointer) (unsafe.Pointer, bool) { - state := &coroProgramV1 - if state.lifecycle != coroProgramUnusedV1 { - state.lifecycle = coroProgramFailedV1 + if coroProgramLifecycleV1State != coroProgramUnusedV1 { + coroProgramLifecycleV1State = coroProgramFailedV1 + return nil, false + } + if !coroalloc.Ready() { + coroProgramLifecycleV1State = coroProgramFailedV1 return nil, false } if manifest == nil { - state.lifecycle = coroProgramFailedV1 + coroProgramLifecycleV1State = coroProgramFailedV1 return nil, false } - if _, code := coro.ValidateRunnableDirectProgramV1( - (*coro.ProgramManifestV1)(manifest), expectedFactory, - ); code != coro.ProgramValidationOKV1 { - state.lifecycle = coroProgramFailedV1 + programManifest := (*coro.ProgramManifestV1)(manifest) + _, v2Code := coro.ValidateRunnableProgramV2(programManifest, expectedFactory) + _, v1Code := coro.ValidateRunnableDirectProgramV1(programManifest, expectedFactory) + if v2Code != coro.ProgramValidationOKV2 && v1Code != coro.ProgramValidationOKV1 { + coroProgramLifecycleV1State = coroProgramFailedV1 return nil, false } - if !coroInitG(&state.g) { - state.lifecycle = coroProgramFailedV1 + if !coroInitG(&coroProgramGV1State) { + coroProgramLifecycleV1State = coroProgramFailedV1 return nil, false } - state.manifest = (*coro.ProgramManifestV1)(manifest) - state.factory = expectedFactory - state.lifecycle = coroProgramBegunV1 - return unsafe.Pointer(&state.g), true + coroProgramManifestV1State = (*coro.ProgramManifestV1)(manifest) + coroProgramFactoryV1State = expectedFactory + coroProgramLifecycleV1State = coroProgramBegunV1 + return unsafe.Pointer(&coroProgramGV1State), true } func coroProgramRunV1(gPointer, handle unsafe.Pointer) bool { - state := &coroProgramV1 - if state.lifecycle != coroProgramBegunV1 || state.manifest == nil || state.factory == nil || - gPointer != unsafe.Pointer(&state.g) || handle == nil { - state.lifecycle = coroProgramFailedV1 + if coroProgramLifecycleV1State != coroProgramBegunV1 || coroProgramManifestV1State == nil || coroProgramFactoryV1State == nil || + gPointer != unsafe.Pointer(&coroProgramGV1State) || handle == nil { + coroProgramLifecycleV1State = coroProgramFailedV1 return false } - if !coroAdoptRoot(&state.g, handle) || !coroEnqueue(&state.p, &state.g) { - state.lifecycle = coroProgramFailedV1 + if !coroAdoptRoot(&coroProgramGV1State, handle) || !coroEnqueue(&coroProgramPV1State, &coroProgramGV1State) { + coroProgramLifecycleV1State = coroProgramFailedV1 return false } - state.lifecycle = coroProgramRunningV1 - if !coroRun(&state.p) || !coro.TerminalG(&state.p, &state.g) { - state.lifecycle = coroProgramFailedV1 + coroProgramLifecycleV1State = coroProgramRunningV1 + if !coroRun(&coroProgramPV1State) || !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { + coroProgramLifecycleV1State = coroProgramFailedV1 return false } - state.lifecycle = coroProgramCompleteV1 + coroProgramLifecycleV1State = coroProgramCompleteV1 return true } diff --git a/runtime/internal/runtime/coro_program_test.go b/runtime/internal/runtime/coro_program_test.go index b22044b8be..bf9a3a8c53 100644 --- a/runtime/internal/runtime/coro_program_test.go +++ b/runtime/internal/runtime/coro_program_test.go @@ -89,6 +89,49 @@ func newCoroProgramTestManifestV1() *coroProgramTestManifestV1 { return fixture } +type coroProgramTestManifestV2 struct { + factoryMarker byte + plainTargets [5]byte + steps [5]coro.ProgramStepV2 + bootstrap coro.ProgramBootstrapV2 + manifest coro.ProgramManifestV1 +} + +func newCoroProgramTestManifestV2() *coroProgramTestManifestV2 { + fixture := new(coroProgramTestManifestV2) + fixture.factoryMarker = 0x42 + fixture.plainTargets = [5]byte{0x31, 0x32, 0x33, 0x34, 0x35} + roles := [...]uint32{ + coro.ProgramStepFlagInternalRuntimeInitV2, + coro.ProgramStepFlagCompilerABIInitV2, + coro.ProgramStepFlagPublicRuntimeInitV2, + coro.ProgramStepFlagMainPackageInitV2, + coro.ProgramStepFlagMainV2, + } + for index, role := range roles { + fixture.steps[index] = coro.ProgramStepV2{ + Kind: uint32(coro.ProgramStepDirectPlainV2), + Flags: role, + Target: unsafe.Pointer(&fixture.plainTargets[index]), + } + } + fixture.bootstrap = coro.ProgramBootstrapV2{ + Version: coro.ProgramBootstrapVersionV2, + HashLo: 0x2122232425262728, + HashHi: 0x3132333435363738, + StepCount: uintptr(len(fixture.steps)), + Steps: unsafe.Pointer(&fixture.steps[0]), + Factory: unsafe.Pointer(&fixture.factoryMarker), + } + fixture.manifest = coro.ProgramManifestV1{ + Version: coro.ProgramManifestVersionV1, + HashLo: fixture.bootstrap.HashLo, + HashHi: fixture.bootstrap.HashHi, + Bootstrap: unsafe.Pointer(&fixture.bootstrap), + } + return fixture +} + type coroProgramTestFrameV1 struct { g *coro.G handle unsafe.Pointer @@ -156,6 +199,14 @@ type coroProgramTestDriverV1 struct { var activeCoroProgramDriver *coroProgramTestDriverV1 +// The named-source host test deliberately does not link BDWGC or libc. Set the +// allocator's private readiness byte to the bootstrapReady value so this test +// can exercise the program adapter independently; coroalloc's own tests and +// compiler IR tests cover the real bootstrap boundary. +// +//go:linkname testCoroAllocatorBootstrapState github.com/goplus/llgo/runtime/internal/coroalloc.state +var testCoroAllocatorBootstrapState uint8 + // coro_program.go aborts through the full LLGo runtime. The named-source host // test intentionally excludes that unrelated runtime implementation (which // defines symbols reserved by the host Go runtime), so failures use this local @@ -216,10 +267,20 @@ func (driver *coroProgramTestDriverV1) destroy(handle unsafe.Pointer) { func resetCoroProgramTestStateV1(t *testing.T) { t.Helper() - coroProgramV1 = coroProgramStateV1{} + testCoroAllocatorBootstrapState = 2 + coroProgramLifecycleV1State = coroProgramUnusedV1 + coroProgramManifestV1State = nil + coroProgramFactoryV1State = nil + coroProgramGV1State = coroG{} + coroProgramPV1State = coroP{} activeCoroProgramDriver = nil t.Cleanup(func() { - coroProgramV1 = coroProgramStateV1{} + testCoroAllocatorBootstrapState = 0 + coroProgramLifecycleV1State = coroProgramUnusedV1 + coroProgramManifestV1State = nil + coroProgramFactoryV1State = nil + coroProgramGV1State = coroG{} + coroProgramPV1State = coroP{} activeCoroProgramDriver = nil }) } @@ -230,28 +291,54 @@ func TestCoroProgramV1BeginRunAndDestroy(t *testing.T) { factory := unsafe.Pointer(&manifest.factoryMarker) gPointer, ok := coroProgramBeginV1(unsafe.Pointer(&manifest.manifest), factory) - if !ok || gPointer != unsafe.Pointer(&coroProgramV1.g) || !coro.ValidG(&coroProgramV1.g) { - t.Fatalf("begin coroutine program = (%p, %t), want initialized static G %p", gPointer, ok, &coroProgramV1.g) + if !ok || gPointer != unsafe.Pointer(&coroProgramGV1State) || !coro.ValidG(&coroProgramGV1State) { + t.Fatalf("begin coroutine program = (%p, %t), want initialized static G %p", gPointer, ok, &coroProgramGV1State) } - if coroProgramV1.lifecycle != coroProgramBegunV1 || coroProgramV1.manifest != &manifest.manifest || coroProgramV1.factory != factory { - t.Fatalf("begun coroutine program state = {lifecycle:%d manifest:%p factory:%p}", coroProgramV1.lifecycle, coroProgramV1.manifest, coroProgramV1.factory) + if coroProgramLifecycleV1State != coroProgramBegunV1 || coroProgramManifestV1State != &manifest.manifest || coroProgramFactoryV1State != factory { + t.Fatalf("begun coroutine program state = {lifecycle:%d manifest:%p factory:%p}", coroProgramLifecycleV1State, coroProgramManifestV1State, coroProgramFactoryV1State) } - frame := newCoroProgramTestFrameV1(t, &coroProgramV1.g) + frame := newCoroProgramTestFrameV1(t, &coroProgramGV1State) driver := &coroProgramTestDriverV1{t: t, frame: frame} activeCoroProgramDriver = driver if !coroProgramRunV1(gPointer, frame.handle) { t.Fatal("run valid coroutine program") } - if coroProgramV1.lifecycle != coroProgramCompleteV1 || !coro.TerminalG(&coroProgramV1.p, &coroProgramV1.g) { - t.Fatalf("completed coroutine program retained scheduler state: lifecycle=%d", coroProgramV1.lifecycle) + if coroProgramLifecycleV1State != coroProgramCompleteV1 || !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { + t.Fatalf("completed coroutine program retained scheduler state: lifecycle=%d", coroProgramLifecycleV1State) } if driver.doneCalls != 2 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released { t.Fatalf("coroutine wrapper calls = done:%d resume:%d destroy:%d released:%t", driver.doneCalls, driver.resumeCalls, driver.destroyCalls, driver.released) } - if _, ok := coroProgramBeginV1(unsafe.Pointer(&manifest.manifest), factory); ok || coroProgramV1.lifecycle != coroProgramFailedV1 { - t.Fatalf("completed coroutine program was reusable: ok=%t lifecycle=%d", ok, coroProgramV1.lifecycle) + if _, ok := coroProgramBeginV1(unsafe.Pointer(&manifest.manifest), factory); ok || coroProgramLifecycleV1State != coroProgramFailedV1 { + t.Fatalf("completed coroutine program was reusable: ok=%t lifecycle=%d", ok, coroProgramLifecycleV1State) + } + runtime.KeepAlive(frame.memory) + runtime.KeepAlive(manifest) +} + +func TestCoroProgramV2BeginRunAndDestroy(t *testing.T) { + resetCoroProgramTestStateV1(t) + manifest := newCoroProgramTestManifestV2() + factory := unsafe.Pointer(&manifest.factoryMarker) + + gPointer, ok := coroProgramBeginV1(unsafe.Pointer(&manifest.manifest), factory) + if !ok || gPointer != unsafe.Pointer(&coroProgramGV1State) || !coro.ValidG(&coroProgramGV1State) { + t.Fatalf("begin coroutine program v2 = (%p, %t), want initialized static G %p", gPointer, ok, &coroProgramGV1State) + } + + frame := newCoroProgramTestFrameV1(t, &coroProgramGV1State) + driver := &coroProgramTestDriverV1{t: t, frame: frame} + activeCoroProgramDriver = driver + if !coroProgramRunV1(gPointer, frame.handle) { + t.Fatal("run valid coroutine program v2") + } + if coroProgramLifecycleV1State != coroProgramCompleteV1 || !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { + t.Fatalf("completed coroutine program v2 retained scheduler state: lifecycle=%d", coroProgramLifecycleV1State) + } + if driver.doneCalls != 2 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released { + t.Fatalf("coroutine v2 wrapper calls = done:%d resume:%d destroy:%d released:%t", driver.doneCalls, driver.resumeCalls, driver.destroyCalls, driver.released) } runtime.KeepAlive(frame.memory) runtime.KeepAlive(manifest) @@ -263,8 +350,8 @@ func TestCoroProgramV1BeginFailsClosedOnFactoryIdentity(t *testing.T) { otherFactory := new(byte) if g, ok := coroProgramBeginV1( unsafe.Pointer(&manifest.manifest), unsafe.Pointer(otherFactory), - ); ok || g != nil || coroProgramV1.lifecycle != coroProgramFailedV1 || coro.ValidG(&coroProgramV1.g) { - t.Fatalf("factory mismatch = (%p, %t), lifecycle=%d validG=%t", g, ok, coroProgramV1.lifecycle, coro.ValidG(&coroProgramV1.g)) + ); ok || g != nil || coroProgramLifecycleV1State != coroProgramFailedV1 || coro.ValidG(&coroProgramGV1State) { + t.Fatalf("factory mismatch = (%p, %t), lifecycle=%d validG=%t", g, ok, coroProgramLifecycleV1State, coro.ValidG(&coroProgramGV1State)) } runtime.KeepAlive(manifest) } @@ -272,8 +359,8 @@ func TestCoroProgramV1BeginFailsClosedOnFactoryIdentity(t *testing.T) { func TestCoroProgramV1BeginFailsClosedOnNilManifest(t *testing.T) { resetCoroProgramTestStateV1(t) if g, ok := coroProgramBeginV1(nil, unsafe.Pointer(new(byte))); ok || g != nil || - coroProgramV1.lifecycle != coroProgramFailedV1 || coro.ValidG(&coroProgramV1.g) { - t.Fatalf("nil manifest = (%p, %t), lifecycle=%d validG=%t", g, ok, coroProgramV1.lifecycle, coro.ValidG(&coroProgramV1.g)) + coroProgramLifecycleV1State != coroProgramFailedV1 || coro.ValidG(&coroProgramGV1State) { + t.Fatalf("nil manifest = (%p, %t), lifecycle=%d validG=%t", g, ok, coroProgramLifecycleV1State, coro.ValidG(&coroProgramGV1State)) } } @@ -285,8 +372,8 @@ func TestCoroProgramV1RunFailsClosedOnInvalidHandle(t *testing.T) { if !ok { t.Fatal("begin coroutine program before invalid run") } - if coroProgramRunV1(g, nil) || coroProgramV1.lifecycle != coroProgramFailedV1 { - t.Fatalf("nil-handle run did not fail closed: lifecycle=%d", coroProgramV1.lifecycle) + if coroProgramRunV1(g, nil) || coroProgramLifecycleV1State != coroProgramFailedV1 { + t.Fatalf("nil-handle run did not fail closed: lifecycle=%d", coroProgramLifecycleV1State) } runtime.KeepAlive(manifest) } diff --git a/runtime/internal/runtime/coro_sched.go b/runtime/internal/runtime/coro_sched.go index 8c66d37762..11573919dc 100644 --- a/runtime/internal/runtime/coro_sched.go +++ b/runtime/internal/runtime/coro_sched.go @@ -66,7 +66,9 @@ func coroRun(p *coroP) bool { return false } if g == nil { - return true + // Platform event-loop integration is the next adapter layer. Never + // confuse an empty ready queue with completion while parked Gs remain. + return !coro.HasWaiting(p) } if !coroRunG(p, g) { return false @@ -78,9 +80,11 @@ func coroRun(p *coroP) bool { // wrappers stay direct calls so scheduler internals do not introduce function // values, interface dispatch, or unnecessary dual sync/async versions. func coroRunActions(p *coroP, g *coroG, action coro.Action) bool { - for action.Kind != coro.ActionComplete { + for { var ok bool switch action.Kind { + case coro.ActionComplete, coro.ActionYield, coro.ActionPark: + return true case coro.ActionCheckResume, coro.ActionCheckDestroy: action, ok = coro.Checked(p, g, action, coroHandleDone(action.Handle)) case coro.ActionResume: @@ -96,5 +100,4 @@ func coroRunActions(p *coroP, g *coroG, action coro.Action) bool { return false } } - return true } diff --git a/runtime/internal/runtime/tinygogc/gc_tinygo.go b/runtime/internal/runtime/tinygogc/gc_tinygo.go index cca6f9fccd..9f05e1f1e2 100644 --- a/runtime/internal/runtime/tinygogc/gc_tinygo.go +++ b/runtime/internal/runtime/tinygogc/gc_tinygo.go @@ -116,6 +116,15 @@ func lazyInit() { } } +// Init performs the bounded phase-0 heap metadata initialization before the +// first stackless coroutine frame is allocated. It is safe to call again from +// the ordinary runtime initialization path. +func Init() { + lock(&gcMutex) + lazyInit() + unlock(&gcMutex) +} + func gcPanic(s *c.Char) { c.Printf(c.Str("%s"), s) c.Exit(2) diff --git a/runtime/internal/runtime/z_signal.go b/runtime/internal/runtime/z_signal.go index 1283dff626..08f977ebda 100644 --- a/runtime/internal/runtime/z_signal.go +++ b/runtime/internal/runtime/z_signal.go @@ -1,4 +1,4 @@ -//go:build !wasm && !baremetal +//go:build !wasm && !baremetal && !llgo_coro /* * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. From fa09d827e4e5208244386809112f4c45e38c3305 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 22:31:52 +0800 Subject: [PATCH 14/32] compiler(coro): lower synchronous Go style onto LLVM frames --- cl/blocks/block.go | 11 +- cl/blocks/block_test.go | 2 +- cl/compilation.go | 2 +- cl/compilation_test.go | 2 +- cl/compile.go | 56 +- cl/coro_abi.go | 567 +++++++++++-- cl/coro_abi_test.go | 694 +++++++++++++++- cl/coro_await.go | 71 +- cl/coro_entry.go | 14 +- cl/coro_entry_test.go | 4 +- cl/coro_lowered_call.go | 95 +++ cl/coro_park_test.go | 203 +++++ cl/coro_pure_ssa.go | 588 ++++++++++++++ cl/coro_pure_ssa_test.go | 356 +++++++++ cl/emission_abi_demand.go | 54 +- cl/emission_abi_demand_test.go | 3 +- cl/emission_allocacstr_coro_test.go | 221 +++++ cl/emission_atomic_coro_test.go | 82 ++ cl/emission_call_roots.go | 3 +- cl/emission_deferdata_coro_test.go | 103 +++ cl/emission_foreign_noblock_test.go | 134 ++++ cl/emission_lowered_call_test.go | 228 ++++++ cl/emission_runtime_abi_test.go | 112 +++ cl/emission_runtime_helpers.go | 734 +++++++++++++++++ cl/emission_sigjmp_coro_test.go | 100 +++ cl/emission_string_coro_test.go | 96 +++ cl/emission_universe.go | 754 +++++++++++++++++- cl/import.go | 3 + cl/instr.go | 7 + internal/build/build.go | 550 ++++++++++++- internal/build/collect.go | 1 + internal/build/coro_bootstrap.go | 357 ++++++++- internal/build/coro_bootstrap_factory.go | 195 +++++ internal/build/coro_bootstrap_factory_test.go | 200 +++++ internal/build/coro_bootstrap_test.go | 494 +++++++++++- internal/build/coro_foreign_noblock_test.go | 157 ++++ internal/build/coro_funcaddr_test.go | 209 +++++ internal/build/coro_panic_legacy_test.go | 59 ++ internal/build/coro_plan_test.go | 376 ++++++++- internal/build/coro_registry.go | 14 +- internal/build/coro_runtime_abi_gate_test.go | 56 ++ internal/build/coro_tls_destructor_test.go | 6 +- internal/build/fingerprint.go | 3 +- internal/build/gc_target_test.go | 54 ++ internal/build/main_module.go | 202 +++-- internal/build/main_module_test.go | 241 ++++++ internal/build/target_config_test.go | 30 + internal/coro/func_flow.go | 75 +- internal/coro/graph.go | 20 +- internal/coro/plan_digest.go | 54 +- internal/coro/plan_digest_test.go | 18 + internal/coro/ssa_plan.go | 182 ++++- internal/coro/ssa_plan_test.go | 58 ++ runtime/internal/clite/pthread/pthread.go | 4 + runtime/internal/clite/pthread/sync/sync.go | 8 + runtime/internal/clite/time/time.go | 4 + runtime/internal/clite/tls/tls_gc.go | 9 +- ssa/abitype.go | 15 + ssa/coro.go | 132 ++- ssa/coro_test.go | 278 ++++++- 60 files changed, 8958 insertions(+), 402 deletions(-) create mode 100644 cl/coro_lowered_call.go create mode 100644 cl/coro_park_test.go create mode 100644 cl/coro_pure_ssa.go create mode 100644 cl/coro_pure_ssa_test.go create mode 100644 cl/emission_allocacstr_coro_test.go create mode 100644 cl/emission_atomic_coro_test.go create mode 100644 cl/emission_deferdata_coro_test.go create mode 100644 cl/emission_foreign_noblock_test.go create mode 100644 cl/emission_runtime_abi_test.go create mode 100644 cl/emission_runtime_helpers.go create mode 100644 cl/emission_sigjmp_coro_test.go create mode 100644 cl/emission_string_coro_test.go create mode 100644 internal/build/coro_foreign_noblock_test.go create mode 100644 internal/build/coro_funcaddr_test.go create mode 100644 internal/build/coro_panic_legacy_test.go create mode 100644 internal/build/coro_runtime_abi_gate_test.go create mode 100644 internal/build/gc_target_test.go diff --git a/cl/blocks/block.go b/cl/blocks/block.go index 2aa4bea11b..372515df9b 100644 --- a/cl/blocks/block.go +++ b/cl/blocks/block.go @@ -22,8 +22,9 @@ import ( ) type Info struct { - Kind llssa.DoAction - Next int + Kind llssa.DoAction + Next int + InLoop bool } // ----------------------------------------------------------------------------- @@ -168,7 +169,11 @@ retry: ret := make([]Info, n) for i := 0; i < n; i++ { iblk := order[i] - ret[iblk] = Info{states[iblk].kind(), order[i+1]} + ret[iblk] = Info{ + Kind: states[iblk].kind(), + Next: order[i+1], + InLoop: states[iblk].inLoop, + } } return ret } diff --git a/cl/blocks/block_test.go b/cl/blocks/block_test.go index d0e5b72b23..4e4702b2f8 100644 --- a/cl/blocks/block_test.go +++ b/cl/blocks/block_test.go @@ -57,7 +57,7 @@ func TestFirstLoop(t *testing.T) { blk.Preds = []*ssa.BasicBlock{blk} blk.Succs = []*ssa.BasicBlock{blk} infos := Infos([]*ssa.BasicBlock{blk}) - if infos[0].Kind != llssa.DeferInLoop { + if infos[0].Kind != llssa.DeferInLoop || !infos[0].InLoop { t.Fatal("TestFirstLoop") } } diff --git a/cl/compilation.go b/cl/compilation.go index 7d5a5e8047..e44dfae1b1 100644 --- a/cl/compilation.go +++ b/cl/compilation.go @@ -112,7 +112,7 @@ func (c *Compilation) validateCoroABIIdentity(required bool) error { if !c.EnableCoroChildAwait { return fmt.Errorf("coroutine program bootstrap runtime requires child-await lowering") } - wantSchedulerABI = coro.SchedulerProgramBootstrapABIV1 + wantSchedulerABI = coro.SchedulerProgramBootstrapABIV2 } if c.EnableCoroPlainDispatch && !c.EnableCoroEntryResolution { return fmt.Errorf("coroutine plain dispatch requires coroutine entry resolution") diff --git a/cl/compilation_test.go b/cl/compilation_test.go index 5b0a41da75..c779eadaae 100644 --- a/cl/compilation_test.go +++ b/cl/compilation_test.go @@ -138,7 +138,7 @@ func TestCompilationCoroABIIdentityValidation(t *testing.T) { } programBootstrap := newChildAwait() programBootstrap.EnableCoroProgramBootstrapRun = true - programBootstrap.SchedulerABI = coro.SchedulerProgramBootstrapABIV1 + programBootstrap.SchedulerABI = coro.SchedulerProgramBootstrapABIV2 if err := programBootstrap.validateCoroABIIdentity(false); err != nil { t.Fatalf("complete program-bootstrap ABI identity: %v", err) } diff --git a/cl/compile.go b/cl/compile.go index 82104bfe85..432ec5994b 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -185,6 +185,7 @@ type context struct { pcLineSeq uint64 sourceParamBase int // hidden physical parameters before source params currentCoro *coroBodyContext + coroSourceBlocks []llssa.BasicBlock // source SSA block index -> logical LLVM block coroRootFactories []coroRootFactoryRegistration coroPlainDescriptors map[string]llssa.Expr @@ -621,7 +622,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun } else { fn.MakeBlocks(nblk) // to set fn.HasBody() = true } - if f.Recover != nil { // set recover block + if f.Recover != nil && physicalABI == nil { // set recover block fn.SetRecover(fn.Block(f.Recover.Index)) } dbgEnabled := enableDbg && (f == nil || f.Origin() == nil) @@ -652,7 +653,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun p.bvals = make(map[ssa.Value]llssa.Expr) p.methodNilDerefChecks = collectMethodNilDerefChecks(f) if physicalABI != nil { - p.compileCoroPhysicalBody(b, f, *physicalABI) + p.compileCoroPhysicalBody(b, f, *physicalABI, isInit) b.EndBuild() return } @@ -827,9 +828,9 @@ func (p *context) debugRef(b llssa.Builder, v *ssa.DebugRef) { diScope := b.DIScope(p.fn, scope) if v.IsAddr { // *ssa.Alloc - b.DIDeclare(variable, value, dbgVar, diScope, pos, b.Func.Block(v.Block().Index)) + b.DIDeclare(variable, value, dbgVar, diScope, pos, p.sourceBlock(v.Block().Index)) } else { - b.DIValue(variable, value, dbgVar, diScope, pos, b.Func.Block(v.Block().Index)) + b.DIValue(variable, value, dbgVar, diScope, pos, p.sourceBlock(v.Block().Index)) } } @@ -844,10 +845,24 @@ func (p *context) debugParams(b llssa.Builder, f *ssa.Function) { if p.paramDIVars != nil { p.paramDIVars[variable] = div } - b.DIParam(variable, v, div, p.fn, pos, p.fn.Block(0)) + b.DIParam(variable, v, div, p.fn, pos, p.sourceBlock(0)) } } +// sourceBlock maps a Go SSA basic-block index to the logical LLVM block used +// by the current lowering. Plain functions retain the historical one-to-one +// Function.Block mapping. A physical coroutine has a dedicated ramp and +// internal suspend blocks, so its source CFG uses an explicit stable map. +func (p *context) sourceBlock(index int) llssa.BasicBlock { + if len(p.coroSourceBlocks) != 0 { + if index < 0 || index >= len(p.coroSourceBlocks) { + panic(fmt.Sprintf("source basic block index %d is outside coroutine map of length %d", index, len(p.coroSourceBlocks))) + } + return p.coroSourceBlocks[index] + } + return p.fn.Block(index) +} + func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, doModInit bool) llssa.BasicBlock { var last int var pyModInit bool @@ -855,7 +870,7 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do var pkg = p.pkg var fn = p.fn var instrs = block.Instrs[n:] - var ret = fn.Block(block.Index) + var ret = p.sourceBlock(block.Index) b.SetBlock(ret) if block.Index == 0 && p.shouldTrackCallerFrames() { p.pushCallerLocationFrame(b, block.Parent()) @@ -889,6 +904,13 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do isCgoC2 := isCgoC2func(fnName) isCgoCmacro := isCgoCmacro(fnName) for i, instr := range instrs { + if p.currentCoro != nil { + if _, debug := instr.(*ssa.DebugRef); debug { + p.compileInstr(b, instr) + continue + } + p.currentCoro.countInstructionAndMaybeYield(b) + } if i == 1 && doModInit && p.state == pkgInPatch { // in patch package but no pkgFNoOldInit initFnNameOld := initFnNameOfHasPatch(p.fn.Name()) fnOld := pkg.NewFunc(initFnNameOld, llssa.NoArgsNoRet, llssa.InC) @@ -1167,8 +1189,7 @@ func isPhi(i ssa.Instruction) bool { } func (p *context) compilePhis(b llssa.Builder, block *ssa.BasicBlock) int { - fn := p.fn - ret := fn.Block(block.Index) + ret := p.sourceBlock(block.Index) b.SetBlockEx(ret, llssa.AtEnd, false) if ninstr := len(block.Instrs); ninstr > 0 { if isPhi(block.Instrs[0]) { @@ -1198,7 +1219,7 @@ func (p *context) compilePhi(b llssa.Builder, v *ssa.Phi) (ret llssa.Expr) { preds := v.Block().Preds bblks := make([]llssa.BasicBlock, len(preds)) for i, pred := range preds { - bblks[i] = p.fn.Block(pred.Index) + bblks[i] = p.sourceBlock(pred.Index) } edges := v.Edges phi.AddIncoming(b, bblks, func(i int, blk llssa.BasicBlock) llssa.Expr { @@ -1575,9 +1596,8 @@ func (p *context) assertNilDerefBase(b llssa.Builder, addr ssa.Value) { } func (p *context) jumpTo(v *ssa.Jump) llssa.BasicBlock { - fn := p.fn succs := v.Block().Succs - return fn.Block(succs[0].Index) + return p.sourceBlock(succs[0].Index) } func (p *context) getDebugLocScope(v *ssa.Function, pos token.Pos) *types.Scope { @@ -1653,13 +1673,20 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { if p.shouldTrackCallerFrames() { p.popCallerLocationFrame(b) } + if p.currentCoro != nil { + if p.currentCoro.completion == nil { + panic("coroutine return has no completion block") + } + p.storeCoroLeafResult(b, p.currentCoro.abi, p.currentCoro.resultSlot, results) + b.Jump(p.currentCoro.completion) + return + } b.Return(results...) case *ssa.If: - fn := p.fn cond := p.compileValue(b, v.Cond) succs := v.Block().Succs - thenb := fn.Block(succs[0].Index) - elseb := fn.Block(succs[1].Index) + thenb := p.sourceBlock(succs[0].Index) + elseb := p.sourceBlock(succs[1].Index) b.If(cond, thenb, elseb) case *ssa.MapUpdate: m := p.compileValue(b, v.Map) @@ -2064,6 +2091,7 @@ func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewri ret.SetResolveLinkname(ctx.resolveLinkname) if opts.Compilation != nil && opts.Compilation.EnableCoroEntryResolution { ret.SetResolveMethodLinkname(ctx.resolveMethodLinkname) + ret.SetResolveRuntimeCall(ctx.resolveCoroLoweredRuntimeCall) } if hasPatch { diff --git a/cl/coro_abi.go b/cl/coro_abi.go index bf9f017d6c..effd8e0d40 100644 --- a/cl/coro_abi.go +++ b/cl/coro_abi.go @@ -25,6 +25,7 @@ import ( "go/types" "strings" + "github.com/goplus/llgo/cl/blocks" "github.com/goplus/llgo/internal/coro" llssa "github.com/goplus/llgo/ssa" "golang.org/x/tools/go/ssa" @@ -42,6 +43,9 @@ const ( coroFrameAllocHookV1 = "__llgo_coro_frame_alloc_v1" coroFramePublishHookV1 = "__llgo_coro_frame_publish_v1" coroAwaitPrepareHookV1 = "__llgo_coro_await_prepare_v1" + coroPreemptPollHookV1 = "__llgo_coro_preempt_poll_v1" + coroYieldPrepareHookV1 = "__llgo_coro_yield_prepare_v1" + coroParkPrepareHookV1 = "__llgo_coro_park_prepare_v1" coroCompletePrepareHookV1 = "__llgo_coro_complete_prepare_v1" coroFrameFreeHookV1 = "__llgo_coro_frame_free_v1" coroDescriptorPrefixV1 = "__llgo_coro_frame_descriptor_v1." @@ -63,6 +67,8 @@ const ( coroSuspendNone uint64 = iota coroSuspendCall coroSuspendFrameComplete + coroSuspendYield + coroSuspendPark ) const ( @@ -75,6 +81,11 @@ const ( coroLifecycleDestroyed ) +// coroPreemptInstructionBudget bounds straight-line source work between +// compiler-inserted scheduler handoffs. Loop SCC entries are separate +// safepoints, so even a tiny loop cannot run forever without a cut. +const coroPreemptInstructionBudget = 64 + type coroPhysicalABI struct { version uint32 hash [16]byte @@ -83,6 +94,9 @@ type coroPhysicalABI struct { frameFreeHook string framePublishHook string awaitPrepareHook string + preemptPollHook string + yieldPrepareHook string + parkPrepareHook string completePrepareHook string physicalSig *types.Signature resultSlotType types.Type @@ -98,8 +112,14 @@ type coroBodyContext struct { header llssa.Expr task llssa.Expr resultSlot llssa.Expr + completion llssa.BasicBlock + preemptPoll llssa.Expr + yieldPrepare llssa.Expr + parkPrepare llssa.Expr completePrepare llssa.Expr nextState uint32 + needsPreempt bool + instructions int } func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *types.Signature) coroPhysicalABI { @@ -109,6 +129,9 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type descriptorPrefix := coroDescriptorPrefix framePublishHook := "" awaitPrepareHook := "" + preemptPollHook := "" + yieldPrepareHook := "" + parkPrepareHook := "" completePrepareHook := "" if p.compilation != nil && p.compilation.EnableCoroChildAwait { version = coroPhysicalABIVersionV1 @@ -117,6 +140,9 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type descriptorPrefix = coroDescriptorPrefixV1 framePublishHook = coroFramePublishHookV1 awaitPrepareHook = coroAwaitPrepareHookV1 + preemptPollHook = coroPreemptPollHookV1 + yieldPrepareHook = coroYieldPrepareHookV1 + parkPrepareHook = coroParkPrepareHookV1 completePrepareHook = coroCompletePrepareHookV1 } resultFields := make([]*types.Var, sourceSig.Results().Len()) @@ -192,6 +218,9 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type frameFreeHook: frameFreeHook, framePublishHook: framePublishHook, awaitPrepareHook: awaitPrepareHook, + preemptPollHook: preemptPollHook, + yieldPrepareHook: yieldPrepareHook, + parkPrepareHook: parkPrepareHook, completePrepareHook: completePrepareHook, physicalSig: physicalSig, resultSlotType: resultSlotType, @@ -271,6 +300,15 @@ func (p *context) beginCoroBody(b llssa.Builder, abi coroPhysicalABI) *coroBodyC if abi.completePrepareHook != "" { body.completePrepare = p.pkg.NewFunc(abi.completePrepareHook, coroCompletePrepareSignature(), llssa.InC).Expr } + if abi.yieldPrepareHook != "" { + body.yieldPrepare = p.pkg.NewFunc(abi.yieldPrepareHook, coroYieldPrepareSignature(), llssa.InC).Expr + } + if abi.parkPrepareHook != "" { + body.parkPrepare = p.pkg.NewFunc(abi.parkPrepareHook, coroParkPrepareSignature(), llssa.InC).Expr + } + if abi.preemptPollHook != "" { + body.preemptPoll = p.pkg.NewFunc(abi.preemptPollHook, coroPreemptPollSignature(), llssa.InC).Expr + } body.coro = b.BeginCoro(llssa.CoroOptions{ Promise: header, Frame: frame, @@ -341,6 +379,27 @@ func coroCompletePrepareSignature() *types.Signature { return types.NewSignatureType(nil, nil, nil, params, nil, false) } +func coroYieldPrepareSignature() *types.Signature { + return coroCompletePrepareSignature() +} + +func coroParkPrepareSignature() *types.Signature { + params := types.NewTuple( + types.NewParam(token.NoPos, nil, "g", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "handle", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "header", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "token", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "ticket", types.Typ[types.Uint32]), + ) + return types.NewSignatureType(nil, nil, nil, params, nil, false) +} + +func coroPreemptPollSignature() *types.Signature { + params := types.NewTuple(types.NewParam(token.NoPos, nil, "g", types.Typ[types.UnsafePointer])) + results := types.NewTuple(types.NewParam(token.NoPos, nil, "requested", types.Typ[types.Bool])) + return types.NewSignatureType(nil, nil, nil, params, results, false) +} + func (c *coroBodyContext) publishState(b llssa.Builder, reason, lifecycle uint64, stateID uint32) { prog := b.Prog b.Store(b.FieldAddr(c.header, coroHeaderSuspendReason), prog.IntVal(reason, prog.Uint16())) @@ -363,10 +422,75 @@ func (c *coroBodyContext) suspendForChild(b llssa.Builder) uint32 { } stateID := c.nextState c.nextState++ + c.instructions = 0 c.publishState(b, coroSuspendCall, coroLifecycleSuspended, stateID) return stateID } +func (c *coroBodyContext) pollAndSuspendForPreempt(b llssa.Builder) uint32 { + if c.abi.version < coroPhysicalABIVersionV1 || c.preemptPoll.IsNil() || c.yieldPrepare.IsNil() { + panic("coroutine preemption requires PhysicalABIV1 poll and scheduler handoff hooks") + } + stateID := c.nextState + c.nextState++ + c.instructions = 0 + requested := b.Call(c.preemptPoll, c.task) + c.coro.SuspendCurrentBlockIf(requested, func(suspend llssa.Builder) { + c.publishState(suspend, coroSuspendYield, coroLifecycleSuspended, stateID) + suspend.Call(c.yieldPrepare, c.task, c.coro.Handle(), suspend.Convert(suspend.Prog.VoidPtr(), c.header)) + }) + // The false poll edge is already active; repeating these stores there keeps + // the joined continuation state-independent while the resumed true edge + // clears its published yield state before executing source instructions. + c.activate(b) + return stateID +} + +// parkCurrentFrame is the exact stack-cut primitive used by future channel, +// timer, syscall, and platform adapters. The suspend must remain here in the +// caller's physical coroutine body; a normal synchronous helper cannot retain +// the caller's native activation across llvm.coro.suspend. +func (c *coroBodyContext) parkCurrentFrame(b llssa.Builder, token, ticket llssa.Expr) uint32 { + if c.abi.version < coroPhysicalABIVersionV1 || c.parkPrepare.IsNil() { + panic("coroutine park requires PhysicalABIV1 scheduler handoff hook") + } + stateID := c.nextState + c.nextState++ + c.instructions = 0 + c.publishState(b, coroSuspendPark, coroLifecycleSuspended, stateID) + b.Call( + c.parkPrepare, + c.task, + c.coro.Handle(), + b.Convert(b.Prog.VoidPtr(), c.header), + b.Convert(b.Prog.VoidPtr(), token), + b.Convert(b.Prog.Uint32(), ticket), + ) + c.coro.SuspendCurrentBlock() + c.activate(b) + return stateID +} + +func (p *context) compileCoroPark(b llssa.Builder, args []llssa.Expr) { + if p.currentCoro == nil || p.compilation == nil || !p.compilation.EnableCoroChildAwait { + panic("llgo.coroPark requires an active PhysicalABIV1 coroutine body") + } + if b.Func != p.fn || len(args) != 2 { + panic("llgo.coroPark requires exactly (token, ticket) in the active coroutine function") + } + p.currentCoro.parkCurrentFrame(b, args[0], args[1]) +} + +func (c *coroBodyContext) countInstructionAndMaybeYield(b llssa.Builder) { + if !c.needsPreempt { + return + } + if c.instructions >= coroPreemptInstructionBudget { + c.pollAndSuspendForPreempt(b) + } + c.instructions++ +} + func (c *coroBodyContext) finish(b llssa.Builder) { if c.abi.version < coroPhysicalABIVersionV1 { c.coro.Finish() @@ -390,55 +514,87 @@ func (p *context) storeCoroLeafResult(b llssa.Builder, abi coroPhysicalABI, resu } resultType := p.prog.Type(abi.resultSlotType, llssa.InGo) typedSlot := b.Convert(p.prog.Pointer(resultType), resultSlot) - b.Store(b.FieldAddr(typedSlot, 0), results[0]) + for i, result := range results { + b.Store(b.FieldAddr(typedSlot, i), result) + } } -func (p *context) compileCoroPhysicalBody(b llssa.Builder, fn *ssa.Function, abi coroPhysicalABI) { - if len(fn.Blocks) != 1 { - panic("coroutine physical body reached codegen without one-block preflight") - } +func (p *context) compileCoroPhysicalBody(b llssa.Builder, fn *ssa.Function, abi coroPhysicalABI, isInit bool) { oldBase := p.sourceParamBase oldCoro := p.currentCoro + oldSourceBlocks := p.coroSourceBlocks p.sourceParamBase = 2 defer func() { p.sourceParamBase = oldBase p.currentCoro = oldCoro + p.coroSourceBlocks = oldSourceBlocks }() b.SetBlock(p.fn.Block(0)) - if enableDbgSyms && fn.Origin() == nil { - p.debugParams(b, fn) - } physical := p.beginCoroBody(b, abi) p.currentCoro = physical - body := physical.coro.InitialResumeBlock() - completion := p.fn.MakeBlock() - b.SetBlock(body) + + // Create source blocks after BeginCoro's canonical ramp/suspend blocks so + // presplit IR remains in execution order for LLVM diagnostics and ABI tests. + sourceBlocks := make([]llssa.BasicBlock, len(fn.Blocks)) + for i := range sourceBlocks { + sourceBlocks[i] = p.fn.MakeBlock() + } + p.coroSourceBlocks = sourceBlocks + physical.completion = p.fn.MakeBlock() + b.SetBlock(physical.coro.InitialResumeBlock()) physical.activate(b) + b.Jump(sourceBlocks[0]) - for _, instr := range fn.Blocks[0].Instrs { - if _, debug := instr.(*ssa.DebugRef); debug { - // Source block 0 is not physical ramp block 0. Until the general - // source-to-resume block map lands, omit local debug intrinsics - // instead of emitting a non-dominating use into the ramp. - continue + off := make([]int, len(fn.Blocks)) + for i, block := range fn.Blocks { + off[i] = p.compilePhis(b, block) + } + p.blkInfos = blocks.Infos(fn.Blocks) + plan, ok := p.compilation.CoroPlan.FunctionPlan(fn) + if !ok { + panic("coroutine physical body has no compilation plan") + } + physical.needsPreempt = plan.Exec.Contains(coro.NeedsPreempt) + + i := 0 + for { + block := fn.Blocks[i] + if physical.needsPreempt { + physical.instructions = 0 + // Every source block, including block zero, begins with a poll. A + // child initial suspend is a scheduler boundary but not necessarily + // a fairness boundary: pendingAwait can immediately resume a long + // static child chain on the same G without returning to ready-queue + // selection. Polling block zero therefore bounds that chain as well + // as ordinary CFG paths and block-zero backedges. + b.SetBlock(p.sourceBlock(i)) + physical.pollAndSuspendForPreempt(b) } - if ret, ok := instr.(*ssa.Return); ok { - results := make([]llssa.Expr, len(ret.Results)) - for i, result := range ret.Results { - results[i] = p.compileValue(b, result) - } - p.storeCoroLeafResult(b, abi, physical.resultSlot, results) - b.Jump(completion) - continue + doModInit := i == 1 && isInit + p.compileBlock(b, block, off[i], doModInit) + if i = p.blkInfos[i].Next; i < 0 { + break } - p.compileInstr(b, instr) } - b.SetBlock(completion) + for _, phi := range p.phis { + phi() + } + + b.SetBlock(physical.completion) physical.finish(b) } -func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, childAwait bool) error { +func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, childAwait, programRun bool) error { + return validateCoroPhysicalABIWithUniverse(fn, plan, whole, nil, childAwait, programRun) +} + +// validateCoroPhysicalABIWithUniverse is the production preflight. The +// prepared emission universe supplies the exact frontend lowering context used +// to prove that an accepted pure SSA instruction emits no hidden runtime call. +// The wrapper above is retained for narrow structural unit tests; active +// Compilation paths always call this form with their frozen universe. +func validateCoroPhysicalABIWithUniverse(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, universe *EmissionUniverse, childAwait, programRun bool) error { if !childAwait { return validateCoroLeafPhysicalABI(fn, plan) } @@ -449,6 +605,9 @@ func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *co if fn == nil || plan.External != coro.Defined || len(fn.Blocks) == 0 { return fail("requires one defined SSA body") } + if emitShadowStackInstrumentation { + return fail("legacy thread-local shadow-stack instrumentation is incompatible with stackless coroutine suspension") + } if plan.Emission != coro.EmitCoroutine || plan.FuncRep != coro.DirectCoro { return fail("requires a direct coroutine emission, got emission=%s representation=%s", plan.Emission, plan.FuncRep) } @@ -458,8 +617,15 @@ func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *co if plan.Recursive { return fail("recursive coroutine lowering requires child frames and preemption polls") } - if unsupported := plan.Exec &^ coro.MayUnwind; unsupported != 0 { - return fail("execution flags %s require lowering outside the linear physical ABI", unsupported) + if plan.Exec.Contains(coro.NeedsPreempt) && !programRun { + return fail("needs-preempt execution requires the runnable scheduler ABI") + } + // IRQUnsafe constrains interrupt roots; an ordinary scheduler-managed G is + // not an IRQ context. Preserve the bit in the plan/digest while allowing the + // CFG lowering to execute it. Thread affinity and opaque execution still + // require scheduler protocols that this ABI does not provide. + if unsupported := plan.Exec &^ (coro.MayUnwind | coro.NeedsPreempt | coro.IRQUnsafe); unsupported != 0 { + return fail("execution flags %s require lowering outside the CFG physical ABI", unsupported) } if fn.Parent() != nil || len(fn.FreeVars) != 0 { return fail("closures require the coroutine context ABI") @@ -467,6 +633,9 @@ func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *co if len(fn.AnonFuncs) != 0 { return fail("nested function literals require closure body lowering") } + if fn.Recover != nil { + return fail("recover blocks require coroutine cleanup/unwind lowering") + } if fn.Signature.Recv() != nil { return fail("methods require descriptor and receiver ABI lowering") } @@ -479,7 +648,8 @@ func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *co if isCgoExternSymbol(fn) { return fail("cgo entry requires a foreign adapter") } - if fn.Synthetic != "" { + programEntry := programRun && isCoroProgramManagedEntry(fn) + if fn.Synthetic != "" && !(programEntry && fn.Name() == "init" && fn.Synthetic == "package initializer") { return fail("synthetic function %q is outside the leaf ABI", fn.Synthetic) } if list := fn.TypeParams(); list != nil && list.Len() != 0 { @@ -488,75 +658,174 @@ func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *co if list := fn.TypeArgs(); len(list) != 0 { return fail("generic instances require a frozen instantiated ABI") } - if fn.Name() == "main" || strings.HasPrefix(fn.Name(), "init") { + if (fn.Name() == "main" || strings.HasPrefix(fn.Name(), "init")) && !programEntry { return fail("program roots require scheduler bootstrap lowering") } - if len(fn.Blocks) != 1 { - return fail("requires exactly one basic block, got %d", len(fn.Blocks)) - } if err := validateCoroLeafPhysicalSignature(plan, fn.Signature); err != nil { return err } + pureSSA, err := newCoroPhysicalPureSSAAudit(universe, fn) + if err != nil { + return fail("cannot audit pure SSA lowering: %v", err) + } returns := 0 awaits := 0 - for _, instr := range fn.Blocks[0].Instrs { - switch instr := instr.(type) { - case *ssa.DebugRef: - case *ssa.Return: - returns++ - case *ssa.BinOp: - if instr.Op == token.QUO || instr.Op == token.REM || instr.Op == token.SHL || instr.Op == token.SHR || - !coroLeafScalar(instr.Type()) || - !coroLeafScalar(instr.X.Type()) || !coroLeafScalar(instr.Y.Type()) { - return coroLeafInstructionError(fn, plan, instr, "potentially panicking or non-scalar binary operation") - } - case *ssa.UnOp: - if (instr.Op != token.SUB && instr.Op != token.XOR && instr.Op != token.NOT) || !coroLeafScalar(instr.Type()) { - return coroLeafInstructionError(fn, plan, instr, "unsupported unary operation") - } - case *ssa.Convert, *ssa.ChangeType: - value, ok := instr.(ssa.Value) - if !ok || !coroLeafScalar(value.Type()) { - return coroLeafInstructionError(fn, plan, instr, "non-scalar conversion") - } - case *ssa.Call: - callee, calleePlan, err := resolveCoroStaticAwait(whole, plan, instr) - if err != nil { - return coroLeafInstructionError(fn, plan, instr, "unsupported child await: "+err.Error()) + parks := 0 + infos := blocks.Infos(fn.Blocks) + hasCyclicBlock := false + for _, info := range infos { + hasCyclicBlock = hasCyclicBlock || info.InLoop + } + if hasCyclicBlock && !plan.Exec.Contains(coro.NeedsPreempt) { + return fail("cyclic CFG requires needs-preempt execution classification") + } + for _, block := range fn.Blocks { + for _, instr := range block.Instrs { + if handled, reason := pureSSA.validate(instr); handled { + if reason != "" { + return coroLeafInstructionError(fn, plan, instr, reason) + } + continue } - if err := validateCoroLeafPhysicalSignature(calleePlan, callee.Signature); err != nil { - return coroLeafInstructionError(fn, plan, instr, "child await signature: "+err.Error()) + switch instr := instr.(type) { + case *ssa.DebugRef, *ssa.Jump: + case *ssa.Return: + returns++ + case *ssa.If: + if !coroLeafScalar(instr.Cond.Type()) { + return coroLeafInstructionError(fn, plan, instr, "non-scalar branch condition") + } + case *ssa.BinOp: + if instr.Op == token.QUO || instr.Op == token.REM || instr.Op == token.SHL || instr.Op == token.SHR || + !coroLeafScalar(instr.Type()) || + !coroLeafScalar(instr.X.Type()) || !coroLeafScalar(instr.Y.Type()) { + return coroLeafInstructionError(fn, plan, instr, "potentially panicking or non-scalar binary operation") + } + case *ssa.UnOp: + if (instr.Op != token.SUB && instr.Op != token.XOR && instr.Op != token.NOT) || !coroLeafScalar(instr.Type()) { + return coroLeafInstructionError(fn, plan, instr, "unsupported unary operation") + } + case *ssa.Call: + if whole != nil && whole.ElidesCall(instr) { + if universe != nil { + rawCallee := instr.Call.StaticCallee() + if _, frozen := universe.Resolve(rawCallee); rawCallee != nil && frozen { + semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(instr) + if err != nil { + return coroLeafInstructionError(fn, plan, instr, "invalid frozen intrinsic: "+err.Error()) + } + if intrinsic && semantics.SuspendsCurrentFrame() { + parks++ + } + } + } + // The frozen frontend proved that this declaration call emits no + // callable edge. A structured park is counted above; ordinary + // noinit/inline intrinsics need no await/plain entry. + continue + } + callee, calleePlan, err := resolveCoroStaticAwait(whole, plan, instr) + if err == nil { + if err := validateCoroLeafPhysicalSignature(calleePlan, callee.Signature); err != nil { + return coroLeafInstructionError(fn, plan, instr, "child await signature: "+err.Error()) + } + awaits++ + continue + } + if !programRun { + return coroLeafInstructionError(fn, plan, instr, "unsupported child await: "+err.Error()) + } + if _, _, plainErr := resolveCoroStaticPlainCall(whole, instr); plainErr != nil { + return coroLeafInstructionError(fn, plan, instr, "unsupported call: child await: "+err.Error()+"; direct plain: "+plainErr.Error()) + } + default: + return coroLeafInstructionError(fn, plan, instr, "instruction is outside the CFG physical ABI allowlist") } - awaits++ - default: - return coroLeafInstructionError(fn, plan, instr, "instruction is outside the linear physical ABI allowlist") } } - if returns != 1 { - return fail("requires exactly one return instruction, got %d", returns) + if returns == 0 { + return fail("requires at least one return instruction") } - if awaits == 0 { - if plan.DeclaredEffect != coro.YieldOnly || plan.LocalEffect != coro.YieldOnly || plan.Effect != coro.YieldOnly { - return fail("requires an explicit, isolated yield-only effect, got declared=%s local=%s final=%s", plan.DeclaredEffect, plan.LocalEffect, plan.Effect) - } - return nil + if !plan.Effect.MaySuspend() { + return fail("CFG physical body lacks a suspension-capable final effect: %s", plan.Effect) } - if !plan.Effect.Contains(coro.AwaitStructured) { + if awaits != 0 && !plan.Effect.Contains(coro.AwaitStructured) { return fail("child-await body lacks await-structured final effect: %s", plan.Effect) } - if unsupported := plan.Effect &^ (coro.YieldOnly | coro.AwaitStructured); unsupported != 0 { + if parks != 0 && !plan.Effect.Contains(coro.MayPark) { + return fail("structured-park body lacks may-park final effect: %s", plan.Effect) + } + if plan.DeclaredEffect.Contains(coro.MayPark) && parks == 0 { + return fail("declared may-park effect has no exact structured park intrinsic") + } + if unsupported := plan.Effect &^ (coro.YieldOnly | coro.AwaitStructured | coro.MayPark); unsupported != 0 { return fail("child-await body has unsupported final effect %s", unsupported) } - if unsupported := plan.DeclaredEffect &^ coro.YieldOnly; unsupported != 0 { + if unsupported := plan.DeclaredEffect &^ (coro.YieldOnly | coro.MayPark); unsupported != 0 { return fail("child-await body has unsupported declared effect %s", unsupported) } - if unsupported := plan.LocalEffect &^ coro.YieldOnly; unsupported != 0 { + if unsupported := plan.LocalEffect &^ (coro.YieldOnly | coro.MayPark); unsupported != 0 { return fail("child-await body has unsupported local effect %s", unsupported) } return nil } +func isCoroProgramManagedEntry(fn *ssa.Function) bool { + if fn == nil { + return false + } + name := fn.Name() + if name == "init" || strings.HasPrefix(name, "init#") { + return true + } + return name == "main" && fn.Pkg != nil && fn.Pkg.Pkg != nil && fn.Pkg.Pkg.Name() == "main" +} + +// resolveCoroStaticPlainCall proves the synchronous island allowed inside a +// runnable physical coroutine. The exact CallPlan must select either one +// defined primary plain body or one frozen known external plain entry, and it +// must be bounded and non-suspending. A missing/open/dynamic edge may not fall +// back to the legacy source symbol. +func resolveCoroStaticPlainCall(plan *coro.SSAPlan, call ssa.CallInstruction) (*ssa.Function, coro.FunctionPlan, error) { + if plan == nil || call == nil || call.Common() == nil { + return nil, coro.FunctionPlan{}, fmt.Errorf("requires a compilation CallPlan") + } + common := call.Common() + if common.IsInvoke() || common.StaticCallee() == nil { + return nil, coro.FunctionPlan{}, fmt.Errorf("requires a static non-invoke call") + } + callPlan, ok := plan.CallPlan(call) + if !ok { + return nil, coro.FunctionPlan{}, fmt.Errorf("call has no compilation CallPlan") + } + if callPlan.Kind != coro.CallDirect || callPlan.Rep != coro.DirectPlain || callPlan.Open || callPlan.MayBeNil || len(callPlan.Targets) != 1 { + return nil, coro.FunctionPlan{}, fmt.Errorf( + "requires one closed non-nil direct plain target, got kind=%v representation=%s open=%t may-be-nil=%t targets=%d", + callPlan.Kind, callPlan.Rep, callPlan.Open, callPlan.MayBeNil, len(callPlan.Targets), + ) + } + target, ok := plan.Function(callPlan.Targets[0]) + if !ok || target == nil { + return nil, coro.FunctionPlan{}, fmt.Errorf("direct plain target %q is absent from the compilation plan", callPlan.Targets[0]) + } + targetPlan, ok := plan.FunctionPlan(target) + if !ok || targetPlan.ID != callPlan.Targets[0] { + return nil, coro.FunctionPlan{}, fmt.Errorf("direct plain target %q has no canonical function plan", callPlan.Targets[0]) + } + validBody := targetPlan.External == coro.Defined && targetPlan.Emission == coro.EmitPlain && targetPlan.Primary == coro.PrimaryPlain + validExternal := targetPlan.External == coro.ExternalKnown && targetPlan.Emission == coro.EmitExternal && targetPlan.Primary == coro.PrimaryExternal + unsupportedExec := targetPlan.Exec &^ (coro.MayUnwind | coro.IRQUnsafe) + if (!validBody && !validExternal) || targetPlan.FuncRep != coro.DirectPlain || targetPlan.Effect != coro.NoSuspend || + targetPlan.Demand == coro.NoDemand || unsupportedExec != 0 { + return nil, coro.FunctionPlan{}, fmt.Errorf( + "target %q is not one demanded defined-or-known-external bounded no-suspend direct plain entry (external=%s emission=%s primary=%s representation=%s effect=%s exec=%s demand=%s)", + targetPlan.ID, targetPlan.External, targetPlan.Emission, targetPlan.Primary, targetPlan.FuncRep, targetPlan.Effect, targetPlan.Exec, targetPlan.Demand, + ) + } + return target, targetPlan, nil +} + // validateCoroLeafPhysicalABI preserves the v0 leaf-only acceptance boundary // and diagnostics. Enabling later physical ABI capabilities must not silently // change an archive still identified as PhysicalABIV0/SchedulerNoneABIV0. @@ -567,6 +836,9 @@ func validateCoroLeafPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan) error if fn == nil || plan.External != coro.Defined || len(fn.Blocks) == 0 { return fail("requires one defined SSA body") } + if emitShadowStackInstrumentation { + return fail("legacy thread-local shadow-stack instrumentation is incompatible with stackless coroutine suspension") + } if plan.Emission != coro.EmitCoroutine || plan.FuncRep != coro.DirectCoro { return fail("requires a direct coroutine emission, got emission=%s representation=%s", plan.Emission, plan.FuncRep) } @@ -676,20 +948,151 @@ func validateCoroLeafPhysicalSignature(plan coro.FunctionPlan, sig *types.Signat if sig.Variadic() { return fail("effective variadic coroutine ABI is not implemented") } + if params := sig.TypeParams(); params != nil && params.Len() != 0 { + return fail("effective generic declaration has %d type parameters", params.Len()) + } + if params := sig.RecvTypeParams(); params != nil && params.Len() != 0 { + return fail("effective generic receiver has %d type parameters", params.Len()) + } for i := 0; i < sig.Params().Len(); i++ { - if !coroLeafScalar(sig.Params().At(i).Type()) { - return fail("parameter %d has unsupported type %s", i, sig.Params().At(i).Type()) + if err := validateCoroPhysicalValueType(sig.Params().At(i).Type(), make(map[types.Type]bool)); err != nil { + return fail("parameter %d has unsupported type %s: %v", i, sig.Params().At(i).Type(), err) } } - if sig.Results().Len() > 1 { - return fail("supports at most one result, got %d", sig.Results().Len()) - } - if sig.Results().Len() == 1 && !coroLeafScalar(sig.Results().At(0).Type()) { - return fail("result has unsupported type %s", sig.Results().At(0).Type()) + for i := 0; i < sig.Results().Len(); i++ { + if err := validateCoroPhysicalValueType(sig.Results().At(i).Type(), make(map[types.Type]bool)); err != nil { + return fail("result %d has unsupported type %s: %v", i, sig.Results().At(i).Type(), err) + } } return nil } +// validateCoroPhysicalFunctionValueABI keeps function-valued transport on the +// one compilation-wide representation path. The generic LLGo type converter +// supplies the canonical two-pointer closure layout, while FuncRepABIV1's +// ValuePlan validation decides whether the first word is a direct entry or a +// descriptor. Accepting the width here must not create a second, unplanned +// function representation at a coroutine boundary. +func validateCoroPhysicalFunctionValueABI(plan coro.FunctionPlan, sig *types.Signature, plainDispatch bool) error { + if sig == nil || !coroPhysicalSignatureContainsFunctionValue(sig) || plainDispatch { + return nil + } + return fmt.Errorf( + "coroutine physical ABI: function %q: function-valued parameters/results require canonical ValuePlan validation and the descriptor/closure ABI", + plan.ID, + ) +} + +func coroPhysicalSignatureContainsFunctionValue(sig *types.Signature) bool { + for _, tuple := range []*types.Tuple{sig.Params(), sig.Results()} { + if tuple == nil { + continue + } + for i := 0; i < tuple.Len(); i++ { + if coroPhysicalTypeContainsFunctionValue(tuple.At(i).Type(), make(map[types.Type]bool)) { + return true + } + } + } + return false +} + +func coroPhysicalTypeContainsFunctionValue(typ types.Type, visiting map[types.Type]bool) bool { + if typ == nil { + return false + } + typ = types.Unalias(typ) + if visiting[typ] { + return false + } + visiting[typ] = true + defer delete(visiting, typ) + switch value := typ.(type) { + case *types.Signature: + return true + case *types.Named: + return coroPhysicalTypeContainsFunctionValue(value.Underlying(), visiting) + case *types.Struct: + for i := 0; i < value.NumFields(); i++ { + if coroPhysicalTypeContainsFunctionValue(value.Field(i).Type(), visiting) { + return true + } + } + case *types.Array, *types.Slice, *types.Chan: + var elem types.Type + switch container := value.(type) { + case *types.Array: + elem = container.Elem() + case *types.Slice: + elem = container.Elem() + case *types.Chan: + elem = container.Elem() + } + return coroPhysicalTypeContainsFunctionValue(elem, visiting) + case *types.Map: + return coroPhysicalTypeContainsFunctionValue(value.Key(), visiting) || + coroPhysicalTypeContainsFunctionValue(value.Elem(), visiting) + case *types.Tuple: + for i := 0; i < value.Len(); i++ { + if coroPhysicalTypeContainsFunctionValue(value.At(i).Type(), visiting) { + return true + } + } + } + return false +} + +// validateCoroPhysicalValueType proves only that a source value has a stable +// LLGo by-value representation that can be copied through the typed coroutine +// result slot. It does not authorize any SSA producer/consumer instruction: +// those remain governed by the physical-body allowlist and ValuePlan checks. +func validateCoroPhysicalValueType(typ types.Type, visiting map[types.Type]bool) error { + if typ == nil { + return fmt.Errorf("nil type") + } + typ = types.Unalias(typ) + if visiting[typ] { + return nil + } + visiting[typ] = true + defer delete(visiting, typ) + + switch value := typ.(type) { + case *types.Named: + return validateCoroPhysicalValueType(value.Underlying(), visiting) + case *types.Basic: + if value.Kind() == types.Invalid || value.Info()&types.IsUntyped != 0 { + return fmt.Errorf("invalid or untyped basic kind %s", value) + } + return nil + case *types.Pointer, *types.Map, *types.Chan, *types.Interface, *types.Slice, *types.Signature: + // These are target-width opaque pointers or LLGo's stable descriptor / + // closure aggregates. Their referent/method/call signature is logical + // identity, not an inline extension of the transported value layout. + return nil + case *types.Struct: + for i := 0; i < value.NumFields(); i++ { + if err := validateCoroPhysicalValueType(value.Field(i).Type(), visiting); err != nil { + return fmt.Errorf("field %d: %w", i, err) + } + } + return nil + case *types.Array: + if value.Len() < 0 { + return fmt.Errorf("negative array length %d", value.Len()) + } + return validateCoroPhysicalValueType(value.Elem(), visiting) + case *types.TypeParam: + return fmt.Errorf("uninstantiated type parameter") + case *types.Tuple: + return fmt.Errorf("tuple is valid only as the outer result list") + case *types.Union: + return fmt.Errorf("union has no runtime value representation") + default: + return fmt.Errorf("unsupported type class %T", typ) + } +} + func coroLeafABIDirective(fn *ssa.Function) string { decl, _ := fn.Syntax().(*ast.FuncDecl) if decl == nil || decl.Doc == nil { diff --git a/cl/coro_abi_test.go b/cl/coro_abi_test.go index 62085f06e5..9641b380aa 100644 --- a/cl/coro_abi_test.go +++ b/cl/coro_abi_test.go @@ -23,6 +23,7 @@ import ( "encoding/binary" "encoding/hex" "go/ast" + "go/types" "regexp" "strconv" "strings" @@ -233,6 +234,7 @@ func TestCoroChildAwaitPhysicalABIV1Presplit(t *testing.T) { coroFrameAllocHookV1, coroFramePublishHookV1, coroAwaitPrepareHookV1, + coroPreemptPollHookV1, coroCompletePrepareHookV1, coroFrameFreeHookV1, } { @@ -332,6 +334,513 @@ func TestCoroChildAwaitPhysicalABIV1CoroSplit(t *testing.T) { } } +func TestCoroPreemptiveLoopPhysicalABIV1(t *testing.T) { + const source = `package foo +func Loop(limit uint32) uint32 { + var value uint32 + for value < limit { + value++ + } + return value +} +` + prog, ssaPkg, files, universe, plan := prepareCoroPreemptTestPlan( + t, + source, + []coroRootFactoryTestRoot{{name: "Loop", demand: coro.AsyncDemand}}, + nil, + -1, + ) + defer prog.Dispose() + loop := ssaPkg.Func("Loop") + loopPlan, ok := plan.FunctionPlan(loop) + if !ok || loopPlan.Emission != coro.EmitCoroutine || loopPlan.FuncRep != coro.DirectCoro || + !loopPlan.Exec.Contains(coro.NeedsPreempt) || !loopPlan.Effect.Contains(coro.YieldOnly) { + t.Fatalf("Loop plan = %+v, present=%t; want direct needs-preempt coroutine", loopPlan, ok) + } + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroPreemptCompilation(compilation) + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + t.Fatal(err) + } + module := pkg.Module() + defer module.Dispose() + body := requireCoroPhysicalFunction(t, module, "foo.Loop").String() + if !strings.Contains(body, "call i1 @"+coroPreemptPollHookV1) { + t.Fatalf("Loop lacks compiler-inserted preemption poll:\n%s", body) + } + if !strings.Contains(body, "call void @"+coroYieldPrepareHookV1) { + t.Fatalf("Loop lacks compiler-inserted scheduler yield handoff:\n%s", body) + } + if !regexp.MustCompile(`(?s)store i16 3,.*store i16 3,.*call void @` + regexp.QuoteMeta(coroYieldPrepareHookV1)).MatchString(body) { + t.Fatalf("Loop does not publish Yield/Suspended before its handoff:\n%s", body) + } + poll := strings.Index(body, "call i1 @"+coroPreemptPollHookV1) + handoff := strings.Index(body, "call void @"+coroYieldPrepareHookV1) + if poll < 0 || handoff < 0 || poll >= handoff || !strings.Contains(body[poll:handoff], "br i1") { + t.Fatalf("Loop yield handoff is not guarded by its preemption poll:\n%s", body) + } + if got := strings.Count(body, "call i8 @llvm.coro.suspend"); got < 3 { + t.Fatalf("Loop coroutine suspends = %d, want initial + yield + final:\n%s", got, body) + } + runCoroABITestPipeline(t, prog, module) + post := module.String() + for _, suffix := range []string{".resume", ".destroy"} { + if fn := module.NamedFunction("foo.Loop$coro" + suffix); fn.IsNil() { + t.Fatalf("CoroSplit did not create Loop%s:\n%s", suffix, post) + } + } +} + +func TestCoroProgramInitPhysicalABIV2(t *testing.T) { + const source = `package foo +import ( + "embed" + _ "unsafe" +) + +var State uint32 +var Files embed.FS + +func Plain() { State = 1 } +func Yield() { State = 2 } +func init() { + Plain() + Yield() +} +` + prog, ssaPkg, files, universe, plan := prepareCoroProgramInitTestPlan(t, source) + defer prog.Dispose() + packageInit := ssaPkg.Func("init") + initPlan, ok := plan.FunctionPlan(packageInit) + if !ok || initPlan.Emission != coro.EmitCoroutine || initPlan.FuncRep != coro.DirectCoro || initPlan.Demand != coro.AsyncDemand { + t.Fatalf("package init plan = %+v, present=%t; want async-only direct coroutine", initPlan, ok) + } + foundElidedUnsafeInit := false + for _, block := range packageInit.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(*ssa.Call) + if !ok || call.Call.StaticCallee() == nil || call.Call.StaticCallee().Pkg == nil || + call.Call.StaticCallee().Pkg.Pkg.Path() != "unsafe" || call.Call.StaticCallee().Name() != "init" { + continue + } + foundElidedUnsafeInit = plan.ElidesCall(call) + if _, planned := plan.CallPlan(call); planned { + t.Fatal("frontend-elided unsafe.init unexpectedly has a CallPlan") + } + } + } + if !foundElidedUnsafeInit { + t.Fatal("package init fixture has no exact frontend-elided unsafe.init call") + } + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroPreemptCompilation(compilation) + embedMap := goembed.VarMap{ + "Files": {Files: []goembed.FileData{{Name: "asset.txt", Data: []byte("payload")}}}, + } + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, embedMap, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + t.Fatal(err) + } + module := pkg.Module() + defer module.Dispose() + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify coroutine package init: %v\n%s", err, module.String()) + } + + packageInitIR := requireCoroPhysicalFunction(t, module, "foo.init").String() + if !strings.Contains(packageInitIR, `load i1, ptr @"foo.init$guard"`) || + !strings.Contains(packageInitIR, `store i1 true, ptr @"foo.init$guard"`) { + t.Fatalf("package init coroutine lost its canonical guard load/store:\n%s", packageInitIR) + } + if !strings.Contains(module.String(), "asset.txt") || !strings.Contains(module.String(), "payload") { + t.Fatalf("package init coroutine did not apply compiler-generated embed initialization:\n%s", packageInitIR) + } + if !regexp.MustCompile(`call void @"?embed\.init"?\(`).MatchString(packageInitIR) { + t.Fatalf("package init lost its exact known-external no-suspend call:\n%s", packageInitIR) + } + declaredInit := requireCoroPhysicalFunction(t, module, "foo.init#1").String() + if !regexp.MustCompile(`call void @"?foo\.Plain"?\(`).MatchString(declaredInit) { + t.Fatalf("declared init lost its exact direct plain call:\n%s", declaredInit) + } + if !regexp.MustCompile(`call ptr @"?foo\.Yield\$coro"?\(`).MatchString(declaredInit) || + !strings.Contains(declaredInit, "call void @"+coroAwaitPrepareHookV1) { + t.Fatalf("declared init lost its static child await:\n%s", declaredInit) + } + runCoroABITestPipeline(t, prog, module) +} + +func TestCoroPhysicalValueTransportABIV1NativeAndWasm(t *testing.T) { + llssa.Initialize(llssa.InitAll) + for _, test := range []struct { + name string + target *llssa.Target + pointerBits int + uintptrIR string + }{ + {name: "native"}, + {name: "wasm32", target: &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}, pointerBits: 32, uintptrIR: "i32"}, + } { + t.Run(test.name, func(t *testing.T) { + prog, ssaPkg, files, universe, plan := prepareCoroPhysicalValueTransportABI(t, test.target) + defer prog.Dispose() + pointerBits := prog.PointerSize() * 8 + if test.pointerBits != 0 && pointerBits != test.pointerBits { + t.Fatalf("pointer width = %d, want %d", pointerBits, test.pointerBits) + } + uintptrIR := test.uintptrIR + if uintptrIR == "" { + uintptrIR = "i" + strconv.Itoa(pointerBits) + } + + child := ssaPkg.Func("Child") + callbackPlan, ok := plan.ValuePlan(child.Params[0]) + if !ok || len(callbackPlan.Funcs) != 1 || len(callbackPlan.Funcs[0].Path) != 0 || + callbackPlan.Funcs[0].Rep != coro.Dispatch { + t.Fatalf("Child callback ValuePlan = %+v, present=%t; want one canonical scalar Dispatch leaf", callbackPlan, ok) + } + parent := ssaPkg.Func("Parent") + var childCall *ssa.Call + for _, block := range parent.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(*ssa.Call) + if ok && call.Call.StaticCallee() == child { + childCall = call + } + } + } + if childCall == nil { + t.Fatal("Parent has no static Child call") + } + nilCallbackPlan, ok := plan.ValuePlan(childCall.Call.Args[0]) + if !ok || len(nilCallbackPlan.Funcs) != 1 || len(nilCallbackPlan.Funcs[0].Path) != 0 || + nilCallbackPlan.Funcs[0].Rep != coro.Dispatch || !nilCallbackPlan.Funcs[0].MayBeNil || + len(nilCallbackPlan.Funcs[0].Targets) != 0 { + t.Fatalf("nil callback ValuePlan = %+v, present=%t; want closed nil canonical Dispatch leaf", nilCallbackPlan, ok) + } + disabled := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroChildAwaitCompilation(disabled) + got, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: disabled}, + ) + if err == nil || !strings.Contains(err.Error(), "require canonical ValuePlan validation") { + t.Fatalf("function-value gate-off result = %v, %v; want canonical ValuePlan rejection", got, err) + } + if got != nil { + t.Fatal("function-value preflight failure returned a partial package") + } + + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroChildAwaitCompilation(compilation) + compilation.EnableCoroPlainDispatch = true + compilation.FuncRepABI = coro.FuncRepABIV1 + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + t.Fatal(err) + } + module := pkg.Module() + defer module.Dispose() + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify physical value transport before CoroSplit: %v\n%s", err, module.String()) + } + + childIR := requireCoroPhysicalFunction(t, module, "foo.Child").String() + parentIR := requireCoroPhysicalFunction(t, module, "foo.Parent").String() + pairIR := requireCoroPhysicalFunction(t, module, "foo.Pair").String() + if !regexp.MustCompile(`define ptr @"?foo\.Child\$coro"?\(ptr [^,]+, ptr [^,]+, \{ ptr, ptr \} [^,]+, ptr `).MatchString(childIR) { + t.Fatalf("Child callback/pointer parameters do not use LLGo's canonical two-pointer closure layout:\n%s", childIR) + } + if !regexp.MustCompile(`call ptr @"?foo\.Child\$coro"?\([^\n]*\{ ptr, ptr \} zeroinitializer, ptr `).MatchString(parentIR) { + t.Fatalf("Parent did not transport the nil callback through the typed canonical closure argument:\n%s", parentIR) + } + assertCoroResultSlotFields(t, "Pair before CoroSplit", pairIR, uintptrIR) + if !regexp.MustCompile(`store %foo\.Payload [^,]+, ptr `).MatchString(childIR) { + t.Fatalf("Child did not copy the complete named struct result into its typed result slot:\n%s", childIR) + } + + runCoroABITestPipeline(t, prog, module) + post := module.String() + for _, function := range []string{"foo.Child$coro", "foo.Parent$coro", "foo.Pair$coro"} { + for _, suffix := range []string{".resume", ".destroy"} { + if module.NamedFunction(function + suffix).IsNil() { + t.Fatalf("CoroSplit did not create %s%s:\n%s", function, suffix, post) + } + } + } + assertCoroResultSlotFields(t, "Pair after CoroSplit", module.NamedFunction("foo.Pair$coro.resume").String(), uintptrIR) + if !regexp.MustCompile(`store %foo\.Payload [^,]+, ptr `).MatchString(module.NamedFunction("foo.Child$coro.resume").String()) { + t.Fatalf("Child struct result store did not survive CoroSplit:\n%s", module.NamedFunction("foo.Child$coro.resume").String()) + } + }) + } +} + +func TestCoroAwaitResultReconstruction(t *testing.T) { + prog := newLLSSAProg(t) + defer prog.Dispose() + pkg := prog.NewPackage("awaitresult", "await/result") + module := pkg.Module() + defer module.Dispose() + physical := &context{prog: prog} + pointer := types.NewPointer(types.Typ[types.Uint32]) + for _, test := range []struct { + name string + results *types.Tuple + loads int + inserts int + }{ + {name: "zero", results: types.NewTuple()}, + {name: "one", results: types.NewTuple(types.NewVar(0, nil, "ptr", pointer)), loads: 1}, + {name: "many", results: types.NewTuple( + types.NewVar(0, nil, "ptr", pointer), + types.NewVar(0, nil, "count", types.Typ[types.Uintptr]), + ), loads: 2, inserts: 2}, + } { + resultCount := 0 + if test.results != nil { + resultCount = test.results.Len() + } + fields := make([]*types.Var, resultCount) + for i := range fields { + fields[i] = types.NewField(0, nil, test.results.At(i).Name(), test.results.At(i).Type(), false) + } + name := "await_" + test.name + fn := pkg.NewFunc(name, llssa.NoArgsNoRet, llssa.InGo) + b := fn.MakeBody(1) + slot := b.AllocaT(prog.Type(types.NewStruct(fields, nil), llssa.InGo)) + got := physical.loadCoroAwaitResult(b, slot, test.results) + switch resultCount { + case 0: + if !got.IsNil() { + t.Fatalf("zero-result await value type = %v, want llssa.Nil", got.RawType()) + } + case 1: + if got.IsNil() || !types.Identical(got.RawType(), prog.Type(pointer, llssa.InGo).RawType()) { + t.Fatalf("one-result await value type = %v, want field type %v", got.RawType(), pointer) + } + default: + if got.IsNil() || !types.Identical(got.RawType(), prog.Type(test.results, llssa.InGo).RawType()) { + t.Fatalf("multi-result await value type = %v, want source tuple %v", got.RawType(), test.results) + } + } + b.Return() + b.EndBuild() + b.Dispose() + body := module.NamedFunction(name).String() + if got := strings.Count(body, "load "); got != test.loads { + t.Fatalf("%s await result loads = %d, want %d:\n%s", test.name, got, test.loads, body) + } + if got := strings.Count(body, "insertvalue "); got != test.inserts { + t.Fatalf("%s await result tuple inserts = %d, want %d:\n%s", test.name, got, test.inserts, body) + } + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify await result reconstruction: %v\n%s", err, module.String()) + } +} + +func assertCoroResultSlotFields(t *testing.T, name, body, uintptrIR string) { + t.Helper() + resultType := regexp.QuoteMeta("{ ptr, " + uintptrIR + " }") + for index, storeType := range []string{"ptr", uintptrIR} { + field := regexp.MustCompile( + `(?m)^\s*(%[-a-zA-Z$._0-9]+) = getelementptr inbounds ` + resultType + + `, ptr [^,]+, i32 0, i32 ` + strconv.Itoa(index) + `\s*$`, + ).FindStringSubmatch(body) + if len(field) != 2 || !regexp.MustCompile(`(?m)^\s*store `+storeType+` [^,]+, ptr `+regexp.QuoteMeta(field[1])+`(?:,|\s*$)`).MatchString(body) { + t.Fatalf("%s has no typed store for result field %d (%s):\n%s", name, index, storeType, body) + } + } +} + +func TestCoroStaticPlainCallExecutionConstraints(t *testing.T) { + for _, test := range []struct { + name string + exec coro.ExecFlags + wantErr string + }{ + {name: "thread affine rejected", exec: coro.ThreadAffine, wantErr: "thread-affine"}, + {name: "IRQ unsafe allowed on ordinary G", exec: coro.IRQUnsafe}, + } { + t.Run(test.name, func(t *testing.T) { + const source = `package foo +func Plain() {} +func Root() { Plain() } +` + ssaPkg, _, files := buildGoSSAPkg(t, source) + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + t.Fatal(err) + } + root, plain := ssaPkg.Func("Root"), ssaPkg.Func("Plain") + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapABIV2 + functionIDs.ArchiveReady = true + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: root, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + switch fn { + case root: + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + case plain: + return coro.SSAFunctionPolicy{Exec: test.exec}, nil + default: + return coro.SSAFunctionPolicy{}, nil + } + }, + }) + if err != nil { + t.Fatal(err) + } + var plainCall *ssa.Call + for _, instruction := range root.Blocks[0].Instrs { + call, ok := instruction.(*ssa.Call) + if ok && call.Call.StaticCallee() == plain { + plainCall = call + break + } + } + if plainCall == nil { + t.Fatal("Root has no static Plain call") + } + _, _, resolveErr := resolveCoroStaticPlainCall(plan, plainCall) + if test.wantErr == "" && resolveErr != nil { + t.Fatalf("ordinary-G direct plain target rejected: %v", resolveErr) + } + if test.wantErr != "" && (resolveErr == nil || !strings.Contains(resolveErr.Error(), test.wantErr)) { + t.Fatalf("direct plain target error = %v, want %q", resolveErr, test.wantErr) + } + if test.wantErr == "" { + rootPlan, ok := plan.FunctionPlan(root) + if !ok { + t.Fatal("Root has no function plan") + } + if err := validateCoroPhysicalABI(root, rootPlan, plan, true, true); err != nil { + t.Fatalf("ordinary-G IRQ-unsafe CFG preflight rejected: %v", err) + } + } + }) + } +} + +func TestCoroPreemptiveStraightLineBudgetPhysicalABIV1(t *testing.T) { + source := "package foo\nfunc Heavy(value uint32) uint32 {\n" + + strings.Repeat("value++\n", 150) + + "return value\n}\n" + prog, ssaPkg, files, universe, plan := prepareCoroPreemptTestPlan( + t, + source, + []coroRootFactoryTestRoot{{name: "Heavy", demand: coro.AsyncDemand}}, + nil, + 16, + ) + defer prog.Dispose() + heavy := ssaPkg.Func("Heavy") + heavyPlan, ok := plan.FunctionPlan(heavy) + if !ok || !heavyPlan.Exec.Contains(coro.NeedsPreempt) || !heavyPlan.Effect.Contains(coro.YieldOnly) { + t.Fatalf("Heavy plan = %+v, present=%t; want instruction-budget preemption", heavyPlan, ok) + } + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroPreemptCompilation(compilation) + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + t.Fatal(err) + } + module := pkg.Module() + defer module.Dispose() + body := requireCoroPhysicalFunction(t, module, "foo.Heavy").String() + if got := strings.Count(body, "call void @"+coroYieldPrepareHookV1); got < 2 { + t.Fatalf("Heavy compiler yield handoffs = %d, want at least two periodic cuts:\n%s", got, body) + } + runCoroABITestPipeline(t, prog, module) +} + +func TestCoroPreemptiveInstructionBudgetBoundary(t *testing.T) { + source := "package foo\nfunc AtLimit(value uint32) uint32 {\n" + + strings.Repeat("value++\n", coroPreemptInstructionBudget-1) + + "return value\n}\nfunc OverLimit(value uint32) uint32 {\n" + + strings.Repeat("value++\n", coroPreemptInstructionBudget) + + "return value\n}\n" + prog, ssaPkg, files, universe, plan := prepareCoroPreemptTestPlan( + t, + source, + []coroRootFactoryTestRoot{ + {name: "AtLimit", demand: coro.AsyncDemand}, + {name: "OverLimit", demand: coro.AsyncDemand}, + }, + nil, + 16, + ) + defer prog.Dispose() + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroPreemptCompilation(compilation) + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + t.Fatal(err) + } + module := pkg.Module() + defer module.Dispose() + if got := strings.Count(requireCoroPhysicalFunction(t, module, "foo.AtLimit").String(), "call i1 @"+coroPreemptPollHookV1); got != 1 { + t.Fatalf("AtLimit preemption polls = %d, want block-zero chain-boundary poll only", got) + } + if got := strings.Count(requireCoroPhysicalFunction(t, module, "foo.OverLimit").String(), "call i1 @"+coroPreemptPollHookV1); got != 2 { + t.Fatalf("OverLimit preemption polls = %d, want block-zero plus one instruction-budget poll", got) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify instruction-budget boundary coroutines: %v\n%s", err, module.String()) + } +} + +func TestCoroNeedsPreemptRequiresRunnableSchedulerABI(t *testing.T) { + const source = `package foo +func Loop(limit uint32) uint32 { + var value uint32 + for value < limit { value++ } + return value +} +` + prog, ssaPkg, files, universe, plan := prepareCoroRootFactoryTestPlan( + t, source, + []coroRootFactoryTestRoot{{name: "Loop", demand: coro.AsyncDemand}}, + nil, + ) + defer prog.Dispose() + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroChildAwaitCompilation(compilation) + if _, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ); err == nil || !strings.Contains(err.Error(), "needs-preempt execution requires the runnable scheduler ABI") { + t.Fatalf("child-await-only preflight error = %v, want runnable-scheduler rejection", err) + } +} + func TestCoroChildAwaitPhysicalABIV1Wasm32(t *testing.T) { llssa.Initialize(llssa.InitAll) prog, pkg := compileCoroChildAwaitPhysicalABI(t, &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}) @@ -901,9 +1410,9 @@ func TestCoroLeafPhysicalABIPreflightRejectsUnsupported(t *testing.T) { want string }{ { - name: "pointer parameter", - source: `package foo; func Leaf(value *int) {}`, - want: "parameter 0 has unsupported type *int", + name: "variadic parameter", + source: `package foo; func Leaf(values ...int) {}`, + want: "variadic coroutine ABI is not implemented", }, { name: "control flow", @@ -941,12 +1450,6 @@ func Leaf(channel chan uint32) uint32 { return <-channel }`, func Leaf(value uint32) uint32 { return value + 1 }`, want: "ABI directive", }, - { - name: "multiple results", - source: `package foo -func Leaf(value uint32) (uint32, uint32) { return value, value }`, - want: "supports at most one result", - }, { name: "shift requires hidden panic check", source: `package foo @@ -1311,6 +1814,82 @@ func Parent(first uint8, second uint32) uint32 { return Child(first, second) + 1 return prog, ssaPkg, files, universe, plan } +func prepareCoroPhysicalValueTransportABI(t *testing.T, target *llssa.Target) ( + llssa.Program, *ssa.Package, []*ast.File, *EmissionUniverse, *coro.SSAPlan, +) { + t.Helper() + const source = `package foo + +type Payload struct { + Ptr *uint32 + Count uintptr + Label string + Bytes []byte + Slots [2]uintptr +} + +func Child(callback func(*uint32), ptr *uint32, value Payload) Payload { + return value +} + +func Parent(ptr *uint32, value Payload) Payload { + return Child(nil, ptr, value) +} + +func Pair(ptr *uint32, count uintptr) (*uint32, uintptr) { + return ptr, count +} +` + ssaPkg, _, files := buildGoSSAPkg(t, source) + var prog llssa.Program + if target == nil { + prog = newLLSSAProg(t) + } else { + prog = newLLSSAProgForTarget(t, target) + } + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerChildAwaitABIV0 + functionIDs.ArchiveReady = true + parent, child, pair := ssaPkg.Func("Parent"), ssaPkg.Func("Child"), ssaPkg.Func("Pair") + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{ + {Function: parent, Demand: coro.AsyncDemand}, + {Function: pair, Demand: coro.AsyncDemand}, + }, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == child || fn == pair { + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + }) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + for name, fn := range map[string]*ssa.Function{"Parent": parent, "Child": child, "Pair": pair} { + function, ok := plan.FunctionPlan(fn) + if !ok || function.Primary != coro.PrimaryCoroutine || function.FuncRep != coro.DirectCoro || function.Demand != coro.AsyncDemand { + prog.Dispose() + t.Fatalf("%s value-transport plan = %+v, present=%t; want async-only direct coroutine", name, function, ok) + } + } + return prog, ssaPkg, files, universe, plan +} + func enableCoroChildAwaitCompilation(compilation *Compilation) { compilation.EnableCoroEntryResolution = true compilation.EnableCoroPhysicalABI = true @@ -1321,6 +1900,12 @@ func enableCoroChildAwaitCompilation(compilation *Compilation) { compilation.FuncRepABI = coro.FuncRepABIV0 } +func enableCoroPreemptCompilation(compilation *Compilation) { + enableCoroChildAwaitCompilation(compilation) + compilation.EnableCoroProgramBootstrapRun = true + compilation.SchedulerABI = coro.SchedulerProgramBootstrapABIV2 +} + func requireCoroPhysicalFunction(t *testing.T, module llvm.Module, sourceName string) llvm.Value { t.Helper() if legacy := module.NamedFunction(sourceName); !legacy.IsNil() { @@ -1496,6 +2081,36 @@ func prepareCoroRootFactoryTestPlan( source string, testRoots []coroRootFactoryTestRoot, yieldOnly []string, +) (llssa.Program, *ssa.Package, []*ast.File, *EmissionUniverse, *coro.SSAPlan) { + return prepareCoroRootFactoryTestPlanWithMaxPlainInstructions(t, source, testRoots, yieldOnly, -1) +} + +func prepareCoroRootFactoryTestPlanWithMaxPlainInstructions( + t *testing.T, + source string, + testRoots []coroRootFactoryTestRoot, + yieldOnly []string, + maxPlainInstructions int, +) (llssa.Program, *ssa.Package, []*ast.File, *EmissionUniverse, *coro.SSAPlan) { + return prepareCoroRootFactoryTestPlanWithScheduler( + t, source, testRoots, yieldOnly, maxPlainInstructions, coro.SchedulerChildAwaitABIV0, + ) +} + +func prepareCoroPreemptTestPlan( + t *testing.T, + source string, + testRoots []coroRootFactoryTestRoot, + yieldOnly []string, + maxPlainInstructions int, +) (llssa.Program, *ssa.Package, []*ast.File, *EmissionUniverse, *coro.SSAPlan) { + return prepareCoroRootFactoryTestPlanWithScheduler( + t, source, testRoots, yieldOnly, maxPlainInstructions, coro.SchedulerProgramBootstrapABIV2, + ) +} + +func prepareCoroProgramInitTestPlan( + t *testing.T, source string, ) (llssa.Program, *ssa.Package, []*ast.File, *EmissionUniverse, *coro.SSAPlan) { t.Helper() ssaPkg, _, files := buildGoSSAPkg(t, source) @@ -1512,7 +2127,64 @@ func prepareCoroRootFactoryTestPlan( } functionIDs := universe.FunctionIDConfig() functionIDs.CoroABI = coro.PhysicalABIV1 - functionIDs.SchedulerABI = coro.SchedulerChildAwaitABIV0 + functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapABIV2 + functionIDs.ArchiveReady = true + packageInit := ssaPkg.Func("init") + yield := ssaPkg.Func("Yield") + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: packageInit, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + switch { + case fn == yield: + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + case fn.Pkg != nil && fn.Pkg.Pkg.Path() == "embed" && fn.Name() == "init": + // The fixture does not compile the standard embed package, but its + // package initializer is an exact frozen no-suspend external edge. + return coro.SSAFunctionPolicy{ + Effect: coro.NoSuspend, External: coro.ExternalKnown, OverrideExternal: true, + }, nil + default: + return coro.SSAFunctionPolicy{}, nil + } + }, + ClassifyElidedCall: func(_ *ssa.Function, call ssa.CallInstruction) (bool, error) { + callee := call.Common().StaticCallee() + return callee != nil && callee.Pkg != nil && callee.Pkg.Pkg.Path() == "unsafe" && callee.Name() == "init", nil + }, + }) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + return prog, ssaPkg, files, universe, plan +} + +func prepareCoroRootFactoryTestPlanWithScheduler( + t *testing.T, + source string, + testRoots []coroRootFactoryTestRoot, + yieldOnly []string, + maxPlainInstructions int, + schedulerABI string, +) (llssa.Program, *ssa.Package, []*ast.File, *EmissionUniverse, *coro.SSAPlan) { + t.Helper() + ssaPkg, _, files := buildGoSSAPkg(t, source) + prog := newLLSSAProg(t) + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = schedulerABI functionIDs.ArchiveReady = true roots := make(coro.Roots, len(testRoots)) for i, root := range testRoots { @@ -1535,7 +2207,7 @@ func prepareCoroRootFactoryTestPlan( plan, err := coro.AnalyzeSSA(ssaPkg.Prog, roots, coro.SSAConfig{ EmissionUniverse: ssaUniverse, FunctionIDs: functionIDs, - MaxPlainInstructions: -1, + MaxPlainInstructions: maxPlainInstructions, ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { if yieldSet[fn] { return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil diff --git a/cl/coro_await.go b/cl/coro_await.go index 38c7dcabfd..3983d229e9 100644 --- a/cl/coro_await.go +++ b/cl/coro_await.go @@ -18,6 +18,7 @@ package cl import ( "fmt" + "go/types" "github.com/goplus/llgo/internal/coro" llssa "github.com/goplus/llgo/ssa" @@ -53,16 +54,23 @@ func resolveCoroStaticAwait(plan *coro.SSAPlan, caller coro.FunctionPlan, call s if !ok || targetPlan.ID != callPlan.Targets[0] { return nil, coro.FunctionPlan{}, fmt.Errorf("direct coroutine target %q has no canonical function plan", callPlan.Targets[0]) } + if err := validateCoroAwaitTarget(caller, targetPlan); err != nil { + return nil, coro.FunctionPlan{}, err + } + return target, targetPlan, nil +} + +func validateCoroAwaitTarget(caller, target coro.FunctionPlan) error { if caller.Emission != coro.EmitCoroutine { - return nil, coro.FunctionPlan{}, fmt.Errorf("caller emission is %s, want coroutine", caller.Emission) + return fmt.Errorf("caller emission is %s, want coroutine", caller.Emission) } - if targetPlan.External != coro.Defined || targetPlan.Emission != coro.EmitCoroutine || targetPlan.FuncRep != coro.DirectCoro || targetPlan.Demand != coro.AsyncDemand { - return nil, coro.FunctionPlan{}, fmt.Errorf( + if target.External != coro.Defined || target.Emission != coro.EmitCoroutine || target.FuncRep != coro.DirectCoro || target.Demand != coro.AsyncDemand { + return fmt.Errorf( "target %q is not an async-only defined direct coroutine (external=%s emission=%s representation=%s demand=%s)", - targetPlan.ID, targetPlan.External, targetPlan.Emission, targetPlan.FuncRep, targetPlan.Demand, + target.ID, target.External, target.Emission, target.FuncRep, target.Demand, ) } - return target, targetPlan, nil + return nil } // tryCompileCoroStaticAwait lowers a source-style synchronous call into one @@ -92,6 +100,31 @@ func (p *context) tryCompileCoroStaticAwait(b llssa.Builder, call *ssa.Call) (ll // Preserve Go's left-to-right argument evaluation before publishing any // child or parent scheduler state. args := p.compileValues(b, call.Call.Args, p.funcKind(call.Call.Value)) + return p.compileCoroTargetAwait(b, callee, args), true +} + +// compileCoroTargetAwait lowers one already-resolved exact managed target. +// args must have been evaluated in source order before this function is called. +// It is shared by source SSA calls and compiler-inserted runtime helper calls. +func (p *context) compileCoroTargetAwait(b llssa.Builder, callee *ssa.Function, args []llssa.Expr) llssa.Expr { + if p.currentCoro == nil || p.compilation == nil || p.compilation.CoroPlan == nil || !p.compilation.EnableCoroChildAwait { + panic("coroutine child await requires an active physical coroutine body") + } + if b.Func != p.fn { + panic("coroutine child await builder does not belong to the active physical coroutine function") + } + callerPlan, ok := p.compilation.CoroPlan.FunctionPlan(p.goFn) + if !ok { + panic("coroutine child await: current function has no compilation plan") + } + targetPlan, ok := p.compilation.CoroPlan.FunctionPlan(callee) + if !ok { + panic("coroutine child await: target has no compilation plan") + } + if err := validateCoroAwaitTarget(callerPlan, targetPlan); err != nil { + panic(fmt.Sprintf("coroutine child await: function %q: %v", callerPlan.ID, err)) + } + entry := p.mustFunctionSymbol(callee) if p.emissionUniverse == nil { panic("coroutine child await requires a prepared emission universe") @@ -124,11 +157,31 @@ func (p *context) tryCompileCoroStaticAwait(b llssa.Builder, call *ssa.Call) (ll } publish := p.pkg.NewFunc(p.currentCoro.abi.awaitPrepareHook, coroAwaitPrepareSignature(), llssa.InC) b.Call(publish.Expr, p.currentCoro.task, p.currentCoro.coro.Handle(), child) - p.currentCoro.coro.Suspend() + p.currentCoro.coro.SuspendCurrentBlock() p.currentCoro.activate(b) - if abi.resultCount == 0 { - return llssa.Nil, true + return p.loadCoroAwaitResult(b, resultSlot, sourceSig.Results()) +} + +// loadCoroAwaitResult reconstructs the exact source call value after the +// scheduler has resumed the parent. Multi-result calls are one SSA tuple value, +// not a result-slot struct: preserving that distinction keeps the ordinary +// Extract lowering and ValuePlan paths identical to a synchronous Go call. +func (p *context) loadCoroAwaitResult(b llssa.Builder, resultSlot llssa.Expr, results *types.Tuple) llssa.Expr { + count := 0 + if results != nil { + count = results.Len() + } + switch count { + case 0: + return llssa.Nil + case 1: + return b.Load(b.FieldAddr(resultSlot, 0)) + default: + fields := make([]llssa.Expr, results.Len()) + for i := range fields { + fields[i] = b.Load(b.FieldAddr(resultSlot, i)) + } + return b.Aggregate(p.prog.Type(results, llssa.InGo), fields...) } - return b.Load(b.FieldAddr(resultSlot, 0)), true } diff --git a/cl/coro_entry.go b/cl/coro_entry.go index 73bbed8c4f..fdaf09417c 100644 --- a/cl/coro_entry.go +++ b/cl/coro_entry.go @@ -40,8 +40,10 @@ type plannedFunctionSymbol struct { planned bool physical bool childAwait bool + programRun bool plainDispatch bool coroPlan *coro.SSAPlan + emission *EmissionUniverse } // resolveFunctionSymbol is shared by function definitions and declarations so @@ -85,8 +87,10 @@ func (p *context) resolveFunctionSymbol(fn *ssa.Function) (plannedFunctionSymbol entry.planned = true entry.physical = p.compilation.EnableCoroPhysicalABI entry.childAwait = p.compilation.EnableCoroChildAwait + entry.programRun = p.compilation.EnableCoroProgramBootstrapRun entry.plainDispatch = p.compilation.EnableCoroPlainDispatch entry.coroPlan = p.compilation.CoroPlan + entry.emission = p.compilation.EmissionUniverse if p.compilation.CoroPlan.IgnoresBody(fn) { return entry, fmt.Errorf("coroutine entry resolution: Go-emitted function %q has an ignored SSA body", plan.ID) } @@ -174,7 +178,10 @@ func (e plannedFunctionSymbol) checkSupported() error { if !e.physical { return fmt.Errorf("coroutine emission %q requires coroutine physical ABI lowering", e.plan.ID) } - return validateCoroPhysicalABI(e.function, e.plan, e.coroPlan, e.childAwait) + if err := validateCoroPhysicalFunctionValueABI(e.plan, e.function.Signature, e.plainDispatch); err != nil { + return err + } + return validateCoroPhysicalABIWithUniverse(e.function, e.plan, e.coroPlan, e.emission, e.childAwait, e.programRun) } if e.plan.Emission == coro.EmitExternal && e.plan.FuncRep == coro.DirectCoro { return fmt.Errorf("external coroutine emission %q requires coroutine physical ABI lowering", e.plan.ID) @@ -244,8 +251,10 @@ func (c *Compilation) preflightCoroPlan() error { planned: true, physical: c.EnableCoroPhysicalABI, childAwait: c.EnableCoroChildAwait, + programRun: c.EnableCoroProgramBootstrapRun, plainDispatch: c.EnableCoroPlainDispatch, coroPlan: c.CoroPlan, + emission: c.EmissionUniverse, } if err := entry.checkSupported(); err != nil { c.coroPreflightErr = err @@ -256,6 +265,9 @@ func (c *Compilation) preflightCoroPlan() error { if err == nil { err = validateCoroLeafPhysicalSignature(function.Plan, sig) } + if err == nil { + err = validateCoroPhysicalFunctionValueABI(function.Plan, sig, c.EnableCoroPlainDispatch) + } if err != nil { c.coroPreflightErr = err return diff --git a/cl/coro_entry_test.go b/cl/coro_entry_test.go index 29df742ca6..2d0a594ec4 100644 --- a/cl/coro_entry_test.go +++ b/cl/coro_entry_test.go @@ -360,8 +360,8 @@ func Complex(ch chan int) int { prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, PackageOptions{Compilation: compilation}, ) - if err == nil || !strings.Contains(err.Error(), "requires exactly one basic block") { - t.Fatalf("demanded complex preflight = %v, %v; want fail-closed CFG diagnostic", got, err) + if err == nil || !strings.Contains(err.Error(), "unsupported unary operation") { + t.Fatalf("demanded complex preflight = %v, %v; want fail-closed unsupported channel-receive instruction diagnostic", got, err) } if got != nil { t.Fatal("demanded complex preflight returned a partial package") diff --git a/cl/coro_lowered_call.go b/cl/coro_lowered_call.go new file mode 100644 index 0000000000..31b392a453 --- /dev/null +++ b/cl/coro_lowered_call.go @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "fmt" + "go/types" + + "github.com/goplus/llgo/internal/coro" + llssa "github.com/goplus/llgo/ssa" +) + +// resolveCoroLoweredRuntimeCall replaces one rtFunc call with the exact +// physical entry frozen for the current SSA owner. Missing or divergent input +// is a compiler-plan error: falling back to the legacy symbol would recreate a +// hidden call edge after the whole-program fixed point was sealed. +func (p *context) resolveCoroLoweredRuntimeCall(b llssa.Builder, helper string, marker llssa.Expr, args []llssa.Expr) (llssa.Expr, bool) { + if p.compilation == nil || !p.compilation.EnableCoroEntryResolution { + return llssa.Nil, false + } + if p.emissionUniverse == nil || !p.emissionUniverse.CompleteRuntimeABI() { + // Isolated package/report tests do not carry the production runtime ABI + // and must keep the legacy rtFunc marker. internal/build always prepares + // a complete universe for active entry resolution, where every missing + // owner-scoped mapping remains a hard compiler-plan error below. + return llssa.Nil, false + } + if p.goFn == nil || p.emissionUniverse == nil || p.compilation.CoroPlan == nil { + panic("coroutine lowered runtime call requires an exact owner, emission universe, and SSA plan") + } + if b.Func != p.fn { + panic(fmt.Errorf("coroutine lowered runtime call %q in %q escaped into another LLVM function", helper, p.goFn.Name())) + } + + target, ok, err := p.emissionUniverse.ResolveCoroLoweredCall(p.goFn, helper) + if err != nil { + panic(fmt.Errorf("coroutine lowered runtime call %q in %q: %w", helper, p.goFn.Name(), err)) + } + if !ok || target == nil { + panic(fmt.Errorf("coroutine lowered runtime call %q in %q is absent from the frozen emission universe", helper, p.goFn.Name())) + } + plannedTarget, planned := p.compilation.CoroPlan.ResolveLoweredCall(p.goFn, helper) + if !planned || plannedTarget != target { + panic(fmt.Errorf("coroutine lowered runtime call %q in %q disagrees between the frozen emission universe and SSA plan", helper, p.goFn.Name())) + } + targetPlan, planned := p.compilation.CoroPlan.FunctionPlan(target) + if !planned { + panic(fmt.Errorf("coroutine lowered runtime call %q in %q targets an unplanned function", helper, p.goFn.Name())) + } + sourceSig, err := p.emissionUniverse.coroPhysicalSourceSignature(target) + if err != nil { + panic(fmt.Errorf("coroutine lowered runtime call %q in %q: derive target %q signature: %w", helper, p.goFn.Name(), targetPlan.ID, err)) + } + markerSig, ok := types.Unalias(marker.RawType()).(*types.Signature) + if !ok || !types.Identical(markerSig, sourceSig) { + panic(fmt.Errorf("coroutine lowered runtime call %q in %q target %q has a different effective source signature", helper, p.goFn.Name(), targetPlan.ID)) + } + + switch targetPlan.Emission { + case coro.EmitPlain: + if targetPlan.External != coro.Defined || targetPlan.Demand == coro.NoDemand || targetPlan.Effect.MaySuspend() || targetPlan.FuncRep == coro.DirectCoro { + panic(fmt.Errorf("coroutine lowered runtime call %q in %q cannot call suspending target %q through a plain entry", helper, p.goFn.Name(), targetPlan.ID)) + } + fn, _, kind := p.compileFunction(target) + if fn == nil || kind != goFunc { + panic(fmt.Errorf("coroutine lowered runtime call %q in %q target %q did not resolve to a Go entry", helper, p.goFn.Name(), targetPlan.ID)) + } + return b.Call(fn.Expr, args...), true + case coro.EmitCoroutine: + if targetPlan.Exec&coro.MayUnwind != 0 { + panic(fmt.Errorf("coroutine lowered runtime call %q in %q target %q may unwind, but child-frame panic propagation is not implemented", helper, p.goFn.Name(), targetPlan.ID)) + } + return p.compileCoroTargetAwait(b, target, args), true + case coro.EmitNone: + panic(fmt.Errorf("coroutine lowered runtime call %q in %q targets non-emitted function %q", helper, p.goFn.Name(), targetPlan.ID)) + case coro.EmitExternal: + panic(fmt.Errorf("coroutine lowered runtime call %q in %q requires an unimplemented external helper adapter for %q", helper, p.goFn.Name(), targetPlan.ID)) + default: + panic(fmt.Errorf("coroutine lowered runtime call %q in %q targets function %q with invalid emission %d", helper, p.goFn.Name(), targetPlan.ID, uint8(targetPlan.Emission))) + } +} diff --git a/cl/coro_park_test.go b/cl/coro_park_test.go new file mode 100644 index 0000000000..4073e6d4b3 --- /dev/null +++ b/cl/coro_park_test.go @@ -0,0 +1,203 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "bytes" + "regexp" + "strings" + "testing" + + "github.com/goplus/llgo/internal/coro" + "github.com/goplus/llgo/internal/goembed" + llssa "github.com/goplus/llgo/ssa" + "github.com/xgo-dev/llvm" + "golang.org/x/tools/go/ssa" +) + +const coroParkTestSource = `package foo + +import _ "unsafe" + +type WaitToken struct { word uint32 } +type WaitTicket uint32 + +//go:linkname park llgo.coroPark +func park(token *WaitToken, ticket WaitTicket) + +func Root(token *WaitToken, ticket WaitTicket) uint32 { + before := uint32(ticket) + 7 + park(token, ticket) + return before + uint32(ticket) +} +` + +func TestCoroParkCurrentFrameNativeAndWasm32(t *testing.T) { + tests := []struct { + name string + target *llssa.Target + }{ + {name: "native"}, + {name: "wasm32", target: &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + prog, pkg, plan, root, parkCall := compileCoroParkFixture(t, test.target) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + rootPlan, ok := plan.FunctionPlan(root) + if !ok || rootPlan.Emission != coro.EmitCoroutine || rootPlan.FuncRep != coro.DirectCoro || + !rootPlan.DeclaredEffect.Contains(coro.MayPark) || !rootPlan.LocalEffect.Contains(coro.MayPark) || + !rootPlan.Effect.Contains(coro.MayPark) { + t.Fatalf("Root plan = %+v, present=%t; want one may-park coroutine primary", rootPlan, ok) + } + if !plan.ElidesCall(parkCall) { + t.Fatal("coroPark declaration call is not frozen as a frontend-elided intrinsic site") + } + if _, ok := plan.CallPlan(parkCall); ok { + t.Fatal("coroPark declaration unexpectedly retained a managed CallPlan") + } + + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify park coroutine before CoroSplit: %v\n%s", err, module.String()) + } + body := requireCoroPhysicalFunction(t, module, "foo.Root").String() + if got := strings.Count(body, "call i8 @llvm.coro.suspend"); got != 3 { + t.Fatalf("Root coro.suspend calls = %d, want initial + park + final:\n%s", got, body) + } + if strings.Contains(body, "@foo.park") || strings.Contains(body, "@llgo.coroPark") { + t.Fatalf("structured park leaked an ordinary sync helper call:\n%s", body) + } + stateAndHook := regexp.MustCompile( + `(?s)store i16 4,.*store i16 3,.*store i32 1,.*call void @` + regexp.QuoteMeta(coroParkPrepareHookV1) + + `\(ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^,]+, i32 [^)]+\)`, + ) + if !stateAndHook.MatchString(body) { + t.Fatalf("Root does not publish Park/Suspended/stateID=1 before the exact v1 hook:\n%s", body) + } + hook := strings.Index(body, "call void @"+coroParkPrepareHookV1) + parkSuspendRelative := strings.Index(body[hook:], "call i8 @llvm.coro.suspend") + if hook < 0 || parkSuspendRelative < 0 { + t.Fatalf("Root has no park hook followed by a caller-frame suspend:\n%s", body) + } + parkSuspend := hook + parkSuspendRelative + activate := regexp.MustCompile(`(?s)store i16 0,.*store i16 2,`).FindStringIndex(body[parkSuspend:]) + if activate == nil { + t.Fatalf("Root does not reactivate its exact frame after resume:\n%s", body) + } + + runCoroABITestPipeline(t, prog, module) + resume := module.NamedFunction("foo.Root$coro.resume") + if resume.IsNil() || !strings.Contains(resume.String(), "call void @"+coroParkPrepareHookV1) { + t.Fatalf("CoroSplit lost the park handoff in Root.resume:\n%s", module.String()) + } + for _, intrinsic := range []string{"llvm.coro.id", "llvm.coro.begin", "llvm.coro.suspend", "llvm.coro.end"} { + if hasLLVMCall(module.String(), intrinsic) { + t.Fatalf("post-split park module still calls %s:\n%s", intrinsic, module.String()) + } + } + object, err := prog.TargetMachine().EmitToMemoryBuffer(module, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit post-CoroSplit park object: %v\n%s", err, module.String()) + } + defer object.Dispose() + if len(object.Bytes()) == 0 || !bytes.Contains(object.Bytes(), []byte(coroParkPrepareHookV1)) { + t.Fatalf("post-CoroSplit object lost unresolved park ABI symbol %q", coroParkPrepareHookV1) + } + }) + } +} + +func compileCoroParkFixture(t *testing.T, target *llssa.Target) ( + llssa.Program, llssa.Package, *coro.SSAPlan, *ssa.Function, *ssa.Call, +) { + t.Helper() + ssaPkg, _, files := buildGoSSAPkg(t, coroParkTestSource) + var prog llssa.Program + if target == nil { + prog = newLLSSAProg(t) + } else { + prog = newLLSSAProgForTarget(t, target) + } + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + root := ssaPkg.Func("Root") + var parkCall *ssa.Call + for _, block := range root.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(*ssa.Call) + if !ok || call.Call.StaticCallee() == nil || call.Call.StaticCallee().Name() != "park" { + continue + } + parkCall = call + } + } + if parkCall == nil { + prog.Dispose() + t.Fatal("fixture has no direct park call") + } + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerChildAwaitABIV0 + functionIDs.ArchiveReady = true + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: root, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == root { + return coro.SSAFunctionPolicy{Effect: coro.MayPark}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + ClassifyElidedCall: func(_ *ssa.Function, call ssa.CallInstruction) (bool, error) { + callee := call.Common().StaticCallee() + if callee != nil && callee.Pkg != nil && callee.Pkg.Pkg.Path() == "unsafe" && callee.Name() == "init" { + return true, nil + } + semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call) + return intrinsic && semantics.ElidesManagedCall(), err + }, + }) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroChildAwaitCompilation(compilation) + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + return prog, pkg, plan, root, parkCall +} diff --git a/cl/coro_pure_ssa.go b/cl/coro_pure_ssa.go new file mode 100644 index 0000000000..8c19ad9e49 --- /dev/null +++ b/cl/coro_pure_ssa.go @@ -0,0 +1,588 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "fmt" + "go/constant" + "go/token" + "go/types" + "strings" + + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +// coroPhysicalPureSSAAudit is the deliberately small proof boundary for SSA +// operations that remain ordinary LLVM values across a coro suspend. It is not +// a general instruction allowlist. Every accepted case below mirrors the +// corresponding compileInstr/compileInstrOrValue and LLSSA Builder lowering. +// An operation that can call a runtime helper, perform dynamic dispatch, or +// introduce a new panic edge is rejected here even when that helper currently +// happens to be classified NoSuspend. +// +// PhysicalABIV1's current frame allocator profiles are conservative or +// non-collecting. Pointer/interface/slice values may therefore live in the LLVM +// coroutine frame, but this slice does not claim a precise frame root map or a +// moving-GC write barrier. A future precise collector must add those two ABI +// capabilities before enabling the same local-frame operations for that +// profile. +type coroPhysicalPureSSAAudit struct { + universe *EmissionUniverse + ctx *context +} + +func newCoroPhysicalPureSSAAudit(universe *EmissionUniverse, fn *ssa.Function) (*coroPhysicalPureSSAAudit, error) { + audit := &coroPhysicalPureSSAAudit{universe: universe} + if universe == nil { + // Structural unit tests may call the validator directly. Active + // Compilation paths always supply their prepared emission universe. + return audit, nil + } + if fn == nil { + return nil, fmt.Errorf("nil function") + } + if canonical := universe.canonicalAlias(fn); canonical == nil || canonical != fn { + return nil, fmt.Errorf("function %q is not the exact canonical emission owner", fn.Name()) + } + if _, frozen := universe.required[fn]; !frozen { + return nil, fmt.Errorf("function %q is outside the prepared emission universe", fn.Name()) + } + owner := universe.ownerOf(fn) + ctx, err := universe.functionABIContext(fn, owner) + if err != nil { + return nil, err + } + audit.ctx = ctx + return audit, nil +} + +func (a *coroPhysicalPureSSAAudit) validate(instr ssa.Instruction) (handled bool, reason string) { + switch instr := instr.(type) { + case *ssa.Alloc: + return true, a.validateAlloc(instr) + case *ssa.FieldAddr: + return true, a.validateFieldAddr(instr) + case *ssa.IndexAddr: + return true, a.validateIndexAddr(instr) + case *ssa.Index: + return true, a.validateIndex(instr) + case *ssa.Slice: + return true, a.validateSlice(instr) + case *ssa.Extract: + return true, a.validateExtract(instr) + case *ssa.Field: + return true, a.validateField(instr) + case *ssa.MakeInterface: + return true, a.validateMakeInterface(instr) + case *ssa.ChangeType: + return true, a.validateChangeType(instr) + case *ssa.Convert: + return true, a.validateConvert(instr) + case *ssa.Phi: + return true, a.validatePhi(instr) + case *ssa.BinOp: + return true, a.validateBinOp(instr) + case *ssa.UnOp: + if instr.Op == token.MUL || instr.Op == token.SUB || instr.Op == token.XOR || instr.Op == token.NOT { + return true, a.validateUnOp(instr) + } + case *ssa.Store: + return true, a.validateStore(instr) + case *ssa.Call: + if _, builtin := instr.Call.Value.(*ssa.Builtin); builtin { + return true, a.validateBuiltin(instr) + } + } + return false, "" +} + +func (a *coroPhysicalPureSSAAudit) validateAlloc(alloc *ssa.Alloc) string { + if alloc == nil || alloc.Heap { + return "heap allocation requires managed allocation and coroutine GC-root lowering" + } + if a.ctx != nil && (a.ctx.skipSyntheticMakeSliceAlloc(alloc) || isEmissionVargsAlloc(a.ctx, alloc)) { + return "synthetic slice/varargs allocation belongs to a non-pure enclosing lowering" + } + pointer, ok := types.Unalias(a.typeOf(alloc.Type())).Underlying().(*types.Pointer) + if !ok { + return "local allocation does not have a pointer type" + } + if err := validateCoroPhysicalSSAValueType(pointer.Elem()); err != nil { + return "local allocation has unsupported value type: " + err.Error() + } + return a.requireNoRuntimeHelpers(alloc) +} + +func (a *coroPhysicalPureSSAAudit) validateFieldAddr(field *ssa.FieldAddr) string { + if field == nil { + return "nil field address" + } + if _, reason := a.stableAddress(field, make(map[ssa.Value]bool)); reason != "" { + return reason + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(field.Type())); err != nil { + return "field address has unsupported type: " + err.Error() + } + return a.requireNoRuntimeHelpers(field) +} + +func (a *coroPhysicalPureSSAAudit) validateIndexAddr(index *ssa.IndexAddr) string { + if index == nil { + return "nil index address" + } + if _, reason := a.stableAddress(index, make(map[ssa.Value]bool)); reason != "" { + return reason + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(index.Type())); err != nil { + return "index address has unsupported type: " + err.Error() + } + return a.requireNoRuntimeHelpers(index) +} + +func (a *coroPhysicalPureSSAAudit) validateIndex(index *ssa.Index) string { + if index == nil || index.X == nil || index.Index == nil { + return "incomplete index operation" + } + array, ok := types.Unalias(a.typeOf(index.X.Type())).Underlying().(*types.Array) + if !ok || !coroConstantIndexInBounds(index.Index, array.Len()) { + return "index may panic; pure coroutine indexing requires a compile-time in-range fixed-array index" + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(index.Type())); err != nil { + return "array index has unsupported result type: " + err.Error() + } + return a.requireNoRuntimeHelpers(index) +} + +func (a *coroPhysicalPureSSAAudit) validateSlice(slice *ssa.Slice) string { + if slice == nil || slice.X == nil || slice.Low != nil || slice.High != nil || slice.Max != nil { + return "slice bounds require runtime validation; only a complete fixed-array view is pure" + } + pointer, ok := types.Unalias(a.typeOf(slice.X.Type())).Underlying().(*types.Pointer) + if !ok { + return "pure slice view requires a pointer to a fixed array" + } + if _, ok := types.Unalias(pointer.Elem()).Underlying().(*types.Array); !ok { + return "pure slice view requires a pointer to a fixed array" + } + if _, reason := a.stableAddress(slice.X, make(map[ssa.Value]bool)); reason != "" { + return "slice base: " + reason + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(slice.Type())); err != nil { + return "slice view has unsupported type: " + err.Error() + } + return a.requireNoRuntimeHelpers(slice) +} + +func (a *coroPhysicalPureSSAAudit) validateExtract(extract *ssa.Extract) string { + if extract == nil || extract.Tuple == nil { + return "incomplete tuple extract" + } + tuple, ok := types.Unalias(a.typeOf(extract.Tuple.Type())).Underlying().(*types.Tuple) + if !ok || extract.Index < 0 || extract.Index >= tuple.Len() { + return "tuple extract index is outside its frozen aggregate shape" + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(extract.Type())); err != nil { + return "tuple extract has unsupported result type: " + err.Error() + } + return a.requireNoRuntimeHelpers(extract) +} + +func (a *coroPhysicalPureSSAAudit) validateField(field *ssa.Field) string { + if field == nil || field.X == nil { + return "incomplete aggregate field extraction" + } + structure, ok := types.Unalias(a.typeOf(field.X.Type())).Underlying().(*types.Struct) + if !ok || field.Field < 0 || field.Field >= structure.NumFields() { + return "aggregate field index is outside its frozen struct shape" + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(field.Type())); err != nil { + return "aggregate field has unsupported result type: " + err.Error() + } + return a.requireNoRuntimeHelpers(field) +} + +func (a *coroPhysicalPureSSAAudit) validateMakeInterface(box *ssa.MakeInterface) string { + if box == nil || box.X == nil { + return "incomplete interface construction" + } + target, ok := types.Unalias(a.typeOf(box.Type())).Underlying().(*types.Interface) + if !ok { + return "MakeInterface target is not an interface" + } + target.Complete() + if !target.Empty() { + return "non-empty interface construction requires itab/runtime lowering" + } + source := a.typeOf(box.X.Type()) + if coroPhysicalTypeContainsFunctionValue(source, make(map[types.Type]bool)) { + return "boxing a function value requires canonical dynamic-dispatch descriptor validation" + } + if !emissionDirectIfaceType(source) { + return "interface construction requires managed backing allocation for this value representation" + } + if err := validateCoroPhysicalSSAValueType(source); err != nil { + return "interface payload has unsupported type: " + err.Error() + } + return a.requireNoRuntimeHelpers(box) +} + +func (a *coroPhysicalPureSSAAudit) validateChangeType(change *ssa.ChangeType) string { + if change == nil || change.X == nil { + return "incomplete value-preserving type change" + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(change.X.Type())); err != nil { + return "type-change source is unsupported: " + err.Error() + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(change.Type())); err != nil { + return "type-change result is unsupported: " + err.Error() + } + return a.requireNoRuntimeHelpers(change) +} + +func (a *coroPhysicalPureSSAAudit) validateConvert(convert *ssa.Convert) string { + if convert == nil || convert.X == nil { + return "incomplete conversion" + } + source, target := a.typeOf(convert.X.Type()), a.typeOf(convert.Type()) + if !coroPureConversion(source, target) { + return "conversion may allocate or call the runtime; pure coroutine conversion supports only numeric and pointer/unsafe-pointer representations" + } + if err := validateCoroPhysicalSSAValueType(source); err != nil { + return "conversion source is unsupported: " + err.Error() + } + if err := validateCoroPhysicalSSAValueType(target); err != nil { + return "conversion result is unsupported: " + err.Error() + } + return a.requireNoRuntimeHelpers(convert) +} + +func (a *coroPhysicalPureSSAAudit) validatePhi(phi *ssa.Phi) string { + if phi == nil { + return "nil phi" + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(phi.Type())); err != nil { + return "phi has unsupported value type: " + err.Error() + } + return a.requireNoRuntimeHelpers(phi) +} + +func (a *coroPhysicalPureSSAAudit) validateBinOp(op *ssa.BinOp) string { + if op == nil || op.X == nil || op.Y == nil { + return "incomplete binary operation" + } + if op.Op == token.QUO || op.Op == token.REM || op.Op == token.SHL || op.Op == token.SHR || + !coroPureBasicScalar(a.typeOf(op.Type())) || !coroPureBasicScalar(a.typeOf(op.X.Type())) || !coroPureBasicScalar(a.typeOf(op.Y.Type())) { + return "potentially panicking or non-scalar binary operation" + } + return a.requireNoRuntimeHelpers(op) +} + +func (a *coroPhysicalPureSSAAudit) validateUnOp(op *ssa.UnOp) string { + if op == nil || op.X == nil { + return "incomplete unary operation" + } + if op.Op != token.MUL { + if !coroPureBasicScalar(a.typeOf(op.Type())) { + return "unsupported unary operation" + } + return a.requireNoRuntimeHelpers(op) + } + if _, reason := a.stableAddress(op.X, make(map[ssa.Value]bool)); reason != "" { + return "typed load: " + reason + } + if !a.nonZeroPhysicalType(op.Type()) { + return "zero-sized typed load lowers through an explicit nil-check helper" + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(op.Type())); err != nil { + return "typed load has unsupported value type: " + err.Error() + } + return a.requireNoRuntimeHelpers(op) +} + +func (a *coroPhysicalPureSSAAudit) validateStore(store *ssa.Store) string { + if store == nil || store.Addr == nil || store.Val == nil { + return "incomplete typed store" + } + root, reason := a.stableAddress(store.Addr, make(map[ssa.Value]bool)) + if reason != "" { + return "typed store: " + reason + } + pointer, ok := types.Unalias(a.typeOf(store.Addr.Type())).Underlying().(*types.Pointer) + if !ok || !types.Identical(pointer.Elem(), a.typeOf(store.Val.Type())) { + return "typed store address/value types do not match" + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(store.Val.Type())); err != nil { + return "typed store has unsupported value type: " + err.Error() + } + if root == coroPhysicalAddressGlobal && coroTypeContainsGCPointer(a.typeOf(store.Val.Type()), make(map[types.Type]bool)) { + return "global typed store of a pointer-containing value requires explicit write-barrier lowering" + } + // A pointer-containing local store is accepted only under PhysicalABIV1's + // current conservative/non-collecting frame profiles described above. It is + // not evidence that precise frame maps or barriers have been implemented. + return a.requireNoRuntimeHelpers(store) +} + +func (a *coroPhysicalPureSSAAudit) validateBuiltin(call *ssa.Call) string { + if call == nil || call.Call.Value == nil || len(call.Call.Args) != 1 { + return "unsupported builtin call in pure coroutine body" + } + builtin, ok := call.Call.Value.(*ssa.Builtin) + if !ok { + return "dynamic/non-builtin call is outside pure SSA lowering" + } + operand := types.Unalias(a.typeOf(call.Call.Args[0].Type())).Underlying() + switch builtin.Name() { + case "len": + switch operand.(type) { + case *types.Slice, *types.Basic: + if basic, ok := operand.(*types.Basic); ok && basic.Kind() != types.String { + return "len builtin is pure here only for slices and strings" + } + default: + return "len builtin is pure here only for slices and strings" + } + case "cap": + if _, ok := operand.(*types.Slice); !ok { + return "cap builtin is pure here only for slices" + } + default: + return fmt.Sprintf("builtin %q is outside the pure coroutine lowering slice", builtin.Name()) + } + if err := validateCoroPhysicalSSAValueType(a.typeOf(call.Type())); err != nil { + return "builtin result has unsupported type: " + err.Error() + } + return a.requireNoRuntimeHelpers(call) +} + +type coroPhysicalAddressRoot uint8 + +const ( + coroPhysicalAddressInvalid coroPhysicalAddressRoot = iota + coroPhysicalAddressLocal + coroPhysicalAddressGlobal +) + +// stableAddress accepts only statically non-nil storage owned by the current +// frame or package. Parameter/heap/foreign pointers remain fail-closed even if +// a particular host would merely trap on nil. +func (a *coroPhysicalPureSSAAudit) stableAddress(value ssa.Value, visiting map[ssa.Value]bool) (coroPhysicalAddressRoot, string) { + if value == nil { + return coroPhysicalAddressInvalid, "nil address" + } + if visiting[value] { + return coroPhysicalAddressInvalid, "cyclic address expression" + } + visiting[value] = true + defer delete(visiting, value) + switch value := value.(type) { + case *ssa.Global: + if _, ok := types.Unalias(a.typeOf(value.Type())).Underlying().(*types.Pointer); !ok { + return coroPhysicalAddressInvalid, "global address does not have pointer type" + } + return coroPhysicalAddressGlobal, "" + case *ssa.Alloc: + if value.Heap { + return coroPhysicalAddressInvalid, "heap allocation requires managed allocation/root lowering" + } + if a.ctx != nil && (a.ctx.skipSyntheticMakeSliceAlloc(value) || isEmissionVargsAlloc(a.ctx, value)) { + return coroPhysicalAddressInvalid, "synthetic slice/varargs storage is not a standalone local address" + } + return coroPhysicalAddressLocal, "" + case *ssa.FieldAddr: + pointer, ok := types.Unalias(a.typeOf(value.X.Type())).Underlying().(*types.Pointer) + if !ok { + return coroPhysicalAddressInvalid, "field base is not a pointer" + } + structure, ok := types.Unalias(pointer.Elem()).Underlying().(*types.Struct) + if !ok || value.Field < 0 || value.Field >= structure.NumFields() { + return coroPhysicalAddressInvalid, "field address is outside its frozen struct shape" + } + return a.stableAddress(value.X, visiting) + case *ssa.IndexAddr: + pointer, ok := types.Unalias(a.typeOf(value.X.Type())).Underlying().(*types.Pointer) + if !ok { + return coroPhysicalAddressInvalid, "index base is not a fixed-array pointer" + } + array, ok := types.Unalias(pointer.Elem()).Underlying().(*types.Array) + if !ok || !coroConstantIndexInBounds(value.Index, array.Len()) { + return coroPhysicalAddressInvalid, "index may panic; address indexing requires a compile-time in-range fixed-array index" + } + return a.stableAddress(value.X, visiting) + default: + return coroPhysicalAddressInvalid, fmt.Sprintf("address root %T is not statically non-nil local/global storage", value) + } +} + +func (a *coroPhysicalPureSSAAudit) requireNoRuntimeHelpers(instr ssa.Instruction) string { + if a == nil || a.ctx == nil || a.universe == nil { + return "" + } + helpers := a.universe.loweredRuntimeHelpers(a.ctx, instr) + if len(helpers) == 0 { + return "" + } + return "operation lowers through managed runtime helper(s) " + strings.Join(helpers, ", ") +} + +func (a *coroPhysicalPureSSAAudit) typeOf(typ types.Type) types.Type { + if typ == nil || a == nil || a.ctx == nil { + return typ + } + return a.ctx.patchType(typ) +} + +func (a *coroPhysicalPureSSAAudit) nonZeroPhysicalType(typ types.Type) bool { + if typ == nil { + return false + } + if a != nil && a.ctx != nil { + return a.ctx.prog.SizeOf(a.ctx.type_(typ, llssa.InGo)) != 0 + } + return coroTypeDefinitelyNonZero(typ, make(map[types.Type]bool)) +} + +func validateCoroPhysicalSSAValueType(typ types.Type) error { + if typ == nil { + return fmt.Errorf("nil type") + } + if tuple, ok := types.Unalias(typ).Underlying().(*types.Tuple); ok { + for i := 0; i < tuple.Len(); i++ { + if err := validateCoroPhysicalValueType(tuple.At(i).Type(), make(map[types.Type]bool)); err != nil { + return fmt.Errorf("tuple field %d: %w", i, err) + } + } + return nil + } + return validateCoroPhysicalValueType(typ, make(map[types.Type]bool)) +} + +func coroConstantIndexInBounds(index ssa.Value, bound int64) bool { + if index == nil || bound < 0 { + return false + } + value, ok := index.(*ssa.Const) + if !ok || value.Value == nil { + return false + } + basic, ok := types.Unalias(value.Type()).Underlying().(*types.Basic) + if !ok || basic.Info()&types.IsInteger == 0 { + return false + } + if basic.Info()&types.IsUnsigned == 0 && constant.Sign(value.Value) < 0 { + return false + } + integer, exact := constant.Uint64Val(value.Value) + return exact && integer < uint64(bound) +} + +func coroPureBasicScalar(typ types.Type) bool { + basic, ok := types.Unalias(typ).Underlying().(*types.Basic) + if !ok { + return false + } + return basic.Info()&(types.IsBoolean|types.IsInteger|types.IsFloat) != 0 +} + +func coroPureConversion(source, target types.Type) bool { + if source == nil || target == nil { + return false + } + sourceUnderlying := types.Unalias(source).Underlying() + targetUnderlying := types.Unalias(target).Underlying() + if types.Identical(sourceUnderlying, targetUnderlying) { + return true + } + sourceBasic, sourceIsBasic := sourceUnderlying.(*types.Basic) + targetBasic, targetIsBasic := targetUnderlying.(*types.Basic) + if sourceIsBasic && targetIsBasic { + if sourceBasic.Kind() == types.String || targetBasic.Kind() == types.String { + return false + } + sourceNumeric := sourceBasic.Info()&(types.IsInteger|types.IsFloat|types.IsComplex) != 0 + targetNumeric := targetBasic.Info()&(types.IsInteger|types.IsFloat|types.IsComplex) != 0 + if sourceNumeric && targetNumeric { + return true + } + return (sourceBasic.Kind() == types.UnsafePointer && targetBasic.Kind() == types.Uintptr) || + (sourceBasic.Kind() == types.Uintptr && targetBasic.Kind() == types.UnsafePointer) + } + _, sourcePointer := sourceUnderlying.(*types.Pointer) + _, targetPointer := targetUnderlying.(*types.Pointer) + if sourcePointer && targetPointer { + return true + } + return (sourcePointer && targetIsBasic && targetBasic.Kind() == types.UnsafePointer) || + (targetPointer && sourceIsBasic && sourceBasic.Kind() == types.UnsafePointer) +} + +func coroTypeContainsGCPointer(typ types.Type, visiting map[types.Type]bool) bool { + if typ == nil { + return false + } + typ = types.Unalias(typ) + if visiting[typ] { + return false + } + visiting[typ] = true + defer delete(visiting, typ) + switch typ := typ.(type) { + case *types.Named: + return coroTypeContainsGCPointer(typ.Underlying(), visiting) + case *types.Pointer, *types.Map, *types.Chan, *types.Signature, *types.Interface, *types.Slice: + return true + case *types.Basic: + return typ.Kind() == types.String || typ.Kind() == types.UnsafePointer + case *types.Array: + return coroTypeContainsGCPointer(typ.Elem(), visiting) + case *types.Struct: + for i := 0; i < typ.NumFields(); i++ { + if coroTypeContainsGCPointer(typ.Field(i).Type(), visiting) { + return true + } + } + } + return false +} + +func coroTypeDefinitelyNonZero(typ types.Type, visiting map[types.Type]bool) bool { + if typ == nil { + return false + } + typ = types.Unalias(typ) + if visiting[typ] { + return false + } + visiting[typ] = true + defer delete(visiting, typ) + switch typ := typ.(type) { + case *types.Named: + return coroTypeDefinitelyNonZero(typ.Underlying(), visiting) + case *types.Basic, *types.Pointer, *types.Map, *types.Chan, *types.Signature, *types.Interface, *types.Slice: + return true + case *types.Array: + return typ.Len() > 0 && coroTypeDefinitelyNonZero(typ.Elem(), visiting) + case *types.Struct: + for i := 0; i < typ.NumFields(); i++ { + if coroTypeDefinitelyNonZero(typ.Field(i).Type(), visiting) { + return true + } + } + } + return false +} diff --git a/cl/coro_pure_ssa_test.go b/cl/coro_pure_ssa_test.go new file mode 100644 index 0000000000..ddb1e09848 --- /dev/null +++ b/cl/coro_pure_ssa_test.go @@ -0,0 +1,356 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "bytes" + "go/ast" + "regexp" + "strings" + "testing" + + "github.com/goplus/llgo/internal/coro" + "github.com/goplus/llgo/internal/goembed" + llssa "github.com/goplus/llgo/ssa" + "github.com/xgo-dev/llvm" + "golang.org/x/tools/go/ssa" +) + +const coroPureSSAFixture = `package foo + +type Pair struct { + A uint32 + B [2]uint32 +} + +type Word uintptr +type NamedPointer *uint32 + +var Global Pair +var Backing [2]uint32 + +func Child(value uint32) uint32 { return value + 1 } +func PairValue() Pair { return Pair{A: 3} } +func ArrayValue() [2]uint32 { return [2]uint32{5, 7} } +func ScalarPair() (uint32, uint32) { return 11, 13 } + +func Aggregate() uint32 { + left, right := ScalarPair() + return PairValue().A + ArrayValue()[1] + left + right +} + +func Root(pointer *uint32) (Pair, any, []uint32, uintptr) { + var local Pair + var values [2]uint32 + local.A = 7 + values[1] = 9 + local.B = values + named := NamedPointer(pointer) + boxed := any(named) + view := Backing[:] + for step := uint32(0); step < 2; step++ { + local.A += step + } + Global = local + next := Child(local.A) + word := Word(next) + global := Global + return local, boxed, view, uintptr(word) + uintptr(global.A) +} +` + +func TestCoroPureSSAPhysicalABIV1NativeAndWasm(t *testing.T) { + llssa.Initialize(llssa.InitAll) + for _, test := range []struct { + name string + target *llssa.Target + }{ + {name: "native"}, + {name: "wasm32", target: &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}}, + } { + t.Run(test.name, func(t *testing.T) { + prog, ssaPkg, files, universe, plan := prepareCoroPureSSATestPlan(t, test.target) + defer prog.Dispose() + assertCoroPureSSAInstructionCoverage(t, ssaPkg) + root := ssaPkg.Func("Root") + rootPlan, ok := plan.FunctionPlan(root) + if !ok || rootPlan.Emission != coro.EmitCoroutine || rootPlan.FuncRep != coro.DirectCoro || + rootPlan.Demand != coro.AsyncDemand || !rootPlan.Exec.Contains(coro.NeedsPreempt) || + !rootPlan.Effect.Contains(coro.AwaitStructured) { + t.Fatalf("Root plan = %+v, present=%t; want preemptible child-await coroutine", rootPlan, ok) + } + + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroPreemptCompilation(compilation) + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + t.Fatal(err) + } + module := pkg.Module() + defer module.Dispose() + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify pure SSA coroutine before CoroSplit: %v\n%s", err, module.String()) + } + rootIR := requireCoroPhysicalFunction(t, module, "foo.Root").String() + aggregateIR := requireCoroPhysicalFunction(t, module, "foo.Aggregate").String() + for _, required := range []string{ + "alloca %foo.Pair", + "getelementptr inbounds %foo.Pair", + "foo.Child$coro", + "call void @" + coroAwaitPrepareHookV1, + "call i1 @" + coroPreemptPollHookV1, + } { + if !strings.Contains(rootIR, required) { + t.Fatalf("Root pure SSA coroutine lacks %q:\n%s", required, rootIR) + } + } + for _, forbidden := range []string{ + "CheckIndexRange", "AssertNilDeref", "AllocU", "AllocZ", "NewSlice2", "NewSlice3Bounds", "NewItab", + } { + if strings.Contains(rootIR, forbidden) { + t.Fatalf("Root pure SSA lowering introduced hidden helper %q:\n%s", forbidden, rootIR) + } + if got := strings.Count(rootIR, "call void @"+coroYieldPrepareHookV1); got < 2 { + t.Fatalf("Root preemption handoffs = %d, want multiple block safepoints after aggregate/interface/slice construction:\n%s", got, rootIR) + } + } + if !strings.Contains(aggregateIR, "foo.PairValue$coro") || + !strings.Contains(aggregateIR, "foo.ArrayValue$coro") || !strings.Contains(aggregateIR, "extractvalue") { + t.Fatalf("Aggregate lost its fixed-array/field/multi-result lowering:\n%s", aggregateIR) + } + + runCoroABITestPipeline(t, prog, module) + resume := module.NamedFunction("foo.Root$coro.resume") + if resume.IsNil() { + t.Fatalf("CoroSplit did not create Root resume entry:\n%s", module.String()) + } + resumeIR := resume.String() + for _, resultStore := range []*regexp.Regexp{ + regexp.MustCompile(`store %foo\.Pair `), + regexp.MustCompile(`store %"[^"]*\.eface" `), + regexp.MustCompile(`store %"[^"]*\.Slice" `), + } { + if !resultStore.MatchString(resumeIR) { + t.Fatalf("value live across await/preempt did not reach its typed result store (%s):\n%s", resultStore, resumeIR) + } + if aggregateResume := module.NamedFunction("foo.Aggregate$coro.resume"); aggregateResume.IsNil() { + t.Fatalf("CoroSplit did not create Aggregate resume entry:\n%s", module.String()) + } + } + object, err := prog.TargetMachine().EmitToMemoryBuffer(module, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit post-CoroSplit object: %v\n%s", err, module.String()) + } + defer object.Dispose() + if len(object.Bytes()) == 0 || !bytes.Contains(object.Bytes(), []byte("foo.Root$coro")) || + !bytes.Contains(object.Bytes(), []byte("foo.Aggregate$coro")) { + t.Fatal("post-CoroSplit object lost a pure SSA coroutine symbol") + } + }) + } +} + +func TestCoroPureSSAPreflightRemainsFailClosed(t *testing.T) { + for _, test := range []struct { + name string + source string + want string + }{ + { + name: "capturing closure", + source: `package foo +func Root(value uint32) func() uint32 { return func() uint32 { return value } } +`, + want: "nested function literals require closure body lowering", + }, + { + name: "type assertion", + source: `package foo +func Root(value any) uint32 { result, _ := value.(uint32); return result } +`, + want: "instruction is outside the CFG physical ABI allowlist", + }, + { + name: "dynamic call", + source: `package foo +func Root(callback func() uint32) uint32 { return callback() } +`, + want: "requires a compilation CallPlan", + }, + { + name: "possibly panicking slice index", + source: `package foo +func Root(values []uint32, index int) uint32 { return values[index] } +`, + want: "index base is not a fixed-array pointer", + }, + { + name: "nested field array needs nil helper", + source: `package foo +type Value struct { Slots [2]uint32 } +func Root() uint32 { var value Value; value.Slots[1] = 9; return value.Slots[1] } +`, + want: "operation lowers through managed runtime helper(s) AssertNilDeref", + }, + { + name: "allocating interface box", + source: `package foo +func Root(value uint64) any { return any(value) } +`, + want: "managed backing allocation", + }, + { + name: "heap allocation", + source: `package foo +func Root() *uint32 { value := uint32(1); return &value } +`, + want: "heap allocation requires managed allocation", + }, + { + name: "pointer global store without barrier", + source: `package foo +var Global *uint32 +func Root(value *uint32) { Global = value } +`, + want: "global typed store of a pointer-containing value requires explicit write-barrier lowering", + }, + } { + t.Run(test.name, func(t *testing.T) { + ssaPkg, _, files := buildGoSSAPkg(t, test.source) + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + t.Fatal(err) + } + root := ssaPkg.Func("Root") + plan := coro.FunctionPlan{ + ID: coro.FunctionID("foo.Root"), + External: coro.Defined, + Demand: coro.AsyncDemand, + Emission: coro.EmitCoroutine, + Primary: coro.PrimaryCoroutine, + FuncRep: coro.DirectCoro, + Effect: coro.YieldOnly, + } + err = validateCoroPhysicalABIWithUniverse(root, plan, nil, universe, true, true) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("preflight error = %v, want %q", err, test.want) + } + }) + } +} + +func prepareCoroPureSSATestPlan(t *testing.T, target *llssa.Target) ( + llssa.Program, *ssa.Package, []*ast.File, *EmissionUniverse, *coro.SSAPlan, +) { + t.Helper() + ssaPkg, _, files := buildGoSSAPkg(t, coroPureSSAFixture) + var prog llssa.Program + if target == nil { + prog = newLLSSAProg(t) + } else { + prog = newLLSSAProgForTarget(t, target) + } + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapABIV2 + functionIDs.ArchiveReady = true + root, aggregate := ssaPkg.Func("Root"), ssaPkg.Func("Aggregate") + child := ssaPkg.Func("Child") + pairValue, arrayValue, scalarPair := ssaPkg.Func("PairValue"), ssaPkg.Func("ArrayValue"), ssaPkg.Func("ScalarPair") + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{ + {Function: root, Demand: coro.AsyncDemand}, + {Function: aggregate, Demand: coro.AsyncDemand}, + }, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: 1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == child || fn == pairValue || fn == arrayValue || fn == scalarPair { + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + }) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + return prog, ssaPkg, files, universe, plan +} + +func assertCoroPureSSAInstructionCoverage(t *testing.T, pkg *ssa.Package) { + t.Helper() + seen := struct { + alloc, fieldAddr, indexAddr, index, slice, extract bool + field, makeInterface, store, load bool + changeType, convert bool + }{} + for _, name := range []string{"Root", "Aggregate"} { + fn := pkg.Func(name) + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + switch instruction := instruction.(type) { + case *ssa.Alloc: + seen.alloc = true + case *ssa.FieldAddr: + seen.fieldAddr = true + case *ssa.IndexAddr: + seen.indexAddr = true + case *ssa.Index: + seen.index = true + case *ssa.Slice: + seen.slice = true + case *ssa.Extract: + seen.extract = true + case *ssa.Field: + seen.field = true + case *ssa.MakeInterface: + seen.makeInterface = true + case *ssa.Store: + seen.store = true + case *ssa.UnOp: + seen.load = seen.load || instruction.Op.String() == "*" + case *ssa.ChangeType: + seen.changeType = true + case *ssa.Convert: + seen.convert = true + } + } + } + } + if !seen.alloc || !seen.fieldAddr || !seen.indexAddr || !seen.index || !seen.slice || !seen.extract || + !seen.field || !seen.makeInterface || !seen.store || !seen.load || !seen.changeType || !seen.convert { + t.Fatalf("pure SSA fixture did not materialize every audited instruction class: %+v", seen) + } +} diff --git a/cl/emission_abi_demand.go b/cl/emission_abi_demand.go index ee8a34ffd0..fe2b158505 100644 --- a/cl/emission_abi_demand.go +++ b/cl/emission_abi_demand.go @@ -218,16 +218,18 @@ func (u *EmissionUniverse) functionABIContext(fn *ssa.Function, owner *preparedE return nil, fmt.Errorf("ABI type demand requires an emission universe, function, and exact owner") } return &context{ - prog: u.prog, - goFn: fn, - fset: u.goProg.Fset, - goProg: u.goProg, - goTyps: owner.pkgTypes, - goPkg: owner.ssa, - patches: u.patches, - loaded: u.loadedPackages(), - linkOnceFns: make(map[*ssa.Function]none), - emissionUniverse: u, + prog: u.prog, + goFn: fn, + fset: u.goProg.Fset, + goProg: u.goProg, + goTyps: owner.pkgTypes, + goPkg: owner.ssa, + patches: u.patches, + loaded: u.loadedPackages(), + linkOnceFns: make(map[*ssa.Function]none), + methodNilDerefChecks: collectMethodNilDerefChecks(fn), + addrOfFieldAddrs: collectAddrOfFieldSelectors(owner.files), + emissionUniverse: u, }, nil } @@ -246,18 +248,30 @@ func (u *EmissionUniverse) materializeABITypeDemand(fn *ssa.Function, owner *pre return llabi.PublicType(u.prog.PhysicalType(typ, llssa.InGo)) } return walkEmissionABITypeDemandEx(root, ctx.patchType, physicalMethodSignature, func(typ types.Type) error { - if !emissionABITypeMayHaveMethods(typ) { - return nil - } - methodState, methodFromPatch := state.state, state.fromPatch - if exactState, exactFromPatch, known := u.typeProvenance(owner, typ); known { - methodState, methodFromPatch = exactState, exactFromPatch + var references []*ssa.Function + if u.prog != nil { + for _, helper := range u.prog.ABITypeRuntimeFunctions(typ) { + target, available, err := u.materializeRuntimeHelperReference(fn, owner, state, helper) + if err != nil { + return fmt.Errorf("ABI type runtime reference %q: %w", helper, err) + } + if available { + references = append(references, target) + } + } } - methods, err := u.selectABITypeMethods(owner, typ, methodState, methodFromPatch) - if err != nil { - return err + if emissionABITypeMayHaveMethods(typ) { + methodState, methodFromPatch := state.state, state.fromPatch + if exactState, exactFromPatch, known := u.typeProvenance(owner, typ); known { + methodState, methodFromPatch = exactState, exactFromPatch + } + methods, err := u.selectABITypeMethods(owner, typ, methodState, methodFromPatch) + if err != nil { + return err + } + references = append(references, methods...) } - return u.recordABIMethodReferences(fn, methods) + return u.recordABIMethodReferences(fn, references) }) } diff --git a/cl/emission_abi_demand_test.go b/cl/emission_abi_demand_test.go index 5bd763b637..ceb5f95464 100644 --- a/cl/emission_abi_demand_test.go +++ b/cl/emission_abi_demand_test.go @@ -39,6 +39,7 @@ func newEmissionABIDemandTestUniverse(testProg *emissionTestProgram, pkg emissio owner := &preparedEmissionPackage{ identity: pkg.types.Path(), ssa: pkg.ssa, + files: []*ast.File{pkg.file}, pkgPath: pkg.types.Path(), oldTypes: pkg.types, pkgTypes: pkg.types, @@ -1304,7 +1305,7 @@ func TestEmissionIntrinsicOperandPolicyCoversRegistry(t *testing.T) { add(emissionIntrinsicRawAllValues, "syscall") add(emissionIntrinsicCompileValues, "boolToUint8", "atomicLoad", "atomicStore", "atomicCmpXchg", - "atomicCmpXchgOK", "atomicAddReturnNew", "atomicXchg", "atomicAdd", + "atomicCmpXchgOK", "atomicAddReturnNew", "coroPark", "atomicXchg", "atomicAdd", "atomicSub", "atomicAnd", "atomicNand", "atomicOr", "atomicXor", "atomicMax", "atomicMin", "atomicUMax", "atomicUMin") add(emissionIntrinsicFirstValue, diff --git a/cl/emission_allocacstr_coro_test.go b/cl/emission_allocacstr_coro_test.go new file mode 100644 index 0000000000..feae92536e --- /dev/null +++ b/cl/emission_allocacstr_coro_test.go @@ -0,0 +1,221 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "go/ast" + "strings" + "testing" + + "github.com/goplus/llgo/internal/coro" + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +func TestAllocaCStrIntrinsicClassificationDoesNotGeneralizeAllocCStr(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/allocacstrclassification", `package allocacstrclassification +//llgo:link AllocaCStr llgo.allocaCStr +func AllocaCStr(string) *int8 +//llgo:link AllocCStr llgo.allocCStr +func AllocCStr(string) *int8 +func UseAlloca(value string) *int8 { return AllocaCStr(value) } +func UseAlloc(value string) *int8 { return AllocCStr(value) } +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{ + SSA: pkg.ssa, Files: []*ast.File{pkg.file}, + }}) + if err != nil { + t.Fatal(err) + } + + if semantics, intrinsic, err := universe.CoroIntrinsicSemantics(pkg.ssa.Func("AllocaCStr")); err != nil || !intrinsic || semantics != CoroIntrinsicCallInlineWithLoweredCalls { + t.Fatalf("AllocaCStr function semantics = %v, %v, %v; want inline-with-lowered-calls, true, nil", semantics, intrinsic, err) + } + if semantics, intrinsic, err := universe.CoroIntrinsicSemantics(pkg.ssa.Func("AllocCStr")); err != nil || !intrinsic || semantics != CoroIntrinsicCallUnsupported { + t.Fatalf("AllocCStr function semantics = %v, %v, %v; want unsupported, true, nil", semantics, intrinsic, err) + } + + owner := universe.packages[pkg.ssa] + useAlloca := pkg.ssa.Func("UseAlloca") + ctx, err := universe.functionABIContext(useAlloca, owner) + if err != nil { + t.Fatal(err) + } + foundCStrCopy := false + var allocaCall ssa.CallInstruction + for _, block := range useAlloca.Blocks { + for _, instruction := range block.Instrs { + for _, helper := range universe.loweredRuntimeHelpers(ctx, instruction) { + if helper == "CStrCopy" { + foundCStrCopy = true + } + } + if call, ok := instruction.(ssa.CallInstruction); ok { + allocaCall = call + } + } + } + if !foundCStrCopy { + t.Fatal("AllocaCStr lowering omitted its CStrCopy runtime helper") + } + if allocaCall == nil { + t.Fatal("UseAlloca has no intrinsic call") + } + if semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(allocaCall); err != nil || !intrinsic || semantics != CoroIntrinsicCallUnsupported { + t.Fatalf("incomplete-runtime AllocaCStr semantics = %v, %v, %v; want legacy unsupported, true, nil", semantics, intrinsic, err) + } +} + +func TestAllocaCStrElidesOnlyIntrinsicAndFreezesCStrCopy(t *testing.T) { + testProg := newEmissionTestProgram() + runtimePkg := testProg.addPackage(t, llssa.PkgRuntime, `package runtime +type Pointer uintptr +type String struct { + data Pointer + len int +} +func CStrCopy(Pointer, String) *int8 { return nil } +`) + callerPkg := testProg.addPackage(t, "example.com/emission/allocacstr", `package allocacstr +//llgo:link AllocaCStr llgo.allocaCStr +func AllocaCStr(string) *int8 +func Use(value string) *int8 { return AllocaCStr(value) } +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverseWithOptions(prog, nil, []EmissionPackage{ + {SSA: runtimePkg.ssa, Files: []*ast.File{runtimePkg.file}}, + {SSA: callerPkg.ssa, Files: []*ast.File{callerPkg.file}}, + }, EmissionUniverseOptions{CompleteRuntimeABI: true}) + if err != nil { + t.Fatal(err) + } + if !universe.CompleteRuntimeABI() { + t.Fatal("complete AllocaCStr test universe lost its runtime ABI contract") + } + + owner := callerPkg.ssa.Func("Use") + helper := runtimePkg.ssa.Func("CStrCopy") + lowered, err := universe.CoroLoweredCalls(owner) + if err != nil { + t.Fatal(err) + } + if len(lowered) != 1 || lowered[0].LogicalName != "CStrCopy" || lowered[0].Target != helper { + t.Fatalf("AllocaCStr lowered calls = %+v; want exact owner-scoped CStrCopy", lowered) + } + calls := allocaCStrTestCalls(owner) + if len(calls) != 1 { + t.Fatalf("Use calls = %d, want one AllocaCStr SSA call", len(calls)) + } + call := calls[0] + semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call) + if err != nil || !intrinsic || semantics != CoroIntrinsicCallInlineWithLoweredCalls { + t.Fatalf("AllocaCStr semantics = %v, %v, %v; want inline-with-lowered-calls, true, nil", semantics, intrinsic, err) + } + + ssaUniverse, err := coro.NewSSAEmissionUniverse(testProg.ssa, universe.Functions()) + if err != nil { + t.Fatal(err) + } + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.EntryResolutionABIV0 + functionIDs.SchedulerABI = coro.SchedulerNoneABIV0 + functionIDs.ArchiveReady = true + analyze := func() (*coro.SSAPlan, error) { + return coro.AnalyzeSSA(testProg.ssa, coro.Roots{{Function: owner, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + FunctionIDs: functionIDs, + EmissionUniverse: ssaUniverse, + ResolveFunction: func(fn *ssa.Function) (*ssa.Function, bool, error) { + resolved, ok := universe.Resolve(fn) + return resolved, ok, nil + }, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == helper { + return coro.SSAFunctionPolicy{Effect: coro.WaitPlatform}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + ClassifyElidedCall: func(_ *ssa.Function, site ssa.CallInstruction) (bool, error) { + semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(site) + return intrinsic && semantics.ElidesManagedCall(), err + }, + ClassifyLoweredCalls: universe.CoroLoweredCalls, + }) + } + plan, err := analyze() + if err != nil { + t.Fatal(err) + } + if !plan.ElidesCall(call) { + t.Fatal("AllocaCStr intrinsic declaration edge was not elided") + } + if _, ok := plan.CallPlan(call); ok { + t.Fatal("AllocaCStr intrinsic declaration unexpectedly retained a managed CallPlan") + } + plannedLowered := plan.LoweredCalls(owner) + if len(plannedLowered) != 1 || plannedLowered[0].LogicalName != "CStrCopy" || plannedLowered[0].Target != helper { + t.Fatalf("planned AllocaCStr lowered calls = %+v; want exact CStrCopy", plannedLowered) + } + ownerPlan, ok := plan.FunctionPlan(owner) + if !ok || !ownerPlan.Effect.Contains(coro.WaitPlatform) { + t.Fatalf("AllocaCStr owner plan = %+v, %v; want CStrCopy suspend effect propagation", ownerPlan, ok) + } + + metadata := coro.PlanDigestMetadata{ + CoroABI: coro.EntryResolutionABIV0, SchedulerABI: coro.SchedulerNoneABIV0, + PanicABI: coro.PanicLegacyABIV0, FuncRepABI: coro.FuncRepABIV0, + TargetTriple: "x86_64-unknown-linux-gnu", PointerBits: 64, + Endianness: "little", DataLayout: "e-p:64:64", + } + digest, err := plan.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + again, err := analyze() + if err != nil { + t.Fatal(err) + } + againDigest, err := again.CoroPlanDigest(metadata) + if err != nil || digest != againDigest { + t.Fatalf("AllocaCStr plan digest = %q, %v; want stable %q", againDigest, err, digest) + } + + delete(universe.loweredCalls[owner], "CStrCopy") + if _, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call); err == nil || !intrinsic || !strings.Contains(err.Error(), "no exact frozen CStrCopy lowered call") { + t.Fatalf("AllocaCStr missing-helper semantics = _, %v, %v; want fail-closed frozen-edge error", intrinsic, err) + } +} + +func allocaCStrTestCalls(fn *ssa.Function) []ssa.CallInstruction { + var calls []ssa.CallInstruction + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + if call, ok := instruction.(ssa.CallInstruction); ok { + calls = append(calls, call) + } + } + } + return calls +} diff --git a/cl/emission_atomic_coro_test.go b/cl/emission_atomic_coro_test.go new file mode 100644 index 0000000000..ed2a32c8b3 --- /dev/null +++ b/cl/emission_atomic_coro_test.go @@ -0,0 +1,82 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "go/ast" + "strings" + "testing" +) + +func TestAtomicIntrinsicIsExactInlineNoSuspend(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/atomicintrinsic", `package atomicintrinsic +type Counter int64 +//llgo:link Add llgo.atomicAdd +func Add(ptr *Counter, value Counter) Counter { return value } +//llgo:link Load llgo.atomicLoad +func Load(ptr *Counter) Counter { return *ptr } +//llgo:link Store llgo.atomicStore +func Store(ptr *Counter, value Counter) {} +//llgo:link Compare llgo.atomicCmpXchg +func Compare(ptr *Counter, old, new Counter) (Counter, bool) { return old, false } +func Use(ptr *Counter) Counter { + Store(ptr, 1) + value, _ := Compare(ptr, 1, 2) + return Add(ptr, value) + Load(ptr) +} +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: pkg.ssa, Files: []*ast.File{pkg.file}}}) + if err != nil { + t.Fatal(err) + } + calls := allocaCStrTestCalls(pkg.ssa.Func("Use")) + if len(calls) != 4 { + t.Fatalf("atomic Use calls = %d, want four", len(calls)) + } + for _, call := range calls { + semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call) + if err != nil || !intrinsic || semantics != CoroIntrinsicCallInlineNoSuspend { + t.Fatalf("atomic call %q semantics = %v, %v, %v; want inline-no-suspend, true, nil", call, semantics, intrinsic, err) + } + } +} + +func TestAtomicIntrinsicRejectsMismatchedValueShape(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/atomicintrinsicbad", `package atomicintrinsicbad +//llgo:link Add llgo.atomicAdd +func Add(ptr *int64, value int32) int64 { return 0 } +func Use(ptr *int64) int64 { return Add(ptr, 1) } +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: pkg.ssa, Files: []*ast.File{pkg.file}}}) + if err != nil { + t.Fatal(err) + } + call := allocaCStrTestCalls(pkg.ssa.Func("Use"))[0] + if _, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call); err == nil || !intrinsic || !strings.Contains(err.Error(), "exact pointer/value/result shape") { + t.Fatalf("mismatched atomic semantics = _, %v, %v; want exact-shape error", intrinsic, err) + } +} diff --git a/cl/emission_call_roots.go b/cl/emission_call_roots.go index 8b74d55dee..04049b9b9d 100644 --- a/cl/emission_call_roots.go +++ b/cl/emission_call_roots.go @@ -87,7 +87,8 @@ func emissionIntrinsicPolicy(instruction int) (emissionIntrinsicOperandPolicy, e return emissionIntrinsicRawAllValues, nil case llgoBoolToUint8, llgoAtomicLoad, llgoAtomicStore, llgoAtomicCmpXchg, - llgoAtomicCmpXchgOK, llgoAtomicAddReturnNew: + llgoAtomicCmpXchgOK, llgoAtomicAddReturnNew, + llgoCoroPark: return emissionIntrinsicCompileValues, nil default: if instruction >= llgoAtomicOpBase && instruction <= llgoAtomicOpLast { diff --git a/cl/emission_deferdata_coro_test.go b/cl/emission_deferdata_coro_test.go new file mode 100644 index 0000000000..883d9cf46b --- /dev/null +++ b/cl/emission_deferdata_coro_test.go @@ -0,0 +1,103 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "go/ast" + "go/types" + "strings" + "testing" + + llssa "github.com/goplus/llgo/ssa" +) + +func TestDeferDataElidesOnlyIntrinsicAndFreezesGetThreadDefer(t *testing.T) { + testProg := newEmissionTestProgram() + testProg.ssa.CreatePackage(types.Unsafe, nil, nil, true) + runtimePkg := testProg.addPackage(t, llssa.PkgRuntime, `package runtime +import "unsafe" +func GetThreadDefer() unsafe.Pointer { return nil } +`) + callerPkg := testProg.addPackage(t, "example.com/emission/deferdata", `package deferdata +import "unsafe" +//llgo:link DeferData llgo.deferData +func DeferData() unsafe.Pointer +func Use() unsafe.Pointer { return DeferData() } +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + inputs := []EmissionPackage{ + {SSA: runtimePkg.ssa, Files: []*ast.File{runtimePkg.file}}, + {SSA: callerPkg.ssa, Files: []*ast.File{callerPkg.file}}, + } + + incomplete, err := PrepareEmissionUniverse(prog, nil, inputs) + if err != nil { + t.Fatal(err) + } + call := allocaCStrTestCalls(callerPkg.ssa.Func("Use"))[0] + if semantics, intrinsic, err := incomplete.CoroIntrinsicCallSiteSemantics(call); err != nil || !intrinsic || semantics != CoroIntrinsicCallUnsupported { + t.Fatalf("incomplete deferData semantics = %v, %v, %v; want legacy unsupported, true, nil", semantics, intrinsic, err) + } + + universe, err := PrepareEmissionUniverseWithOptions(prog, nil, inputs, EmissionUniverseOptions{CompleteRuntimeABI: true}) + if err != nil { + t.Fatal(err) + } + owner := callerPkg.ssa.Func("Use") + helper := runtimePkg.ssa.Func("GetThreadDefer") + lowered, err := universe.CoroLoweredCalls(owner) + if err != nil { + t.Fatal(err) + } + if len(lowered) != 1 || lowered[0].LogicalName != "GetThreadDefer" || lowered[0].Target != helper { + t.Fatalf("deferData lowered calls = %+v; want exact owner-scoped GetThreadDefer", lowered) + } + call = allocaCStrTestCalls(owner)[0] + semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call) + if err != nil || !intrinsic || semantics != CoroIntrinsicCallInlineWithLoweredCalls { + t.Fatalf("deferData semantics = %v, %v, %v; want inline-with-lowered-calls, true, nil", semantics, intrinsic, err) + } + + delete(universe.loweredCalls[owner], "GetThreadDefer") + if _, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call); err == nil || !intrinsic || !strings.Contains(err.Error(), "no exact frozen GetThreadDefer lowered call") { + t.Fatalf("deferData missing-helper semantics = _, %v, %v; want fail-closed frozen-edge error", intrinsic, err) + } +} + +func TestDeferDataRejectsWrongResultShape(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/deferdatabad", `package deferdatabad +//llgo:link DeferData llgo.deferData +func DeferData() uintptr +func Use() uintptr { return DeferData() } +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: pkg.ssa, Files: []*ast.File{pkg.file}}}) + if err != nil { + t.Fatal(err) + } + call := allocaCStrTestCalls(pkg.ssa.Func("Use"))[0] + if _, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call); err == nil || !intrinsic || !strings.Contains(err.Error(), "func() unsafe.Pointer") { + t.Fatalf("wrong-shape deferData semantics = _, %v, %v; want exact-shape error", intrinsic, err) + } +} diff --git a/cl/emission_foreign_noblock_test.go b/cl/emission_foreign_noblock_test.go new file mode 100644 index 0000000000..de7ed36937 --- /dev/null +++ b/cl/emission_foreign_noblock_test.go @@ -0,0 +1,134 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "go/ast" + "strings" + "testing" + + llssa "github.com/goplus/llgo/ssa" +) + +func TestEmissionUniverseFreezesExactForeignNoBlockCertificate(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/noblock", `package noblock + +//llgo:coro noblock +//go:linkname Safe C.audit_safe +func Safe(int) int + +//go:linkname Memcpy C.memcpy +func Memcpy(uintptr) + +func SameDisplayName() {} +func root(n uintptr) { _ = Safe(1); Memcpy(n); SameDisplayName() } +`) + testProg.ssa.Build() + prog := llssa.NewProgram(nil) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{ + SSA: pkg.ssa, Files: []*ast.File{pkg.file}, Identity: "noblock-owner", + }}) + if err != nil { + t.Fatal(err) + } + safe := pkg.ssa.Func("Safe") + certificate, certified, err := universe.CoroForeignNoBlockCertificate(safe) + if err != nil || !certified || certificate.ID == "" || certificate.ABISignature == "" || !strings.Contains(certificate.PhysicalSymbol, "audit_safe") { + t.Fatalf("Safe certificate = %+v, %t, %v; want exact frozen physical proof", certificate, certified, err) + } + for _, name := range []string{"Memcpy", "SameDisplayName"} { + if certificate, certified, err := universe.CoroForeignNoBlockCertificate(pkg.ssa.Func(name)); err != nil || certified || certificate != (CoroForeignNoBlockCertificate{}) { + t.Fatalf("%s certificate = %+v, %t, %v; want no name-derived proof", name, certificate, certified, err) + } + } + // The certificate is immutable construction metadata, not a late AST query. + for _, comment := range safe.Syntax().(*ast.FuncDecl).Doc.List { + if strings.Contains(comment.Text, "llgo:coro") { + comment.Text = "// ordinary comment" + } + } + again, certified, err := universe.CoroForeignNoBlockCertificate(safe) + if err != nil || !certified || again != certificate { + t.Fatalf("mutated-source certificate = %+v, %t, %v; want frozen %+v", again, certified, err, certificate) + } +} + +func TestEmissionUniverseForeignNoBlockFailsClosed(t *testing.T) { + for _, test := range []struct { + name string + source string + wantErr string + }{ + { + name: "Go body", + source: `package bad +//llgo:coro noblock +func Fake() {} +`, + wantErr: "requires an exact frozen C declaration", + }, + { + name: "unsupported spelling", + source: `package bad +//llgo:coro nosuspend +//go:linkname Fake C.fake +func Fake() +`, + wantErr: "unsupported directive", + }, + { + name: "duplicate", + source: `package bad +//llgo:coro noblock +//llgo:coro noblock +//go:linkname Fake C.fake +func Fake() +`, + wantErr: "duplicate", + }, + { + name: "physical signature conflict", + source: `package bad +//llgo:coro noblock +//go:linkname Safe C.same_physical +func Safe(int) int +//go:linkname Conflict C.same_physical +func Conflict(string) string +func root() { _ = Safe(1); _ = Conflict("") } +`, + wantErr: "conflicting frozen ABI signatures", + }, + } { + t.Run(test.name, func(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/badnoblock", test.source) + testProg.ssa.Build() + prog := llssa.NewProgram(nil) + defer prog.Dispose() + _, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{ + SSA: pkg.ssa, Files: []*ast.File{pkg.file}, Identity: "bad-noblock-owner", + }}) + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("PrepareEmissionUniverse error = %v; want %q", err, test.wantErr) + } + }) + } +} diff --git a/cl/emission_lowered_call_test.go b/cl/emission_lowered_call_test.go index 3bf1d32bc5..c6558987ee 100644 --- a/cl/emission_lowered_call_test.go +++ b/cl/emission_lowered_call_test.go @@ -22,6 +22,8 @@ import ( "go/ast" "strings" "testing" + + "golang.org/x/tools/go/ssa" ) func TestEmissionUniverseCoroLoweredCallsAreExactSortedAndFailClosed(t *testing.T) { @@ -82,3 +84,229 @@ func Second() {} t.Fatalf("nil lowered-call owner error = %v", err) } } + +func TestEmissionUniverseLoweredCallUnwindOnlyUsesCFGAndAllSites(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/loweredunwind", `package loweredunwind +func Owner(ok bool) int { + if !ok { panic("bad") } + return 1 +} + func Helper() {} +`) + testProg.ssa.Build() + owner := pkg.ssa.Func("Owner") + helper := pkg.ssa.Func("Helper") + // Keep normalReturnBlocks at its zero value: hand-built/report universes + // must receive the same structural CFG answer as production universes whose + // constructor eagerly initializes the cache. + universe := &EmissionUniverse{ + required: map[*ssa.Function]none{owner: {}, helper: {}}, + aliases: make(map[*ssa.Function]*ssa.Function), + loweredCalls: make(map[*ssa.Function]map[string]coroLoweredCallTarget), + } + universe.required[owner] = none{} + universe.required[helper] = none{} + var panicInstr, returnInstr ssa.Instruction + for _, block := range owner.Blocks { + for _, instr := range block.Instrs { + switch instr.(type) { + case *ssa.Panic: + panicInstr = instr + case *ssa.Return: + returnInstr = instr + } + } + } + if panicInstr == nil || returnInstr == nil { + t.Fatalf("fixture lacks panic/return instructions:\n%s", owner.String()) + } + if !universe.loweredCallUnwindOnly(owner, panicInstr) { + t.Fatal("panic-only CFG block was not classified unwind-only") + } + if universe.loweredCallUnwindOnly(owner, returnInstr) { + t.Fatal("normal Return block was classified unwind-only") + } + if err := universe.recordCoroLoweredCallSite(owner, "runtime.Helper", helper, true); err != nil { + t.Fatal(err) + } + if got, err := universe.CoroLoweredCalls(owner); err != nil || len(got) != 1 || !got[0].UnwindOnly { + t.Fatalf("unwind-only call = %+v, err=%v", got, err) + } + if err := universe.recordCoroLoweredCallSite(owner, "runtime.Helper", helper, false); err != nil { + t.Fatal(err) + } + if got, err := universe.CoroLoweredCalls(owner); err != nil || len(got) != 1 || got[0].UnwindOnly { + t.Fatalf("mixed-site call = %+v, err=%v; normal-return-reachable site must win", got, err) + } +} + +func TestLoweredRuntimeHelpersIncludeMapKeyAndInvokeEdges(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/loweredruntime", `package loweredruntime +type I interface { M() } +func Use(m map[int]int, key int, value I) { + _ = m[key] + _, _ = m[key] + m[key] = 1 + delete(m, key) + value.M() +} + +`) + testProg.ssa.Build() + universe, owner := newEmissionABIDemandTestUniverse(testProg, pkg) + fn := pkg.ssa.Func("Use") + ctx, err := universe.functionABIContext(fn, owner) + if err != nil { + t.Fatal(err) + } + got := make(map[string]bool) + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + for _, helper := range universe.loweredRuntimeHelpers(ctx, instruction) { + got[helper] = true + } + } + } + for _, helper := range []string{"AllocU", "MapAccess1", "MapAccess2", "MapAssign", "MapDelete", "IfacePtrData"} { + if !got[helper] { + t.Errorf("lowered runtime helpers %v omit %q", got, helper) + } + } +} + +func TestLoweredRuntimeHelpersMatchStaticIndexFastPath(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/loweredindex", `package loweredindex +func StaticArray(value [4]int) int { return value[1] } +func DynamicArray(value [4]int, index int) int { return value[index] } +func StaticPointer(value *[4]int) int { return value[1] } +func StaticSlice(value []int) int { return value[1] } +`) + testProg.ssa.Build() + universe, owner := newEmissionABIDemandTestUniverse(testProg, pkg) + for _, test := range []struct { + name string + wantRange bool + }{ + {name: "StaticArray"}, + {name: "DynamicArray", wantRange: true}, + {name: "StaticPointer"}, + {name: "StaticSlice", wantRange: true}, + } { + fn := pkg.ssa.Func(test.name) + ctx, err := universe.functionABIContext(fn, owner) + if err != nil { + t.Fatal(err) + } + hasRange := false + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + for _, helper := range universe.loweredRuntimeHelpers(ctx, instruction) { + if helper == "CheckIndexRange" { + hasRange = true + } + } + } + } + if hasRange != test.wantRange { + t.Errorf("%s CheckIndexRange edge = %v, want %v", test.name, hasRange, test.wantRange) + } + } +} + +func TestLoweredRuntimeHelpersMatchPointerArraySliceFastPath(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/loweredslice", `package loweredslice +func Whole(value *[4]int) []int { return value[:] } +func Partial(value *[4]int) []int { return value[1:] } +func Dynamic(value []int) []int { return value[:] } +`) + testProg.ssa.Build() + universe, owner := newEmissionABIDemandTestUniverse(testProg, pkg) + for _, test := range []struct { + name string + wantHelper bool + }{ + {name: "Whole"}, + {name: "Partial", wantHelper: true}, + {name: "Dynamic", wantHelper: true}, + } { + fn := pkg.ssa.Func(test.name) + ctx, err := universe.functionABIContext(fn, owner) + if err != nil { + t.Fatal(err) + } + hasSliceHelper := false + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + for _, helper := range universe.loweredRuntimeHelpers(ctx, instruction) { + if helper == "NewSlice2" || helper == "NewSlice3Bounds" { + hasSliceHelper = true + } + } + } + } + if hasSliceHelper != test.wantHelper { + t.Errorf("%s slice helper edge = %v, want %v", test.name, hasSliceHelper, test.wantHelper) + } + } +} + +func TestLoweredRuntimeHelpersIncludeValueReceiverNilCheck(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/loweredreceiver", `package loweredreceiver +type Value struct { N int } +func (Value) Method() {} +func Call(value *Value) { value.Method() } +`) + testProg.ssa.Build() + universe, owner := newEmissionABIDemandTestUniverse(testProg, pkg) + fn := pkg.ssa.Func("Call") + ctx, err := universe.functionABIContext(fn, owner) + if err != nil { + t.Fatal(err) + } + found := false + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + for _, helper := range universe.loweredRuntimeHelpers(ctx, instruction) { + if helper == "AssertNilDerefPtr" { + found = true + } + } + } + } + if !found { + t.Fatal("value-receiver lowering omitted AssertNilDerefPtr") + } +} + +func TestLoweredRuntimeHelpersIncludeAddressOfFieldNilCheck(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/loweredfieldaddr", `package loweredfieldaddr +type Value struct { N int } +func Addr(value *Value) *int { return &value.N } +`) + testProg.ssa.Build() + universe, owner := newEmissionABIDemandTestUniverse(testProg, pkg) + fn := pkg.ssa.Func("Addr") + ctx, err := universe.functionABIContext(fn, owner) + if err != nil { + t.Fatal(err) + } + found := false + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + for _, helper := range universe.loweredRuntimeHelpers(ctx, instruction) { + if helper == "AssertNilDeref" { + found = true + } + } + } + } + if !found { + t.Fatal("address-of field lowering omitted AssertNilDeref") + } +} diff --git a/cl/emission_runtime_abi_test.go b/cl/emission_runtime_abi_test.go new file mode 100644 index 0000000000..086fbc0c72 --- /dev/null +++ b/cl/emission_runtime_abi_test.go @@ -0,0 +1,112 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "go/ast" + "strings" + "testing" + + llssa "github.com/goplus/llgo/ssa" +) + +func TestEmissionUniverseCompleteRuntimeABIGate(t *testing.T) { + testProg := newEmissionTestProgram() + runtimePkg := testProg.addPackage(t, llssa.PkgRuntime, `package runtime +func Present() {} +`) + callerPkg := testProg.addPackage(t, "example.com/emission/runtimeabigate", `package runtimeabigate +func Allocate() *int { return new(int) } +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + inputs := []EmissionPackage{ + {SSA: runtimePkg.ssa, Files: []*ast.File{runtimePkg.file}}, + {SSA: callerPkg.ssa, Files: []*ast.File{callerPkg.file}}, + } + + incomplete, err := PrepareEmissionUniverse(prog, nil, inputs) + if err != nil { + t.Fatalf("prepare incomplete/report universe: %v", err) + } + if incomplete.CompleteRuntimeABI() { + t.Fatal("compatibility PrepareEmissionUniverse unexpectedly claims a complete runtime ABI") + } + lowered, err := incomplete.CoroLoweredCalls(callerPkg.ssa.Func("Allocate")) + if err != nil { + t.Fatal(err) + } + if len(lowered) != 0 { + t.Fatalf("incomplete/report universe lowered calls = %+v; want legacy unresolved runtime markers", lowered) + } + + _, err = PrepareEmissionUniverseWithOptions(prog, nil, inputs, EmissionUniverseOptions{CompleteRuntimeABI: true}) + if err == nil || !strings.Contains(err.Error(), `missing runtime helper "AllocZ"`) { + t.Fatalf("complete runtime ABI error = %v; want missing AllocZ failure", err) + } +} + +func TestEmissionUniverseCompleteRuntimeABIFreezesExactHelper(t *testing.T) { + testProg := newEmissionTestProgram() + runtimePkg := testProg.addPackage(t, llssa.PkgRuntime, `package runtime +func AllocZ(size uintptr) uintptr { return 0 } +`) + callerPkg := testProg.addPackage(t, "example.com/emission/runtimeabiexact", `package runtimeabiexact +func Allocate() *int { return new(int) } +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverseWithOptions(prog, nil, []EmissionPackage{ + {SSA: runtimePkg.ssa, Files: []*ast.File{runtimePkg.file}}, + {SSA: callerPkg.ssa, Files: []*ast.File{callerPkg.file}}, + }, EmissionUniverseOptions{CompleteRuntimeABI: true}) + if err != nil { + t.Fatal(err) + } + if !universe.CompleteRuntimeABI() { + t.Fatal("complete construction did not retain its runtime ABI contract") + } + owner := callerPkg.ssa.Func("Allocate") + target := runtimePkg.ssa.Func("AllocZ") + lowered, err := universe.CoroLoweredCalls(owner) + if err != nil { + t.Fatal(err) + } + if len(lowered) != 1 || lowered[0].LogicalName != "AllocZ" || lowered[0].Target != target { + t.Fatalf("complete runtime ABI lowered calls = %+v; want exact AllocZ target", lowered) + } +} + +func TestEmissionUniverseCompleteRuntimeABIRequiresRuntimePackage(t *testing.T) { + testProg := newEmissionTestProgram() + callerPkg := testProg.addPackage(t, "example.com/emission/runtimeabimissing", `package runtimeabimissing +func Use() {} +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + _, err := PrepareEmissionUniverseWithOptions(prog, nil, []EmissionPackage{{ + SSA: callerPkg.ssa, Files: []*ast.File{callerPkg.file}, + }}, EmissionUniverseOptions{CompleteRuntimeABI: true}) + if err == nil || !strings.Contains(err.Error(), "complete runtime ABI requires package") { + t.Fatalf("complete runtime ABI without runtime error = %v", err) + } +} diff --git a/cl/emission_runtime_helpers.go b/cl/emission_runtime_helpers.go new file mode 100644 index 0000000000..b93e62fca7 --- /dev/null +++ b/cl/emission_runtime_helpers.go @@ -0,0 +1,734 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "fmt" + "go/constant" + "go/token" + "go/types" + "sort" + + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +// materializeLoweredRuntimeHelpers freezes the runtime calls that LLGo's +// instruction lowering inserts without an x/tools SSA CallInstruction. An +// explicitly complete runtime ABI is required. Report-only/unit-test +// universes retain legacy symbol resolution and intentionally freeze no such +// edges; whole-program active builds enable the contract and fail closed. +func (u *EmissionUniverse) materializeLoweredRuntimeHelpers(ctx *context, ownerFn *ssa.Function, ownerPkg *preparedEmissionPackage, state emissionFunctionState, instr ssa.Instruction) error { + if u == nil || !u.completeRuntimeABI { + return nil + } + if u.prog == nil { + return fmt.Errorf("prepare emission universe: complete runtime ABI requires an LLVM SSA program") + } + runtimePkg := u.byPath[llssa.PkgRuntime] + if runtimePkg == nil { + return fmt.Errorf("prepare emission universe: complete runtime ABI requires package %q", llssa.PkgRuntime) + } + if u.pathDup[llssa.PkgRuntime] { + return fmt.Errorf("prepare emission universe: runtime helper resolution has ambiguous package path %q", llssa.PkgRuntime) + } + for _, helper := range u.loweredRuntimeHelpers(ctx, instr) { + target := runtimePkg.ssa.Func(helper) + if target == nil { + return fmt.Errorf("prepare emission universe: function %q lowers to missing runtime helper %q", ownerFn.Name(), helper) + } + canonical, err := u.addResolvedRequired(target, ownerPkg, ownerFn, state) + if err != nil { + return fmt.Errorf("prepare emission universe: function %q runtime helper %q: %w", ownerFn.Name(), helper, err) + } + if err := u.recordCoroLoweredCallSite(ownerFn, helper, canonical, u.loweredCallUnwindOnly(ownerFn, instr)); err != nil { + return err + } + } + return nil +} + +// loweredCallUnwindOnly reports a structural CFG proof: the instruction's +// block cannot reach any normal Return in owner. It deliberately does not use +// helper names, runtime package policy, dominance guesses, or panic text. +// +// The result is cached per immutable SSA body. recordCoroLoweredCallSite merges +// all occurrences of one logical helper with AND, so any normal-return-reachable +// physical use makes the frozen edge ordinary. +func (u *EmissionUniverse) loweredCallUnwindOnly(owner *ssa.Function, instr ssa.Instruction) bool { + if u == nil || owner == nil || instr == nil || instr.Parent() != owner || instr.Block() == nil { + return false + } + if u.normalReturnBlocks == nil { + u.normalReturnBlocks = make(map[*ssa.Function]map[*ssa.BasicBlock]none) + } + reachable, ok := u.normalReturnBlocks[owner] + if !ok { + reachable = make(map[*ssa.BasicBlock]none) + queue := make([]*ssa.BasicBlock, 0, len(owner.Blocks)) + for _, block := range owner.Blocks { + for _, blockInstr := range block.Instrs { + if _, normalReturn := blockInstr.(*ssa.Return); normalReturn { + reachable[block] = none{} + queue = append(queue, block) + break + } + } + } + for head := 0; head < len(queue); head++ { + for _, predecessor := range queue[head].Preds { + if _, seen := reachable[predecessor]; seen { + continue + } + reachable[predecessor] = none{} + queue = append(queue, predecessor) + } + } + u.normalReturnBlocks[owner] = reachable + } + _, reachesReturn := reachable[instr.Block()] + return !reachesReturn +} + +func (u *EmissionUniverse) materializeRuntimeHelperReference(ownerFn *ssa.Function, ownerPkg *preparedEmissionPackage, state emissionFunctionState, helper string) (*ssa.Function, bool, error) { + if u == nil || !u.completeRuntimeABI { + return nil, false, nil + } + if u.prog == nil { + return nil, false, fmt.Errorf("complete runtime ABI requires an LLVM SSA program") + } + if u.byPath[llssa.PkgRuntime] == nil { + return nil, false, fmt.Errorf("complete runtime ABI requires package %q", llssa.PkgRuntime) + } + if u.pathDup[llssa.PkgRuntime] { + return nil, false, fmt.Errorf("runtime helper resolution has ambiguous package path %q", llssa.PkgRuntime) + } + target := u.byPath[llssa.PkgRuntime].ssa.Func(helper) + if target == nil { + return nil, false, fmt.Errorf("missing runtime helper %q", helper) + } + canonical, err := u.addResolvedRequired(target, ownerPkg, ownerFn, state) + if err != nil { + return nil, false, err + } + return canonical, true, nil +} + +func (u *EmissionUniverse) loweredRuntimeHelpers(ctx *context, instr ssa.Instruction) []string { + set := make(map[string]struct{}) + add := func(names ...string) { + for _, name := range names { + if name != "" { + set[name] = struct{}{} + } + } + } + + switch v := instr.(type) { + case *ssa.BinOp: + u.binOpRuntimeHelpers(ctx, v, add) + case *ssa.UnOp: + switch v.Op { + case token.ARROW: + add("ChanRecv") + case token.MUL: + if _, checkedReceiver := ctx.methodNilDerefChecks[v]; checkedReceiver { + // compileCheckedDeref preserves the checked pointer through the + // value-receiver call and therefore uses the pointer-returning ABI. + add("AssertNilDerefPtr") + } else if shouldAssertDirectNilDeref(v) { + add("AssertNilDeref") + } + } + case *ssa.Convert: + u.convertRuntimeHelpers(ctx, v, add) + case *ssa.Alloc: + if v.Heap && !ctx.skipSyntheticMakeSliceAlloc(v) && !isEmissionVargsAlloc(ctx, v) { + elem := types.Unalias(v.Type()).(*types.Pointer).Elem() + physical := ctx.type_(elem, llssa.InGo) + if u.prog.SizeOf(physical) != 0 { + add("AllocZ") + } + } + case *ssa.FieldAddr: + if ctx.isAddressOfFieldAddr(v) { + add("AssertNilDeref") + } + case *ssa.Index: + if emissionIndexNeedsRangeCheck(ctx, v.X, v.Index) { + add("CheckIndexRange") + } + case *ssa.IndexAddr: + // compileValue consumes varargs IndexAddr nodes in the enclosing varargs + // lowering and emits neither an address nor bounds/nil helpers here. + if emissionIsVargsAlloc(ctx, v.X) { + break + } + if emissionIndexNeedsRangeCheck(ctx, v.X, v.Index) { + add("CheckIndexRange") + } + if _, pointer := types.Unalias(ctx.patchType(v.X.Type())).Underlying().(*types.Pointer); pointer && !emissionKnownNonNilArrayBase(v.X) { + add("AssertNilDeref") + } + case *ssa.Slice: + if _, synthetic := ctx.syntheticMakeSliceCap(v); synthetic { + add("MakeSlice") + break + } + if emissionIsVargsAlloc(ctx, v.X) { + break + } + switch types.Unalias(ctx.patchType(v.X.Type())).Underlying().(type) { + case *types.Basic: + add("StringSlice2") + case *types.Slice: + if v.Max == nil { + add("NewSlice2") + } else { + add("NewSlice3Bounds") + } + case *types.Pointer: + // Builder.Slice returns unsafeSlice directly for the complete p[:] + // view of a pointer-to-array. No bounds helper is emitted. + if v.Low == nil && v.High == nil && v.Max == nil { + break + } + if v.Max == nil { + add("NewSlice2") + } else { + add("NewSlice3Bounds") + } + } + case *ssa.MakeInterface: + u.makeInterfaceRuntimeHelpers(ctx, v, add) + case *ssa.MakeSlice: + add("MakeSlice") + case *ssa.MakeMap: + add("MakeMap") + case *ssa.MakeClosure: + if len(v.Bindings) != 0 { + add("AllocU") + } + case *ssa.Lookup: + // Builder.Lookup always materializes the map key through mapKeyPtr + // before calling MapAccess1/MapAccess2. mapKeyPtr owns an AllocU call; + // it is not represented by an x/tools SSA instruction. + add("AllocU") + if v.CommaOk { + add("MapAccess2") + } else { + add("MapAccess1") + } + case *ssa.TypeAssert: + u.typeAssertRuntimeHelpers(ctx, v, add) + case *ssa.Range: + switch types.Unalias(ctx.patchType(v.X.Type())).Underlying().(type) { + case *types.Basic: + add("NewStringIter") + case *types.Map: + add("NewMapIter") + } + case *ssa.Next: + if v.IsString { + add("StringIterNext") + } else { + add("MapIterNext") + } + case *ssa.ChangeInterface: + if interfaceIsNonEmpty(ctx.patchType(v.X.Type())) { + add("IfaceType") + } + if interfaceIsNonEmpty(ctx.patchType(v.Type())) { + add("NewItab") + } + case *ssa.MakeChan: + add("NewChan") + case *ssa.Select: + if v.Blocking { + add("Select") + } else { + add("TrySelect") + } + case *ssa.SliceToArrayPointer: + add("PanicSliceConvert") + case *ssa.MapUpdate: + // Builder.MapUpdate uses the same mapKeyPtr lowering as Lookup. + add("AllocU", "MapAssign") + case *ssa.Panic: + add("Panic") + case *ssa.Send: + add("ChanSend") + case *ssa.Call: + if v.Call.IsInvoke() { + // Builder.Imethod extracts the receiver through this runtime helper + // before issuing the physical closure call. + add("IfacePtrData") + } + // Exact intrinsic opcodes are frozen by the LLSSA link table. Pure + // frontend/report universes intentionally have no such table and do not + // materialize physical runtime-helper edges. + if u.prog != nil { + opcode, intrinsic := emissionCallIntrinsicInstruction(ctx, &v.Call) + switch { + case intrinsic && opcode == llgoAllocaCStr: + // Builder.AllocaCStr emits StringLen, +1, and LLVM alloca + // directly, then inserts this one managed runtime call. The + // intrinsic declaration edge is elided, so CStrCopy must remain + // an exact owner-scoped lowered edge for effect propagation and + // coroutine-aware codegen resolution. + add("CStrCopy") + case intrinsic && opcode == llgoDeferData: + // Builder.DeferData replaces the compiler declaration with an + // ordinary runtime.GetThreadDefer call. + add("GetThreadDefer") + case intrinsic && opcode == llgoString: + // Builder.MakeString selects exactly one runtime helper from the + // already-lowered varargs shape. Invalid shapes are rejected later + // by CoroIntrinsicCallSiteSemantics. + if helper, err := emissionStringIntrinsicHelper(ctx, v); err == nil { + add(helper) + } + case intrinsic && opcode == llgoSigsetjmp && u.coroUsesRuntimeSigjmpHelpers(): + add("Sigsetjmp") + case intrinsic && opcode == llgoSiglongjmp && u.coroUsesRuntimeSigjmpHelpers(): + add("Siglongjmp") + } + } + u.builtinRuntimeHelpers(ctx, &v.Call, add) + } + + ret := make([]string, 0, len(set)) + for name := range set { + ret = append(ret, name) + } + sort.Strings(ret) + return ret +} + +// emissionStringIntrinsicHelper mirrors context.string, compileVArg, and +// Builder.MakeString closely enough to select the one physical runtime call. +// The trailing x/tools SSA argument is always the materialized variadic slice: +// nil/empty means StringFromCStr, while one or more values selects StringFrom. +func emissionStringIntrinsicHelper(ctx *context, call *ssa.Call) (string, error) { + if ctx == nil || call == nil || call.Common() == nil || call.Common().IsInvoke() { + return "", fmt.Errorf("llgo.string must be an exact direct call") + } + common := call.Common() + if len(common.Args) != 2 { + return "", fmt.Errorf("llgo.string call %q requires a C string pointer and one variadic slice operand", call.String()) + } + signature := common.Signature() + if signature == nil || signature.Recv() != nil || !signature.Variadic() || signature.Params() == nil || signature.Params().Len() != 2 { + return "", fmt.Errorf("llgo.string call %q requires the exact func(*int8, ...any) string shape", call.String()) + } + first, ok := types.Unalias(signature.Params().At(0).Type()).Underlying().(*types.Pointer) + if !ok { + return "", fmt.Errorf("llgo.string call %q requires the exact func(*int8, ...any) string shape", call.String()) + } + firstElem, ok := types.Unalias(first.Elem()).Underlying().(*types.Basic) + if !ok || firstElem.Kind() != types.Int8 { + return "", fmt.Errorf("llgo.string call %q requires the exact func(*int8, ...any) string shape", call.String()) + } + variadic, ok := types.Unalias(signature.Params().At(1).Type()).Underlying().(*types.Slice) + if !ok || !isAny(variadic.Elem()) { + return "", fmt.Errorf("llgo.string call %q requires the exact func(*int8, ...any) string shape", call.String()) + } + results := signature.Results() + if results == nil || results.Len() != 1 { + return "", fmt.Errorf("llgo.string call %q requires the exact func(*int8, ...any) string shape", call.String()) + } + result, ok := types.Unalias(results.At(0).Type()).Underlying().(*types.Basic) + if !ok || result.Kind() != types.String { + return "", fmt.Errorf("llgo.string call %q requires the exact func(*int8, ...any) string shape", call.String()) + } + actualPointer, ok := types.Unalias(common.Args[0].Type()).Underlying().(*types.Pointer) + if !ok { + return "", fmt.Errorf("llgo.string call %q has a non-pointer C string operand", call.String()) + } + actualElem, ok := types.Unalias(actualPointer.Elem()).Underlying().(*types.Basic) + if !ok || actualElem.Kind() != types.Int8 { + return "", fmt.Errorf("llgo.string call %q has a non-*int8 C string operand", call.String()) + } + + switch varargs := common.Args[1].(type) { + case *ssa.Const: + if varargs.Value == nil { + return "StringFromCStr", nil + } + case *ssa.Parameter: + if varargs.Parent() != nil && llssa.HasNameValist(varargs.Parent().Signature) { + // compileVArg intentionally treats a named va-list parameter as an + // empty frontend-owned list. + return "StringFromCStr", nil + } + case *ssa.Slice: + if !emissionIsVargsAlloc(ctx, varargs.X) { + break + } + alloc := varargs.X.(*ssa.Alloc) + pointer := types.Unalias(alloc.Type()).(*types.Pointer) + array := types.Unalias(pointer.Elem()).(*types.Array) + if array.Len() == 0 { + return "StringFromCStr", nil + } + return "StringFrom", nil + } + return "", fmt.Errorf("llgo.string call %q has an unsupported variadic lowering shape %T", call.String(), common.Args[1]) +} + +// emissionIndexNeedsRangeCheck mirrors ssa.Builder.checkRange for the source +// operands available before LLVM construction. Slice and string lengths are +// dynamic, while arrays and pointers to arrays have a frozen constant bound. +func emissionIndexNeedsRangeCheck(ctx *context, collection, index ssa.Value) bool { + if ctx == nil || collection == nil || index == nil { + return true + } + var bound int64 = -1 + switch typ := types.Unalias(ctx.patchType(collection.Type())).Underlying().(type) { + case *types.Array: + bound = typ.Len() + case *types.Pointer: + if array, ok := types.Unalias(typ.Elem()).Underlying().(*types.Array); ok { + bound = array.Len() + } + } + constantIndex, ok := index.(*ssa.Const) + if !ok || constantIndex.Value == nil { + return true + } + basic, ok := types.Unalias(index.Type()).Underlying().(*types.Basic) + if !ok || basic.Info()&types.IsInteger == 0 { + return true + } + if basic.Info()&types.IsUnsigned == 0 && constant.Sign(constantIndex.Value) < 0 { + return true + } + if bound < 0 { + return true + } + value, exact := constant.Uint64Val(constantIndex.Value) + return !exact || value >= uint64(bound) +} + +// emissionKnownNonNilArrayBase deliberately matches the narrow LLVM-side +// isKnownNonNilArrayBase predicate: direct globals, stack allocas, and the +// AllocU/AllocZ calls produced for an SSA Alloc. Recursive field/index address +// reasoning would incorrectly suppress a physical AssertNilDeref call. +func emissionKnownNonNilArrayBase(value ssa.Value) bool { + switch value.(type) { + case *ssa.Global, *ssa.Alloc: + return true + default: + return false + } +} + +func isEmissionVargsAlloc(ctx *context, alloc *ssa.Alloc) bool { + if alloc == nil || alloc.Comment != "varargs" { + return false + } + ptr, ok := types.Unalias(alloc.Type()).(*types.Pointer) + if !ok { + return false + } + arr, ok := types.Unalias(ptr.Elem()).(*types.Array) + return ok && isAny(arr.Elem()) && isAllocVargs(ctx, alloc) +} + +func (u *EmissionUniverse) binOpRuntimeHelpers(ctx *context, op *ssa.BinOp, add func(...string)) { + typ := types.Unalias(ctx.patchType(op.X.Type())).Underlying() + switch typ := typ.(type) { + case *types.Basic: + switch { + case typ.Kind() == types.String: + switch op.Op { + case token.ADD: + add("StringCat") + case token.EQL, token.NEQ: + add("StringEqual") + case token.LSS, token.LEQ, token.GTR, token.GEQ: + add("StringLess") + } + case typ.Info()&types.IsComplex != 0 && op.Op == token.QUO: + add("Complex128Div") + case typ.Info()&types.IsInteger != 0 && (op.Op == token.QUO || op.Op == token.REM): + if !constantIntegerKnownNonZero(op.Y) { + add("AssertDivideByZero") + } + } + if (op.Op == token.SHL || op.Op == token.SHR) && signedIntegerMayBeNegative(op.Y) { + add("AssertNegativeShift") + } + case *types.Interface: + if op.Op == token.EQL || op.Op == token.NEQ { + add("EfaceEqual") + if !typ.Empty() { + add("IfaceType") + } + if interfaceIsNonEmpty(ctx.patchType(op.Y.Type())) { + add("IfaceType") + } + } + case *types.Array: + if op.Op == token.EQL || op.Op == token.NEQ { + u.compositeCompareRuntimeHelpers(ctx, typ.Elem(), add) + } + case *types.Struct: + if op.Op == token.EQL || op.Op == token.NEQ { + for i := 0; i < typ.NumFields(); i++ { + if typ.Field(i).Name() != "_" { + u.compositeCompareRuntimeHelpers(ctx, typ.Field(i).Type(), add) + } + } + } + } +} + +func (u *EmissionUniverse) compositeCompareRuntimeHelpers(ctx *context, typ types.Type, add func(...string)) { + typ = types.Unalias(ctx.patchType(typ)).Underlying() + switch typ := typ.(type) { + case *types.Basic: + if typ.Kind() == types.String { + add("StringEqual") + } + case *types.Interface: + add("EfaceEqual") + if !typ.Empty() { + add("IfaceType") + } + case *types.Array: + u.compositeCompareRuntimeHelpers(ctx, typ.Elem(), add) + case *types.Struct: + for i := 0; i < typ.NumFields(); i++ { + if typ.Field(i).Name() != "_" { + u.compositeCompareRuntimeHelpers(ctx, typ.Field(i).Type(), add) + } + } + } +} + +func constantIntegerKnownNonZero(value ssa.Value) bool { + c, ok := value.(*ssa.Const) + return ok && c.Value != nil && constant.Sign(c.Value) != 0 +} + +func signedIntegerMayBeNegative(value ssa.Value) bool { + basic, ok := types.Unalias(value.Type()).Underlying().(*types.Basic) + if !ok || basic.Info()&types.IsInteger == 0 || basic.Info()&types.IsUnsigned != 0 { + return false + } + if c, ok := value.(*ssa.Const); ok && c.Value != nil { + return constant.Sign(c.Value) < 0 + } + return true +} + +func (u *EmissionUniverse) convertRuntimeHelpers(ctx *context, convert *ssa.Convert, add func(...string)) { + dst := types.Unalias(ctx.patchType(convert.Type())).Underlying() + src := types.Unalias(ctx.patchType(convert.X.Type())).Underlying() + if basic, ok := dst.(*types.Basic); ok && basic.Kind() == types.String { + switch src := src.(type) { + case *types.Slice: + if elem, ok := types.Unalias(src.Elem()).Underlying().(*types.Basic); ok { + switch elem.Kind() { + case types.Byte: + add("StringFromBytes") + case types.Rune: + add("StringFromRunes") + } + } + case *types.Basic: + if src.Info()&types.IsInteger != 0 { + if src.Info()&types.IsUnsigned != 0 { + add("StringFromUint64") + } else { + add("StringFromInt64") + } + } + } + } + if slice, ok := dst.(*types.Slice); ok { + if basic, ok := src.(*types.Basic); ok && basic.Kind() == types.String { + if elem, ok := types.Unalias(slice.Elem()).Underlying().(*types.Basic); ok { + switch elem.Kind() { + case types.Byte: + add("StringToBytes") + case types.Rune: + add("StringToRunes") + } + } + } + } +} + +func (u *EmissionUniverse) makeInterfaceRuntimeHelpers(ctx *context, makeInterface *ssa.MakeInterface, add func(...string)) { + // compileValue deliberately consumes these nodes without calling + // Builder.MakeInterface: untyped nil becomes a constant, varargs stores + // are lowered by their consumer, and funcAddr/funcPCABI0 inspect the SSA + // operand directly. + if !u.makeInterfaceEmitsABIType(makeInterface, ctx) { + return + } + if interfaceIsNonEmpty(ctx.patchType(makeInterface.Type())) { + add("NewItab") + } + physical := ctx.type_(makeInterface.X.Type(), llssa.InGo) + if !emissionDirectIfaceType(physical.RawType()) { + add("AllocU") + } + if unop, ok := makeInterface.X.(*ssa.UnOp); ok && unop.Op == token.MUL && (ctx.isLargeNonPointerValue(physical) || ctx.isZeroSizedValue(physical)) { + add("AssertNilDeref") + // MakeInterfaceFromPtr uses the indirect representation for both large + // and zero-sized values and therefore always copies through AllocU. + add("AllocU", "Typedmemmove") + } +} + +func emissionDirectIfaceType(typ types.Type) bool { + switch typ := types.Unalias(typ).(type) { + case *types.Named: + return emissionDirectIfaceType(typ.Underlying()) + case *types.Pointer, *types.Chan, *types.Map, *types.Signature: + return true + case *types.Basic: + return typ.Kind() == types.UnsafePointer + case *types.Array: + return typ.Len() == 1 && emissionDirectIfaceType(typ.Elem()) + case *types.Struct: + return typ.NumFields() == 1 && emissionDirectIfaceType(typ.Field(0).Type()) + } + return false +} + +func (u *EmissionUniverse) typeAssertRuntimeHelpers(ctx *context, assertion *ssa.TypeAssert, add func(...string)) { + asserted := ctx.patchType(assertion.AssertedType) + if !types.Identical(ctx.patchType(assertion.X.Type()), asserted) { + if _, ok := types.Unalias(asserted).Underlying().(*types.Interface); ok { + add("Implements") + if interfaceIsNonEmpty(asserted) { + add("NewItab") + } + } else if _, ok := types.Unalias(asserted).Underlying().(*types.Signature); ok { + add("MatchesClosure") + } + } + if interfaceIsNonEmpty(ctx.patchType(assertion.X.Type())) { + add("IfaceType") + } + if !assertion.CommaOk { + add("PanicTypeAssert") + } +} + +func interfaceIsNonEmpty(typ types.Type) bool { + iface, ok := types.Unalias(typ).Underlying().(*types.Interface) + if !ok { + return false + } + iface.Complete() + return !iface.Empty() +} + +func (u *EmissionUniverse) builtinRuntimeHelpers(ctx *context, call *ssa.CallCommon, add func(...string)) { + builtin, ok := call.Value.(*ssa.Builtin) + if !ok { + return + } + args := call.Args + switch builtin.Name() { + case "ssa:wrapnilchk": + add("PanicWrapNilPointer") + case "len": + if len(args) == 1 { + switch types.Unalias(ctx.patchType(args[0].Type())).Underlying().(type) { + case *types.Chan: + add("ChanLen") + case *types.Map: + add("MapLen") + } + } + case "cap": + if len(args) == 1 { + if _, ok := types.Unalias(ctx.patchType(args[0].Type())).Underlying().(*types.Chan); ok { + add("ChanCap") + } + } + case "append": + add("SliceAppend") + case "copy": + add("SliceCopy") + case "close": + add("ChanClose") + case "recover": + add("Recover") + case "panic": + add("Panic") + case "delete": + // The delete builtin also lowers its key through Builder.mapKeyPtr. + add("AllocU", "MapDelete") + case "clear": + if len(args) == 1 { + switch types.Unalias(ctx.patchType(args[0].Type())).Underlying().(type) { + case *types.Map: + add("MapClear") + case *types.Slice: + add("SliceClear") + } + } + case "print", "println": + for _, arg := range args { + add(runtimePrintHelper(ctx.patchType(arg.Type()))) + } + if builtin.Name() == "println" { + add("PrintByte") + } + case "String", "Slice": + add("AssertRuntimeError") + } +} + +func runtimePrintHelper(typ types.Type) string { + switch typ := types.Unalias(typ).Underlying().(type) { + case *types.Basic: + switch { + case typ.Kind() == types.Bool: + return "PrintBool" + case typ.Info()&types.IsInteger != 0 && typ.Info()&types.IsUnsigned == 0: + return "PrintInt" + case typ.Info()&types.IsInteger != 0: + return "PrintUint" + case typ.Info()&types.IsFloat != 0: + return "PrintFloat" + case typ.Kind() == types.String: + return "PrintString" + case typ.Info()&types.IsComplex != 0: + return "PrintComplex" + case typ.Kind() == types.UnsafePointer: + return "PrintPointer" + } + case *types.Pointer, *types.Signature, *types.Chan, *types.Map: + return "PrintPointer" + case *types.Slice: + return "PrintSlice" + case *types.Interface: + if typ.Empty() { + return "PrintEface" + } + return "PrintIface" + } + return "" +} diff --git a/cl/emission_sigjmp_coro_test.go b/cl/emission_sigjmp_coro_test.go new file mode 100644 index 0000000000..db407a5f89 --- /dev/null +++ b/cl/emission_sigjmp_coro_test.go @@ -0,0 +1,100 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "go/ast" + "go/types" + "strings" + "testing" + + llssa "github.com/goplus/llgo/ssa" +) + +func TestLegacySigjmpIntrinsicsFreezeNativeRuntimeLeaves(t *testing.T) { + testProg := newEmissionTestProgram() + testProg.ssa.CreatePackage(types.Unsafe, nil, nil, true) + runtimePkg := testProg.addPackage(t, llssa.PkgRuntime, `package runtime +import "unsafe" +func Sigsetjmp(unsafe.Pointer, int32) int32 { return 0 } +func Siglongjmp(unsafe.Pointer, int32) {} +`) + callerPkg := testProg.addPackage(t, "example.com/emission/sigjmp", `package sigjmp +import "unsafe" +//llgo:link Sigjmpbuf llgo.sigjmpbuf +func Sigjmpbuf() unsafe.Pointer +//llgo:link Sigsetjmp llgo.sigsetjmp +func Sigsetjmp(unsafe.Pointer, int32) int32 +//llgo:link Siglongjmp llgo.siglongjmp +func Siglongjmp(unsafe.Pointer, int32) +func Use() int32 { + buf := Sigjmpbuf() + value := Sigsetjmp(buf, 0) + Siglongjmp(buf, 1) + return value +} +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverseWithOptions(prog, nil, []EmissionPackage{ + {SSA: runtimePkg.ssa, Files: []*ast.File{runtimePkg.file}}, + {SSA: callerPkg.ssa, Files: []*ast.File{callerPkg.file}}, + }, EmissionUniverseOptions{CompleteRuntimeABI: true}) + if err != nil { + t.Fatal(err) + } + owner := callerPkg.ssa.Func("Use") + lowered, err := universe.CoroLoweredCalls(owner) + if err != nil { + t.Fatal(err) + } + if len(lowered) != 2 || lowered[0].LogicalName != "Siglongjmp" || lowered[0].Target != runtimePkg.ssa.Func("Siglongjmp") || + lowered[1].LogicalName != "Sigsetjmp" || lowered[1].Target != runtimePkg.ssa.Func("Sigsetjmp") { + t.Fatalf("legacy sigjmp lowered calls = %+v; want exact native runtime leaves", lowered) + } + for _, call := range allocaCStrTestCalls(owner) { + semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call) + if err != nil || !intrinsic || !semantics.ElidesManagedCall() { + t.Fatalf("legacy sigjmp call %q semantics = %v, %v, %v; want exact elided intrinsic", call, semantics, intrinsic, err) + } + } +} + +func TestLegacySigjmpIntrinsicsFailClosedOnWasm(t *testing.T) { + testProg := newEmissionTestProgram() + testProg.ssa.CreatePackage(types.Unsafe, nil, nil, true) + pkg := testProg.addPackage(t, "example.com/emission/sigjmpwasm", `package sigjmpwasm +import "unsafe" +//llgo:link Siglongjmp llgo.siglongjmp +func Siglongjmp(unsafe.Pointer, int32) +func Use(value unsafe.Pointer) { Siglongjmp(value, 1) } +`) + testProg.ssa.Build() + prog := newLLSSAProgForTarget(t, &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: pkg.ssa, Files: []*ast.File{pkg.file}}}) + if err != nil { + t.Fatal(err) + } + call := allocaCStrTestCalls(pkg.ssa.Func("Use"))[0] + if _, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call); err == nil || !intrinsic || !strings.Contains(err.Error(), "requires a non-legacy coroutine PanicABI") { + t.Fatalf("wasm legacy siglongjmp semantics = _, %v, %v; want PanicABI fail-closed error", intrinsic, err) + } +} diff --git a/cl/emission_string_coro_test.go b/cl/emission_string_coro_test.go new file mode 100644 index 0000000000..58e5a6d37a --- /dev/null +++ b/cl/emission_string_coro_test.go @@ -0,0 +1,96 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "go/ast" + "strings" + "testing" + + llssa "github.com/goplus/llgo/ssa" +) + +func TestStringIntrinsicFreezesExactVarargsHelper(t *testing.T) { + testProg := newEmissionTestProgram() + runtimePkg := testProg.addPackage(t, llssa.PkgRuntime, `package runtime +func StringFromCStr(*int8) string { return "" } +func StringFrom(*int8, int) string { return "" } +`) + callerPkg := testProg.addPackage(t, "example.com/emission/stringintrinsic", `package stringintrinsic +//llgo:link String llgo.string +func String(value *int8, __llgo_va_list ...any) string +func WithoutLen(value *int8) string { return String(value) } +func WithLen(value *int8, length int) string { return String(value, length) } +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverseWithOptions(prog, nil, []EmissionPackage{ + {SSA: runtimePkg.ssa, Files: []*ast.File{runtimePkg.file}}, + {SSA: callerPkg.ssa, Files: []*ast.File{callerPkg.file}}, + }, EmissionUniverseOptions{CompleteRuntimeABI: true}) + if err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + owner string + helper string + }{ + {owner: "WithoutLen", helper: "StringFromCStr"}, + {owner: "WithLen", helper: "StringFrom"}, + } { + owner := callerPkg.ssa.Func(test.owner) + lowered, err := universe.CoroLoweredCalls(owner) + if err != nil { + t.Fatal(err) + } + if len(lowered) != 1 || lowered[0].LogicalName != test.helper || lowered[0].Target != runtimePkg.ssa.Func(test.helper) { + t.Fatalf("%s lowered calls = %+v; want exact %s", test.owner, lowered, test.helper) + } + calls := allocaCStrTestCalls(owner) + if len(calls) != 1 { + t.Fatalf("%s calls = %d, want one", test.owner, len(calls)) + } + semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(calls[0]) + if err != nil || !intrinsic || semantics != CoroIntrinsicCallInlineWithLoweredCalls { + t.Fatalf("%s semantics = %v, %v, %v; want inline-with-lowered-calls, true, nil", test.owner, semantics, intrinsic, err) + } + } +} + +func TestStringIntrinsicRejectsWrongDeclarationShape(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/stringintrinsicbad", `package stringintrinsicbad +//llgo:link String llgo.string +func String(value *int8, __llgo_va_list ...any) uintptr +func Use(value *int8) uintptr { return String(value) } +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: pkg.ssa, Files: []*ast.File{pkg.file}}}) + if err != nil { + t.Fatal(err) + } + call := allocaCStrTestCalls(pkg.ssa.Func("Use"))[0] + if _, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call); err == nil || !intrinsic || !strings.Contains(err.Error(), "func(*int8, ...any) string") { + t.Fatalf("wrong-shape string semantics = _, %v, %v; want exact-shape error", intrinsic, err) + } +} diff --git a/cl/emission_universe.go b/cl/emission_universe.go index 3641640f1a..5a879a85af 100644 --- a/cl/emission_universe.go +++ b/cl/emission_universe.go @@ -47,6 +47,16 @@ type EmissionPackage struct { MetadataOnly bool // freeze frontend directives/ownership without selecting definitions } +// EmissionUniverseOptions selects construction contracts that are available +// only to a complete whole-program frontend. Report-only and single-package +// callers should use the zero value. +type EmissionUniverseOptions struct { + // CompleteRuntimeABI requires the exact LLGo runtime package and freezes + // every compiler-inserted runtime helper edge. Missing runtime helpers fail + // construction instead of being left to the legacy LLVM symbol resolver. + CompleteRuntimeABI bool +} + type preparedEmissionPackage struct { order int identity string @@ -70,14 +80,15 @@ type preparedEmissionPackage struct { // the aliases that codegen may use to reach them. Its public accessors return // copies; construction completes all permitted lazy SSA materialization. type EmissionUniverse struct { - prog llssa.Program - goProg *ssa.Program - patches Patches - packages map[*ssa.Package]*preparedEmissionPackage - byTypes map[*types.Package]*preparedEmissionPackage - typesDup map[*types.Package]bool - byPath map[string]*preparedEmissionPackage - pathDup map[string]bool + prog llssa.Program + goProg *ssa.Program + patches Patches + completeRuntimeABI bool + packages map[*ssa.Package]*preparedEmissionPackage + byTypes map[*types.Package]*preparedEmissionPackage + typesDup map[*types.Package]bool + byPath map[string]*preparedEmissionPackage + pathDup map[string]bool functions []*ssa.Function required map[*ssa.Function]none @@ -100,7 +111,9 @@ type EmissionUniverse struct { materializedOwners map[*ssa.Function]map[*preparedEmissionPackage]none ownerStateErr error abiMethodReferences map[*ssa.Function]map[*ssa.Function]none - loweredCalls map[*ssa.Function]map[string]*ssa.Function + loweredCalls map[*ssa.Function]map[string]coroLoweredCallTarget + normalReturnBlocks map[*ssa.Function]map[*ssa.BasicBlock]none + foreignNoBlock map[*ssa.Function]CoroForeignNoBlockCertificate localGenericMu sync.Mutex localGenericTypes map[*types.Named]emissionLocalGenericType @@ -108,6 +121,23 @@ type EmissionUniverse struct { genericNamedTypes map[*types.Named]*types.Named } +// CoroForeignNoBlockCertificate is the immutable frontend proof attached to +// one exact C declaration by //llgo:coro noblock. ID is domain-separated and +// includes the frozen owner, physical symbol, and structural ABI signature. +// PhysicalSymbol and ABISignature are exposed only for diagnostics and audit; +// consumers must compare/use ID rather than reclassifying a declaration from +// either display field. +type CoroForeignNoBlockCertificate struct { + ID string + PhysicalSymbol string + ABISignature string +} + +type coroLoweredCallTarget struct { + target *ssa.Function + unwindOnly bool +} + // CoroIntrinsicCallSemantics is the frozen physical call-edge behavior of an // llgo compiler intrinsic. It deliberately says nothing about ordinary C/Go // functions and does not expose cl's private intrinsic opcode/name table. @@ -122,8 +152,35 @@ const ( // coroutine edge, although the exact SSA call site remains in the plan // digest and the intrinsic operation is still emitted by cl. CoroIntrinsicCallInlineNoSuspend + // CoroIntrinsicCallInlineWithLoweredCalls means cl erases the intrinsic + // declaration call, but the operation emits one or more ordinary runtime + // helper calls. Those calls are frozen separately in CoroLoweredCalls and + // therefore retain their own suspension and unwind effects. Consumers may + // elide only the intrinsic declaration edge, never the lowered helper edges. + CoroIntrinsicCallInlineWithLoweredCalls + // CoroIntrinsicCallInlineSuspend means cl erases the declaration call and + // emits a structured suspension in the current physical coroutine frame. + // The build analyzer seeds the owner with MayPark; there is no callable sync + // helper and no managed callee edge. + CoroIntrinsicCallInlineSuspend ) +// ElidesManagedCall reports whether cl removes the original SSA call to the +// intrinsic declaration. It does not imply that the complete lowered +// operation is no-suspend: InlineWithLoweredCalls carries its physical effects +// through the owner's exact frozen lowered-call set. +func (s CoroIntrinsicCallSemantics) ElidesManagedCall() bool { + return s == CoroIntrinsicCallInlineNoSuspend || s == CoroIntrinsicCallInlineWithLoweredCalls || + s == CoroIntrinsicCallInlineSuspend +} + +// SuspendsCurrentFrame reports the one intrinsic semantic that requires its +// owner to have a coroutine primary even though the declaration call itself is +// erased by frontend lowering. +func (s CoroIntrinsicCallSemantics) SuspendsCurrentFrame() bool { + return s == CoroIntrinsicCallInlineSuspend +} + type intrinsicWrapperKey struct { owner *ssa.Package intrinsic *ssa.Function @@ -146,8 +203,17 @@ type emissionLocalGenericType struct { // PrepareEmissionUniverse freezes package patch/skip selection and // materializes the exact SSA functions that cl can later request. It creates -// no LLVM package or function. +// no LLVM package or function. This compatibility entry point prepares an +// incomplete/report universe and therefore does not claim that the complete +// compiler-to-runtime ABI is available. func PrepareEmissionUniverse(prog llssa.Program, patches Patches, inputs []EmissionPackage) (*EmissionUniverse, error) { + return PrepareEmissionUniverseWithOptions(prog, patches, inputs, EmissionUniverseOptions{}) +} + +// PrepareEmissionUniverseWithOptions is PrepareEmissionUniverse with explicit +// whole-program construction contracts. Production active coroutine builds +// set CompleteRuntimeABI; unit/report universes deliberately leave it false. +func PrepareEmissionUniverseWithOptions(prog llssa.Program, patches Patches, inputs []EmissionPackage, options EmissionUniverseOptions) (*EmissionUniverse, error) { pathCounts := make(map[string]int, len(inputs)) for _, input := range inputs { if input.SSA != nil && input.SSA.Pkg != nil { @@ -158,6 +224,7 @@ func PrepareEmissionUniverse(prog llssa.Program, patches Patches, inputs []Emiss u := &EmissionUniverse{ prog: prog, patches: patches, + completeRuntimeABI: options.CompleteRuntimeABI, packages: make(map[*ssa.Package]*preparedEmissionPackage, len(inputs)), byTypes: make(map[*types.Package]*preparedEmissionPackage, len(inputs)*3), typesDup: make(map[*types.Package]bool), @@ -176,7 +243,9 @@ func PrepareEmissionUniverse(prog llssa.Program, patches Patches, inputs []Emiss callWrapInfo: make(map[*ssa.Function]intrinsicWrapperKey), syntheticKeys: make(map[*ssa.Function]string), abiMethodReferences: make(map[*ssa.Function]map[*ssa.Function]none), - loweredCalls: make(map[*ssa.Function]map[string]*ssa.Function), + loweredCalls: make(map[*ssa.Function]map[string]coroLoweredCallTarget), + normalReturnBlocks: make(map[*ssa.Function]map[*ssa.BasicBlock]none), + foreignNoBlock: make(map[*ssa.Function]CoroForeignNoBlockCertificate), linkIdentities: make(map[*ssa.Function]string), excluded: make(map[*ssa.Function]none), materialized: make(map[*ssa.Function]none), @@ -267,6 +336,21 @@ func PrepareEmissionUniverse(prog llssa.Program, patches Patches, inputs []Emiss u.byPath[pkgPath] = prepared } } + if options.CompleteRuntimeABI { + if prog == nil { + return nil, fmt.Errorf("prepare emission universe: complete runtime ABI requires an LLVM SSA program") + } + if u.pathDup[llssa.PkgRuntime] { + return nil, fmt.Errorf("prepare emission universe: complete runtime ABI has ambiguous package path %q", llssa.PkgRuntime) + } + runtimePkg := u.byPath[llssa.PkgRuntime] + if runtimePkg == nil { + return nil, fmt.Errorf("prepare emission universe: complete runtime ABI requires package %q", llssa.PkgRuntime) + } + if runtimePkg.metadataOnly { + return nil, fmt.Errorf("prepare emission universe: complete runtime ABI package %q cannot be metadata-only", llssa.PkgRuntime) + } + } // Link directives of every frontend package are now registered. Select // definitions in exactly the same alt-first order as @@ -341,9 +425,19 @@ func PrepareEmissionUniverse(prog llssa.Program, patches Patches, inputs []Emiss if err := u.freezeFunctionIdentities(); err != nil { return nil, err } + if err := u.freezeCoroForeignNoBlockCertificates(); err != nil { + return nil, err + } return u, nil } +// CompleteRuntimeABI reports whether construction froze the complete set of +// compiler-inserted runtime ABI edges. A false result is valid only for +// report-only or isolated frontend compilation. +func (u *EmissionUniverse) CompleteRuntimeABI() bool { + return u != nil && u.completeRuntimeABI +} + // Functions returns canonical required functions in deterministic order. func (u *EmissionUniverse) Functions() []*ssa.Function { if u == nil { @@ -455,7 +549,8 @@ func (u *EmissionUniverse) CoroLoweredCalls(owner *ssa.Function) ([]coro.SSALowe } byName := u.loweredCalls[owner] calls := make([]coro.SSALoweredCall, 0, len(byName)) - for logicalName, target := range byName { + for logicalName, frozen := range byName { + target := frozen.target if logicalName == "" || !utf8.ValidString(logicalName) || strings.IndexByte(logicalName, 0) >= 0 { return nil, fmt.Errorf("coroutine lowered-call owner %q has invalid logical name %q", owner.Name(), logicalName) } @@ -468,7 +563,11 @@ func (u *EmissionUniverse) CoroLoweredCalls(owner *ssa.Function) ([]coro.SSALowe if _, frozen := u.required[target]; !frozen { return nil, fmt.Errorf("coroutine lowered call %q in %q targets helper %q outside the frozen emission universe", logicalName, owner.Name(), target.Name()) } - calls = append(calls, coro.SSALoweredCall{LogicalName: logicalName, Target: target}) + calls = append(calls, coro.SSALoweredCall{ + LogicalName: logicalName, + Target: target, + UnwindOnly: frozen.unwindOnly, + }) } sort.Slice(calls, func(i, j int) bool { return calls[i].LogicalName < calls[j].LogicalName @@ -498,6 +597,13 @@ func (u *EmissionUniverse) ResolveCoroLoweredCall(owner *ssa.Function, logicalNa // helper in one owner are idempotent; resolving that identity to two exact // targets fails closed. func (u *EmissionUniverse) recordCoroLoweredCall(owner *ssa.Function, logicalName string, target *ssa.Function) error { + return u.recordCoroLoweredCallSite(owner, logicalName, target, false) +} + +// recordCoroLoweredCallSite freezes one physical helper-use class. A logical +// helper is unwind-only only when every occurrence in the owner is proven to +// be unwind-only; one normal-return-reachable occurrence conservatively wins. +func (u *EmissionUniverse) recordCoroLoweredCallSite(owner *ssa.Function, logicalName string, target *ssa.Function, unwindOnly bool) error { if owner == nil { return fmt.Errorf("prepare emission universe: lowered call has no owner") } @@ -521,15 +627,23 @@ func (u *EmissionUniverse) recordCoroLoweredCall(owner *ssa.Function, logicalNam if _, frozen := u.required[target]; !frozen { return fmt.Errorf("prepare emission universe: lowered call %q in %q targets helper %q outside the emission universe", logicalName, owner.Name(), target.Name()) } + if u.loweredCalls == nil { + u.loweredCalls = make(map[*ssa.Function]map[string]coroLoweredCallTarget) + } byName := u.loweredCalls[owner] if byName == nil { - byName = make(map[string]*ssa.Function) + byName = make(map[string]coroLoweredCallTarget) u.loweredCalls[owner] = byName } - if previous := byName[logicalName]; previous != nil && previous != target { - return fmt.Errorf("prepare emission universe: lowered call %q in %q resolves to both %q and %q", logicalName, owner.Name(), previous.Name(), target.Name()) + if previous, ok := byName[logicalName]; ok { + if previous.target != target { + return fmt.Errorf("prepare emission universe: lowered call %q in %q resolves to both %q and %q", logicalName, owner.Name(), previous.target.Name(), target.Name()) + } + previous.unwindOnly = previous.unwindOnly && unwindOnly + byName[logicalName] = previous + return nil } - byName[logicalName] = target + byName[logicalName] = coroLoweredCallTarget{target: target, unwindOnly: unwindOnly} return nil } @@ -652,6 +766,28 @@ func (u *EmissionUniverse) FunctionBackground(fn *ssa.Function) (background llss } } +// CoroForeignNoBlockCertificate returns the frozen declaration certificate for +// fn. The proof exists only for an exact emitted C declaration carrying the +// //llgo:coro noblock directive. Ordinary C declarations remain unclassified +// and therefore retain the conservative BlockForeign/WaitForeign boundary. +func (u *EmissionUniverse) CoroForeignNoBlockCertificate(fn *ssa.Function) (certificate CoroForeignNoBlockCertificate, certified bool, err error) { + if u == nil { + return CoroForeignNoBlockCertificate{}, false, fmt.Errorf("coroutine foreign noblock certificate: nil emission universe") + } + if fn == nil { + return CoroForeignNoBlockCertificate{}, false, fmt.Errorf("coroutine foreign noblock certificate: nil function") + } + canonical := u.canonicalAlias(fn) + if canonical == nil { + return CoroForeignNoBlockCertificate{}, false, fmt.Errorf("coroutine foreign noblock certificate: function has cyclic canonical aliases") + } + if _, required := u.required[canonical]; !required { + return CoroForeignNoBlockCertificate{}, false, fmt.Errorf("coroutine foreign noblock certificate: function %q is absent from the frozen emission universe", canonical.Name()) + } + certificate, certified = u.foreignNoBlock[canonical] + return certificate, certified, nil +} + // CoroIntrinsicSemantics reports whether fn is an exact frozen llgo compiler // intrinsic and, if so, its narrow coroutine call-edge semantics. The result // is recorded during universe construction and never inferred from the Go @@ -685,7 +821,7 @@ func (u *EmissionUniverse) CoroIntrinsicCallSiteSemantics(call ssa.CallInstructi return CoroIntrinsicCallUnsupported, intrinsic, err } semantics = coroIntrinsicCallSemantics(opcode) - if semantics != CoroIntrinsicCallInlineNoSuspend { + if !semantics.ElidesManagedCall() { return semantics, true, nil } direct, ok := call.(*ssa.Call) @@ -694,6 +830,12 @@ func (u *EmissionUniverse) CoroIntrinsicCallSiteSemantics(call ssa.CallInstructi "emission universe intrinsic call semantics: inline intrinsic %q must be an exact direct call", callee.Name(), ) } + if isCoroAtomicIntrinsic(opcode) { + if err := validateCoroAtomicIntrinsicCallSite(opcode, direct); err != nil { + return CoroIntrinsicCallUnsupported, true, err + } + return CoroIntrinsicCallInlineNoSuspend, true, nil + } switch opcode { case llgoCstr: args := direct.Common().Args @@ -709,6 +851,210 @@ func (u *EmissionUniverse) CoroIntrinsicCallSiteSemantics(call ssa.CallInstructi ) } return CoroIntrinsicCallInlineNoSuspend, true, nil + case llgoAdvance: + args := direct.Common().Args + if len(args) != 2 { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.advance call %q requires exactly two arguments", direct.String(), + ) + } + // context.advance passes these operands directly to Builder.Advance. + // Builder.Advance accepts an actual Go pointer or unsafe.Pointer and an + // LLVM integer GEP index; accepting a merely pointer-shaped named value + // here would disagree with that lowering's raw-type switch. + pointerType := types.Unalias(args[0].Type()) + switch pointerType := pointerType.(type) { + case *types.Pointer: + case *types.Basic: + if pointerType.Kind() != types.UnsafePointer { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.advance call %q requires a pointer first argument", direct.String(), + ) + } + default: + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.advance call %q requires a pointer first argument", direct.String(), + ) + } + offsetType, ok := types.Unalias(args[1].Type()).Underlying().(*types.Basic) + if !ok || offsetType.Info()&types.IsInteger == 0 { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.advance call %q requires an integer offset argument", direct.String(), + ) + } + results := direct.Common().Signature().Results() + if results == nil || results.Len() != 1 || !types.Identical(results.At(0).Type(), args[0].Type()) { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.advance call %q requires one result matching its pointer argument", direct.String(), + ) + } + return CoroIntrinsicCallInlineNoSuspend, true, nil + case llgoAllocaCStr: + args := direct.Common().Args + if len(args) != 1 { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.allocaCStr call %q requires exactly one string argument", direct.String(), + ) + } + argType, ok := types.Unalias(args[0].Type()).Underlying().(*types.Basic) + if !ok || argType.Kind() != types.String { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.allocaCStr call %q requires exactly one string argument", direct.String(), + ) + } + signature := direct.Common().Signature() + results := signature.Results() + if signature.Recv() != nil || signature.Variadic() || results == nil || results.Len() != 1 { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.allocaCStr call %q requires one *int8 result", direct.String(), + ) + } + resultPointer, ok := types.Unalias(results.At(0).Type()).Underlying().(*types.Pointer) + if !ok { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.allocaCStr call %q requires one *int8 result", direct.String(), + ) + } + resultElem, ok := types.Unalias(resultPointer.Elem()).Underlying().(*types.Basic) + if !ok || resultElem.Kind() != types.Int8 { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.allocaCStr call %q requires one *int8 result", direct.String(), + ) + } + if !u.CompleteRuntimeABI() { + // Isolated/report compilation retains the legacy rtFunc call and has + // no complete owner-scoped runtime-helper map. Do not elide the + // intrinsic declaration in that mode. + return CoroIntrinsicCallUnsupported, true, nil + } + helper, frozen, helperErr := u.ResolveCoroLoweredCall(direct.Parent(), "CStrCopy") + if helperErr != nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.allocaCStr call %q resolve frozen CStrCopy helper: %w", direct.String(), helperErr, + ) + } + if !frozen || helper == nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.allocaCStr call %q has no exact frozen CStrCopy lowered call", direct.String(), + ) + } + return CoroIntrinsicCallInlineWithLoweredCalls, true, nil + case llgoDeferData: + if len(direct.Common().Args) != 0 { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.deferData call %q requires no arguments", direct.String(), + ) + } + signature := direct.Common().Signature() + if signature == nil || signature.Recv() != nil || signature.Variadic() || (signature.Params() != nil && signature.Params().Len() != 0) { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.deferData call %q requires the exact func() unsafe.Pointer shape", direct.String(), + ) + } + results := signature.Results() + if results == nil || results.Len() != 1 { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.deferData call %q requires the exact func() unsafe.Pointer shape", direct.String(), + ) + } + result, ok := types.Unalias(results.At(0).Type()).Underlying().(*types.Basic) + if !ok || result.Kind() != types.UnsafePointer { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.deferData call %q requires the exact func() unsafe.Pointer shape", direct.String(), + ) + } + if !u.CompleteRuntimeABI() { + return CoroIntrinsicCallUnsupported, true, nil + } + helper, frozen, helperErr := u.ResolveCoroLoweredCall(direct.Parent(), "GetThreadDefer") + if helperErr != nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.deferData call %q resolve frozen GetThreadDefer helper: %w", direct.String(), helperErr, + ) + } + if !frozen || helper == nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.deferData call %q has no exact frozen GetThreadDefer lowered call", direct.String(), + ) + } + return CoroIntrinsicCallInlineWithLoweredCalls, true, nil + case llgoString: + owner := u.ownerOf(direct.Parent()) + if owner == nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.string call %q has no exact frozen owner", direct.String(), + ) + } + ctx, ctxErr := u.functionABIContext(direct.Parent(), owner) + if ctxErr != nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.string call %q build exact lowering context: %w", direct.String(), ctxErr, + ) + } + helperName, helperShapeErr := emissionStringIntrinsicHelper(ctx, direct) + if helperShapeErr != nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: %w", helperShapeErr, + ) + } + if !u.CompleteRuntimeABI() { + return CoroIntrinsicCallUnsupported, true, nil + } + helper, frozen, helperErr := u.ResolveCoroLoweredCall(direct.Parent(), helperName) + if helperErr != nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.string call %q resolve frozen %s helper: %w", direct.String(), helperName, helperErr, + ) + } + if !frozen || helper == nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.string call %q has no exact frozen %s lowered call", direct.String(), helperName, + ) + } + return CoroIntrinsicCallInlineWithLoweredCalls, true, nil + case llgoSigjmpbuf: + if err := validateCoroSigjmpIntrinsicCallSite(opcode, direct); err != nil { + return CoroIntrinsicCallUnsupported, true, err + } + return CoroIntrinsicCallInlineNoSuspend, true, nil + case llgoSigsetjmp, llgoSiglongjmp: + if err := validateCoroSigjmpIntrinsicCallSite(opcode, direct); err != nil { + return CoroIntrinsicCallUnsupported, true, err + } + if !u.coroUsesRuntimeSigjmpHelpers() { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: legacy llgo setjmp/longjmp call %q lowers directly to a target C leaf and requires a non-legacy coroutine PanicABI", direct.String(), + ) + } + if !u.CompleteRuntimeABI() { + return CoroIntrinsicCallUnsupported, true, nil + } + helperName := "Sigsetjmp" + if opcode == llgoSiglongjmp { + helperName = "Siglongjmp" + } + helper, frozen, helperErr := u.ResolveCoroLoweredCall(direct.Parent(), helperName) + if helperErr != nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: legacy %s call %q resolve frozen runtime helper: %w", helperName, direct.String(), helperErr, + ) + } + if !frozen || helper == nil { + return CoroIntrinsicCallUnsupported, true, fmt.Errorf( + "emission universe intrinsic call semantics: legacy %s call %q has no exact frozen lowered call", helperName, direct.String(), + ) + } + return CoroIntrinsicCallInlineWithLoweredCalls, true, nil + case llgoFuncAddr: + if _, _, err := u.validateCoroFuncAddrCallSite(direct); err != nil { + return CoroIntrinsicCallUnsupported, true, err + } + return CoroIntrinsicCallInlineNoSuspend, true, nil + case llgoCoroPark: + if err := validateCoroParkIntrinsicCallSite(direct); err != nil { + return CoroIntrinsicCallUnsupported, true, err + } + return CoroIntrinsicCallInlineSuspend, true, nil default: return CoroIntrinsicCallUnsupported, true, fmt.Errorf( "emission universe intrinsic call semantics: inline intrinsic %q has no exact call-site verifier", callee.Name(), @@ -716,6 +1062,94 @@ func (u *EmissionUniverse) CoroIntrinsicCallSiteSemantics(call ssa.CallInstructi } } +// CoroRawFunctionAddressCallArgument reports the one exact call argument that +// funcAddr consumes as a raw static entry address. Unlike an ordinary +// MakeInterface, this operand is inspected structurally and no interface value +// is emitted. Consumers use this frozen fact to avoid forcing the target into +// Dispatch representation solely because x/tools SSA inserted the transient +// MakeInterface node. +func (u *EmissionUniverse) CoroRawFunctionAddressCallArgument(call ssa.CallInstruction, argument int) (bool, error) { + if call == nil || call.Common() == nil || argument < 0 || argument >= len(call.Common().Args) { + return false, nil + } + callee := call.Common().StaticCallee() + if callee == nil { + return false, nil + } + opcode, intrinsic, err := u.coroIntrinsicOpcode(callee) + if err != nil || !intrinsic || opcode != llgoFuncAddr { + return false, err + } + direct, ok := call.(*ssa.Call) + if !ok || direct.Common() == nil || direct.Common().IsInvoke() { + return false, fmt.Errorf("emission universe raw function address: llgo.funcAddr must be an exact direct call") + } + if _, _, err := u.validateCoroFuncAddrCallSite(direct); err != nil { + return false, err + } + return argument == 0, nil +} + +func (u *EmissionUniverse) validateCoroFuncAddrCallSite(direct *ssa.Call) (*ssa.MakeInterface, *ssa.Function, error) { + if direct == nil || direct.Common() == nil || direct.Common().IsInvoke() { + return nil, nil, fmt.Errorf("emission universe intrinsic call semantics: llgo.funcAddr must be an exact direct call") + } + args := direct.Common().Args + if len(args) != 1 { + return nil, nil, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.funcAddr call %q requires exactly one argument", direct.String(), + ) + } + signature := direct.Common().Signature() + if signature == nil || signature.Recv() != nil || signature.Variadic() || signature.Params() == nil || signature.Params().Len() != 1 { + return nil, nil, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.funcAddr call %q requires the exact func(any) unsafe.Pointer shape", direct.String(), + ) + } + parameterInterface, ok := types.Unalias(signature.Params().At(0).Type()).Underlying().(*types.Interface) + if !ok || !parameterInterface.Empty() { + return nil, nil, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.funcAddr call %q requires the exact func(any) unsafe.Pointer shape", direct.String(), + ) + } + results := signature.Results() + if results == nil || results.Len() != 1 { + return nil, nil, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.funcAddr call %q requires the exact func(any) unsafe.Pointer shape", direct.String(), + ) + } + result, ok := types.Unalias(results.At(0).Type()).Underlying().(*types.Basic) + if !ok || result.Kind() != types.UnsafePointer { + return nil, nil, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.funcAddr call %q requires the exact func(any) unsafe.Pointer shape", direct.String(), + ) + } + boxed, ok := args[0].(*ssa.MakeInterface) + if !ok { + return nil, nil, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.funcAddr call %q requires a direct MakeInterface function operand", direct.String(), + ) + } + target, ok := boxed.X.(*ssa.Function) + if !ok || len(target.FreeVars) != 0 { + return nil, nil, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.funcAddr call %q requires MakeInterface{X:*ssa.Function} without captured state", direct.String(), + ) + } + refs := boxed.Referrers() + if refs == nil || len(*refs) != 1 || (*refs)[0] != direct { + return nil, nil, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.funcAddr call %q requires its MakeInterface operand to have this exact sole consumer", direct.String(), + ) + } + if canonical, resolved := u.Resolve(target); !resolved || canonical == nil { + return nil, nil, fmt.Errorf( + "emission universe intrinsic call semantics: llgo.funcAddr call %q targets function %q outside the frozen emission universe", direct.String(), target.Name(), + ) + } + return boxed, target, nil +} + func (u *EmissionUniverse) coroIntrinsicOpcode(fn *ssa.Function) (opcode int, intrinsic bool, err error) { _, classified, err := u.FunctionBackground(fn) if err != nil { @@ -754,16 +1188,194 @@ func (u *EmissionUniverse) coroIntrinsicOpcode(fn *ssa.Function) (opcode int, in } func coroIntrinsicCallSemantics(opcode int) CoroIntrinsicCallSemantics { + if isCoroAtomicIntrinsic(opcode) { + return CoroIntrinsicCallInlineNoSuspend + } switch opcode { case llgoCstr: // cstr accepts only a compile-time string literal and lowers directly // to an LLVM constant C string pointer. return CoroIntrinsicCallInlineNoSuspend + case llgoAdvance: + // advance lowers directly to one LLVM GEP after its exact operand and + // result shape has been verified at the physical call site. + return CoroIntrinsicCallInlineNoSuspend + case llgoAllocaCStr: + // allocaCStr lowers its string length arithmetic and storage directly, + // then calls runtime.CStrCopy. The intrinsic declaration disappears, but + // the exact CStrCopy edge is retained in the owner's lowered-call set. + return CoroIntrinsicCallInlineWithLoweredCalls + case llgoDeferData: + // deferData removes the intrinsic declaration but emits the exact + // runtime.GetThreadDefer call owned by the surrounding function. + return CoroIntrinsicCallInlineWithLoweredCalls + case llgoString: + // string replaces the intrinsic declaration with exactly one of + // runtime.StringFromCStr or runtime.StringFrom based on the frozen + // frontend variadic shape. + return CoroIntrinsicCallInlineWithLoweredCalls + case llgoSigjmpbuf: + // sigjmpbuf is a target-sized LLVM alloca and has no callable edge. + return CoroIntrinsicCallInlineNoSuspend + case llgoSigsetjmp, llgoSiglongjmp: + // Native legacy PanicABI replaces these declarations with the exact + // runtime C-linkname leaves. WASM and explicit embedded targets fail + // closed until their non-legacy PanicABI is selected. + return CoroIntrinsicCallInlineWithLoweredCalls + case llgoFuncAddr: + // funcAddr structurally unwraps one exact MakeInterface{X:*ssa.Function} + // and emits the selected raw function entry address directly. + return CoroIntrinsicCallInlineNoSuspend + case llgoCoroPark: + return CoroIntrinsicCallInlineSuspend default: return CoroIntrinsicCallUnsupported } } +func validateCoroParkIntrinsicCallSite(call *ssa.Call) error { + if call == nil || call.Common() == nil { + return fmt.Errorf("llgo.coroPark requires an exact direct call") + } + common := call.Common() + if common.IsInvoke() || len(common.Args) != 2 { + return fmt.Errorf("llgo.coroPark call %q requires exactly (pointer, uint32) arguments", call.String()) + } + signature := common.Signature() + if signature == nil || signature.Recv() != nil || signature.Variadic() || + (signature.Results() != nil && signature.Results().Len() != 0) || + signature.Params() == nil || signature.Params().Len() != 2 { + return fmt.Errorf("llgo.coroPark call %q requires the exact func(pointer, uint32) shape", call.String()) + } + pointerLike := func(typ types.Type) bool { + typ = types.Unalias(typ) + if _, ok := typ.Underlying().(*types.Pointer); ok { + return true + } + basic, ok := typ.Underlying().(*types.Basic) + return ok && basic.Kind() == types.UnsafePointer + } + uint32Like := func(typ types.Type) bool { + basic, ok := types.Unalias(typ).Underlying().(*types.Basic) + return ok && basic.Kind() == types.Uint32 + } + if !pointerLike(common.Args[0].Type()) || !pointerLike(signature.Params().At(0).Type()) || + !uint32Like(common.Args[1].Type()) || !uint32Like(signature.Params().At(1).Type()) { + return fmt.Errorf("llgo.coroPark call %q requires the exact func(pointer, uint32) shape", call.String()) + } + return nil +} + +func isCoroAtomicIntrinsic(opcode int) bool { + return opcode == llgoAtomicLoad || opcode == llgoAtomicStore || opcode == llgoAtomicCmpXchg || + opcode == llgoAtomicCmpXchgOK || opcode == llgoAtomicAddReturnNew || + opcode >= llgoAtomicOpBase && opcode <= llgoAtomicOpLast +} + +func validateCoroAtomicIntrinsicCallSite(opcode int, direct *ssa.Call) error { + if direct == nil || direct.Common() == nil || direct.Common().IsInvoke() { + return fmt.Errorf("emission universe intrinsic call semantics: llgo atomic intrinsic must be an exact direct call") + } + common := direct.Common() + signature := common.Signature() + if signature == nil || signature.Recv() != nil || signature.Variadic() || signature.Params() == nil { + return fmt.Errorf("emission universe intrinsic call semantics: llgo atomic call %q has an invalid declaration shape", direct.String()) + } + params := signature.Params() + results := signature.Results() + if params.Len() == 0 { + return fmt.Errorf("emission universe intrinsic call semantics: llgo atomic call %q has no pointer operand", direct.String()) + } + pointer, ok := types.Unalias(params.At(0).Type()).Underlying().(*types.Pointer) + if !ok || !emissionIsAtomicScalarType(pointer.Elem()) { + return fmt.Errorf("emission universe intrinsic call semantics: llgo atomic call %q requires a pointer to an integer or unsafe.Pointer value", direct.String()) + } + elem := pointer.Elem() + matchingParam := func(index int) bool { + return index >= 0 && index < params.Len() && types.Identical(params.At(index).Type(), elem) + } + matchingResult := func(index int) bool { + return results != nil && index >= 0 && index < results.Len() && types.Identical(results.At(index).Type(), elem) + } + boolResult := func(index int) bool { + return results != nil && index >= 0 && index < results.Len() && emissionIsBasicKind(results.At(index).Type(), types.Bool) + } + noResults := results == nil || results.Len() == 0 + + valid := false + switch { + case opcode == llgoAtomicLoad: + valid = len(common.Args) == 1 && params.Len() == 1 && results != nil && results.Len() == 1 && matchingResult(0) + case opcode == llgoAtomicStore: + valid = len(common.Args) == 2 && params.Len() == 2 && matchingParam(1) && noResults + case opcode == llgoAtomicCmpXchg: + valid = len(common.Args) == 3 && params.Len() == 3 && matchingParam(1) && matchingParam(2) && results != nil && results.Len() == 2 && matchingResult(0) && boolResult(1) + case opcode == llgoAtomicCmpXchgOK: + valid = len(common.Args) == 3 && params.Len() == 3 && matchingParam(1) && matchingParam(2) && results != nil && results.Len() == 1 && boolResult(0) + case opcode == llgoAtomicAddReturnNew || opcode >= llgoAtomicOpBase && opcode <= llgoAtomicOpLast: + valid = len(common.Args) == 2 && params.Len() == 2 && matchingParam(1) && results != nil && results.Len() == 1 && matchingResult(0) + } + if !valid { + return fmt.Errorf("emission universe intrinsic call semantics: llgo atomic call %q does not match opcode %d's exact pointer/value/result shape", direct.String(), opcode) + } + return nil +} + +func emissionIsAtomicScalarType(typ types.Type) bool { + basic, ok := types.Unalias(typ).Underlying().(*types.Basic) + return ok && (basic.Info()&types.IsInteger != 0 || basic.Kind() == types.UnsafePointer) +} + +func (u *EmissionUniverse) coroUsesRuntimeSigjmpHelpers() bool { + if u == nil || u.prog == nil || u.prog.Target() == nil { + return false + } + target := u.prog.Target() + return target.GOARCH != "wasm" && target.Target == "" +} + +func validateCoroSigjmpIntrinsicCallSite(opcode int, direct *ssa.Call) error { + if direct == nil || direct.Common() == nil || direct.Common().IsInvoke() { + return fmt.Errorf("emission universe intrinsic call semantics: llgo setjmp/longjmp intrinsic must be an exact direct call") + } + common := direct.Common() + signature := common.Signature() + if signature == nil || signature.Recv() != nil || signature.Variadic() { + return fmt.Errorf("emission universe intrinsic call semantics: llgo setjmp/longjmp call %q has an invalid declaration shape", direct.String()) + } + params := signature.Params() + results := signature.Results() + switch opcode { + case llgoSigjmpbuf: + if len(common.Args) != 0 || params != nil && params.Len() != 0 || results == nil || results.Len() != 1 || !emissionIsUnsafePointerType(results.At(0).Type()) { + return fmt.Errorf("emission universe intrinsic call semantics: llgo.sigjmpbuf call %q requires the exact func() unsafe.Pointer shape", direct.String()) + } + case llgoSigsetjmp: + if len(common.Args) != 2 || params == nil || params.Len() != 2 || results == nil || results.Len() != 1 || + !emissionIsUnsafePointerType(params.At(0).Type()) || !emissionIsBasicKind(params.At(1).Type(), types.Int32) || !emissionIsBasicKind(results.At(0).Type(), types.Int32) { + return fmt.Errorf("emission universe intrinsic call semantics: llgo.sigsetjmp call %q requires the exact func(unsafe.Pointer, int32) int32 shape", direct.String()) + } + case llgoSiglongjmp: + if len(common.Args) != 2 || params == nil || params.Len() != 2 || results != nil && results.Len() != 0 || + !emissionIsUnsafePointerType(params.At(0).Type()) || !emissionIsBasicKind(params.At(1).Type(), types.Int32) { + return fmt.Errorf("emission universe intrinsic call semantics: llgo.siglongjmp call %q requires the exact func(unsafe.Pointer, int32) shape", direct.String()) + } + default: + return fmt.Errorf("emission universe intrinsic call semantics: unknown llgo setjmp/longjmp opcode %d", opcode) + } + return nil +} + +func emissionIsUnsafePointerType(typ types.Type) bool { + basic, ok := types.Unalias(typ).Underlying().(*types.Basic) + return ok && basic.Kind() == types.UnsafePointer +} + +func emissionIsBasicKind(typ types.Type, kind types.BasicKind) bool { + basic, ok := types.Unalias(typ).Underlying().(*types.Basic) + return ok && basic.Kind() == kind +} + func (u *EmissionUniverse) physicalName(ownerSSA *ssa.Package, fn *ssa.Function, legacy string) (string, error) { if u == nil || fn == nil { return legacy, nil @@ -1983,6 +2595,9 @@ func (u *EmissionUniverse) materializeFunctionForOwner(fn *ssa.Function, owner * } for _, block := range fn.Blocks { for _, instr := range block.Instrs { + if err := u.materializeLoweredRuntimeHelpers(ctx, fn, owner, emissionState, instr); err != nil { + return err + } if call, ok := instr.(ssa.CallInstruction); ok { roots, err := u.callValueRoots(ctx, call.Common()) if err != nil { @@ -3008,6 +3623,109 @@ func (u *EmissionUniverse) freezeFunctionIdentities() error { return nil } +type coroForeignPhysicalABI struct { + symbol string + signature string +} + +// freezeCoroForeignNoBlockCertificates binds source directives to the same +// exact physical identities already frozen for codegen. It deliberately scans +// every required C declaration before accepting a certificate: if another +// required declaration names the same physical symbol with a different ABI +// signature, the proof fails closed instead of blessing one guessed spelling. +func (u *EmissionUniverse) freezeCoroForeignNoBlockCertificates() error { + abiByFunction := make(map[*ssa.Function]coroForeignPhysicalABI) + signaturesBySymbol := make(map[string]map[string]none) + for _, fn := range u.functions { + owners := u.sortedUseOwners(fn) + var abi coroForeignPhysicalABI + haveABI := false + for _, owner := range owners { + key := u.finalKeys[emissionFunctionOwnerKey{function: fn, owner: owner}] + ftype, symbol, signature, ok := splitManagedSymbolKey(key) + if !ok || ftype != cFunc { + continue + } + candidate := coroForeignPhysicalABI{symbol: symbol, signature: signature} + if haveABI && candidate != abi { + return fmt.Errorf("prepare emission universe: C declaration %q has owner-dependent physical ABI while freezing coroutine noblock metadata", fn.Name()) + } + abi, haveABI = candidate, true + } + if !haveABI { + continue + } + abiByFunction[fn] = abi + signatures := signaturesBySymbol[abi.symbol] + if signatures == nil { + signatures = make(map[string]none) + signaturesBySymbol[abi.symbol] = signatures + } + signatures[abi.signature] = none{} + } + + for _, fn := range u.functions { + annotated, err := coroForeignNoBlockDirective(fn) + if err != nil { + return fmt.Errorf("prepare emission universe: coroutine noblock directive on %q: %w", fn.Name(), err) + } + if !annotated { + continue + } + abi, ok := abiByFunction[fn] + if !ok { + return fmt.Errorf("prepare emission universe: //llgo:coro noblock on %q requires an exact frozen C declaration", fn.Name()) + } + if signatures := signaturesBySymbol[abi.symbol]; len(signatures) != 1 { + return fmt.Errorf("prepare emission universe: //llgo:coro noblock physical symbol %q has conflicting frozen ABI signatures", abi.symbol) + } + linkIdentity, ok := u.linkIdentities[fn] + if !ok || linkIdentity == "" { + return fmt.Errorf("prepare emission universe: //llgo:coro noblock on %q has no frozen link identity", fn.Name()) + } + u.foreignNoBlock[fn] = CoroForeignNoBlockCertificate{ + ID: framedEmissionKey( + "llgo-coro-foreign-noblock-v0", + linkIdentity, + abi.symbol, + abi.signature, + ), + PhysicalSymbol: abi.symbol, + ABISignature: abi.signature, + } + } + return nil +} + +func coroForeignNoBlockDirective(fn *ssa.Function) (bool, error) { + if fn == nil { + return false, nil + } + decl, _ := fn.Syntax().(*ast.FuncDecl) + if decl == nil || decl.Doc == nil { + return false, nil + } + found := false + for _, comment := range decl.Doc.List { + if comment == nil { + continue + } + line := strings.TrimSpace(comment.Text) + switch line { + case "//llgo:coro noblock", "// llgo:coro noblock": + if found { + return false, fmt.Errorf("duplicate //llgo:coro noblock directive") + } + found = true + default: + if strings.HasPrefix(line, "//llgo:coro") || strings.HasPrefix(line, "// llgo:coro") { + return false, fmt.Errorf("unsupported directive %q", line) + } + } + } + return found, nil +} + func (u *EmissionUniverse) freezeManagedPhysicalNameCollisions() { // Linkonce definitions from different use-site modules meet in one linker // namespace. Grouping by the emission owner would therefore miss the most diff --git a/cl/import.go b/cl/import.go index 7b07fe97b2..39f41a79a5 100644 --- a/cl/import.go +++ b/cl/import.go @@ -575,6 +575,9 @@ const ( llgoAtomicCmpXchgOK = llgoInstrBase + 0x45 llgoAtomicAddReturnNew = llgoInstrBase + 0x46 llgoBoolToUint8 = llgoInstrBase + 0x47 + // llgoCoroPark is a compiler-owned stack-cut operation. It is lowered only + // in the current physical coroutine frame and has no callable sync body. + llgoCoroPark = llgoInstrBase + 0x48 llgoAtomicOpLast = llgoAtomicOpBase + int(llssa.OpUMin) ) diff --git a/cl/instr.go b/cl/instr.go index 836ac92ee0..3a92902d5c 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -600,6 +600,7 @@ var llgoInstrs = map[string]int{ "skip": llgoSkip, "syscall": llgoSyscall, "boolToUint8": llgoBoolToUint8, + "coroPark": llgoCoroPark, "pystr": llgoPyStr, "pyList": llgoPyList, "pyTuple": llgoPyTuple, @@ -2084,6 +2085,12 @@ func (p *context) callEx(b llssa.Builder, act llssa.DoAction, call *ssa.CallComm ret = b.Do(act, llssa.Nil, func(b llssa.Builder, _ llssa.Expr, args ...llssa.Expr) llssa.Expr { return p.boolToUint8(b, args) }, args...) + case llgoCoroPark: + if act != llssa.Call || ds != nil { + panic("llgo.coroPark requires an exact direct call") + } + args := p.compileValues(b, args, kind) + p.compileCoroPark(b, args) case llgoUnreachable: // func unreachable() b.Unreachable() case llgoAtomicLoad: diff --git a/internal/build/build.go b/internal/build/build.go index e26482a627..0c6097fc1a 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -133,17 +133,19 @@ type CoroPlanInput struct { Program *ssa.Program EmissionUniverse *coro.SSAEmissionUniverse - resolveFunction func(*ssa.Function) (*ssa.Function, bool) - augmentFunctionIDs func(coro.FunctionIDConfig) coro.FunctionIDConfig - functionBackground func(*ssa.Function) (llssa.Background, bool, error) - intrinsicCallSemantics func(ssa.CallInstruction) (cl.CoroIntrinsicCallSemantics, bool, error) - demandReferences func(*ssa.Function) ([]*ssa.Function, error) - loweredCalls func(*ssa.Function) ([]coro.SSALoweredCall, error) - requiredRoots coro.Roots - requiredPlain map[*ssa.Function]struct{} - requiredDirectPlain []requiredCoroDirectPlainCallArgument - requiredClosedDynamic map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate - recordAnalysis func(*coro.SSAPlan) + resolveFunction func(*ssa.Function) (*ssa.Function, bool) + augmentFunctionIDs func(coro.FunctionIDConfig) coro.FunctionIDConfig + functionBackground func(*ssa.Function) (llssa.Background, bool, error) + foreignNoBlock func(*ssa.Function) (cl.CoroForeignNoBlockCertificate, bool, error) + intrinsicCallSemantics func(ssa.CallInstruction) (cl.CoroIntrinsicCallSemantics, bool, error) + rawFunctionAddressCallArgument func(ssa.CallInstruction, int) (bool, error) + demandReferences func(*ssa.Function) ([]*ssa.Function, error) + loweredCalls func(*ssa.Function) ([]coro.SSALoweredCall, error) + requiredRoots coro.Roots + requiredPlain map[*ssa.Function]struct{} + requiredDirectPlain []requiredCoroDirectPlainCallArgument + requiredClosedDynamic map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate + recordAnalysis func(*coro.SSAPlan) } type coroCallArgumentKey struct { @@ -190,7 +192,7 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. // SSA body. It does not by itself prove that the foreign operation is // nonblocking. Preserve an explicit known/unknown-foreign effect summary; // otherwise use the conservative unknown-foreign boundary. - if in.functionBackground != nil || config.ClassifyFunction != nil { + if in.functionBackground != nil || in.foreignNoBlock != nil || config.ClassifyFunction != nil { classify := config.ClassifyFunction config.ClassifyFunction = func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { var policy coro.SSAFunctionPolicy @@ -209,6 +211,39 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. } frontendC = classified && background == llssa.InC } + var certificate cl.CoroForeignNoBlockCertificate + certified := false + if in.foreignNoBlock != nil { + certificate, certified, err = in.foreignNoBlock(fn) + if err != nil { + return coro.SSAFunctionPolicy{}, fmt.Errorf("classify frozen frontend foreign noblock certificate for %q: %w", fn.Name(), err) + } + } + if requested := policy.ForeignNoBlockCertificate; requested != "" { + if !certified { + return coro.SSAFunctionPolicy{}, fmt.Errorf("builder cannot certify foreign function %q without exact frozen frontend noblock metadata", fn.Name()) + } + if requested != certificate.ID { + return coro.SSAFunctionPolicy{}, fmt.Errorf("builder foreign noblock certificate for %q conflicts with the frozen frontend proof", fn.Name()) + } + } + if certified { + if !frontendC { + return coro.SSAFunctionPolicy{}, fmt.Errorf("frozen foreign noblock certificate for %q does not name a frontend C declaration", fn.Name()) + } + if policy.Effect != coro.NoSuspend || policy.Exec != 0 || policy.NeedsDispatch || + policy.OverrideExternal && policy.External != coro.ExternalKnown { + return coro.SSAFunctionPolicy{}, fmt.Errorf("frontend C declaration %q conflicts with its frozen foreign noblock certificate", fn.Name()) + } + policy.IgnoreBody = true + policy.External = coro.ExternalKnown + policy.OverrideExternal = true + // A noblock proof is not an async-signal-safety proof. Preserve + // IRQUnsafe in the plan/digest while removing only the opaque + // BlockForeign/WaitForeign boundary. + policy.Exec = coro.IRQUnsafe + policy.ForeignNoBlockCertificate = certificate.ID + } if policy.IgnoreBody && !frontendC { return coro.SSAFunctionPolicy{}, fmt.Errorf("builder cannot ignore the SSA body of non-C function %q", fn.Name()) } @@ -226,6 +261,50 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. return policy, nil } } + // A structured coroutine intrinsic has no managed callee edge: cl replaces + // its declaration call with a suspend in the owner's exact frame. Seed that + // physical effect from the same frozen call-site semantics used to elide the + // declaration, so synchronous source callers are transparently coroutine + // primary bodies and the plan digest records both the owner effect and site. + if in.intrinsicCallSemantics != nil { + classify := config.ClassifyFunction + config.ClassifyFunction = func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + var policy coro.SSAFunctionPolicy + var err error + if classify != nil { + policy, err = classify(fn) + if err != nil { + return coro.SSAFunctionPolicy{}, err + } + } + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(*ssa.Call) + if !ok { + continue + } + rawCallee := call.Common().StaticCallee() + if rawCallee == nil { + continue + } + if _, frozen := in.ResolveFunction(rawCallee); !frozen { + // Frontend-elided noinit declarations (notably unsafe.init) + // are intentionally outside the frozen emission universe and + // carry no structured intrinsic effect. + continue + } + semantics, intrinsic, err := in.intrinsicCallSemantics(call) + if err != nil { + return coro.SSAFunctionPolicy{}, fmt.Errorf("classify frozen intrinsic effect in %q: %w", fn.Name(), err) + } + if intrinsic && semantics.SuspendsCurrentFrame() { + policy.Effect = policy.Effect.Join(coro.MayPark) + } + } + } + return policy, nil + } + } if len(in.requiredPlain) != 0 { classify := config.ClassifyFunction config.ClassifyFunction = func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { @@ -243,7 +322,13 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. if policy.Effect != coro.NoSuspend { return coro.SSAFunctionPolicy{}, fmt.Errorf("compiler runtime ABI function %q conflicts with required no-suspend policy: %s", fn.Name(), policy.Effect) } - const supportedExec = coro.MayUnwind | coro.NeedsCleanupFrame + // IRQUnsafe is an entry-context restriction, not a requirement for a + // second physical body. Compiler/runtime ABI helpers execute on the + // ordinary scheduler/executor stack, never as an IRQ root, so retain + // the bit in the frozen plan while keeping the exact required-plain + // implementation. ThreadAffine and opaque/blocking execution remain + // rejected until their scheduler protocols exist. + const supportedExec = coro.MayUnwind | coro.NeedsCleanupFrame | coro.IRQUnsafe if unsupported := policy.Exec &^ supportedExec; unsupported != 0 { return coro.SSAFunctionPolicy{}, fmt.Errorf("compiler runtime ABI function %q conflicts with required plain execution policy: %s", fn.Name(), unsupported) } @@ -283,7 +368,7 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. if err != nil { return false, fmt.Errorf("classify frozen intrinsic call in %q: %w", caller.Name(), err) } - frontendElided = intrinsic && semantics == cl.CoroIntrinsicCallInlineNoSuspend + frontendElided = intrinsic && semantics.ElidesManagedCall() } if classifyElided != nil { requested, err := classifyElided(caller, call) @@ -296,6 +381,29 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. } return frontendElided, nil } + if in.rawFunctionAddressCallArgument != nil || config.ClassifyRawFunctionAddressCallArgument != nil { + classifyRawAddress := config.ClassifyRawFunctionAddressCallArgument + config.ClassifyRawFunctionAddressCallArgument = func(caller *ssa.Function, call ssa.CallInstruction, argument int) (bool, error) { + compilerRequired := false + var err error + if in.rawFunctionAddressCallArgument != nil { + compilerRequired, err = in.rawFunctionAddressCallArgument(call, argument) + if err != nil { + return false, fmt.Errorf("classify frozen raw function-address argument %d in %q: %w", argument, caller.Name(), err) + } + } + if classifyRawAddress != nil { + requested, err := classifyRawAddress(caller, call, argument) + if err != nil { + return false, err + } + if requested && !compilerRequired { + return false, fmt.Errorf("builder cannot authorize raw function-address lowering for non-compiler call argument %d in %q", argument, caller.Name()) + } + } + return compilerRequired, nil + } + } if len(in.requiredDirectPlain) != 0 || config.ClassifyDirectPlainCallArgument != nil { required := make(map[coroCallArgumentKey]struct{}, len(in.requiredDirectPlain)) for _, use := range in.requiredDirectPlain { @@ -435,7 +543,11 @@ func sameExactCoroLoweredCalls(left, right []coro.SSALoweredCall) bool { if len(left) != len(right) { return false } - byName := make(map[string]*ssa.Function, len(left)) + type exactLoweredCall struct { + target *ssa.Function + unwindOnly bool + } + byName := make(map[string]exactLoweredCall, len(left)) for _, call := range left { if call.LogicalName == "" || call.Target == nil { return false @@ -443,10 +555,11 @@ func sameExactCoroLoweredCalls(left, right []coro.SSALoweredCall) bool { if _, duplicate := byName[call.LogicalName]; duplicate { return false } - byName[call.LogicalName] = call.Target + byName[call.LogicalName] = exactLoweredCall{target: call.Target, unwindOnly: call.UnwindOnly} } for _, call := range right { - if call.LogicalName == "" || call.Target == nil || byName[call.LogicalName] != call.Target { + frozen, ok := byName[call.LogicalName] + if call.LogicalName == "" || call.Target == nil || !ok || frozen.target != call.Target || frozen.unwindOnly != call.UnwindOnly { return false } delete(byName, call.LogicalName) @@ -597,8 +710,10 @@ type Config struct { // EnableCoroProgramBootstrapRun activates the production v1 bootstrap // driver. It requires EnableCoroProgramBootstrapABI, emits a compiler-owned // LLVM coroutine factory, and replaces only the legacy init/main calls in - // the platform entry. Keeping this separate preserves the descriptor-only - // ABI gate as an independently testable and reversible boundary. + // the platform entry. This is also the first scheduler ABI that accepts + // NeedsPreempt and emits conditional poll/yield handoffs. Keeping it separate + // preserves the descriptor-only ABI gate as an independently testable and + // reversible boundary. EnableCoroProgramBootstrapRun bool CoroPlanBuilder CoroPlanBuilder CoroPlanObserver CoroPlanObserver @@ -723,6 +838,22 @@ func Do(args []string, conf *Config) ([]Package, error) { if conf.AbiMode == cabi.ModeAllFunc { tags += ",llgo_abi_2" } + if conf.EnableCoroProgramBootstrapRun { + // The stackless runtime does not yet have a RawCritical bridge that can + // turn a synchronous hardware fault into a G-owned panic completion. + // Exclude the legacy pthread-TLS/SJLJ SIGSEGV recovery hook instead of + // admitting a signal callback that can allocate, block, or retain the + // native signal stack. Language-level nil/bounds/divide checks remain + // explicit compiler operations. + tags += ",llgo_coro" + } + gcTags, err := targetGCBuildTags(export.GC) + if err != nil { + return nil, err + } + if len(gcTags) != 0 { + tags += "," + strings.Join(gcTags, ",") + } if conf.Tags != "" { tags += "," + conf.Tags } @@ -1017,6 +1148,17 @@ func Do(args []string, conf *Config) ([]Package, error) { return allPkgs, nil } +func targetGCBuildTags(gc string) ([]string, error) { + switch gc { + case "", "precise", "conservative": + return nil, nil + case "leaking", "none": + return []string{"nogc"}, nil + default: + return nil, fmt.Errorf("unsupported target GC capability %q", gc) + } +} + func buildCoroPlan(ctx *context, packages ...*aPackage) error { if ctx == nil || ctx.buildConf == nil { return nil @@ -1054,10 +1196,28 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { } analyzedPlans := make(map[*coro.SSAPlan]struct{}) var analyzedPlansMu sync.Mutex - requiredRoots, requiredPlain, requiredDirectPlain, requiredClosedDynamic, err := requiredCoroProgramRuntimePlan(ctx) + var requiredRoots coro.Roots + var requiredPlain map[*ssa.Function]struct{} + var requiredDirectPlain []requiredCoroDirectPlainCallArgument + var requiredClosedDynamic map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate + if ctx.coroEmission != nil && ctx.coroEmission.CompleteRuntimeABI() { + // Compiler-owned runtime edges belong only to a frozen whole-program + // universe containing the exact LLGo runtime package. Isolated frontend + // fixtures and report universes intentionally remain incomplete; making + // them resolve production runtime roots would guess bodies outside their + // declared emission universe. Real Do builds pass all packages above and + // therefore retain the fail-closed complete-runtime path. + var err error + requiredRoots, requiredPlain, requiredDirectPlain, requiredClosedDynamic, err = requiredCoroProgramRuntimePlan(ctx) + if err != nil { + return err + } + } + managedEntryRoots, err := requiredCoroProgramManagedEntryRoots(ctx) if err != nil { return err } + requiredRoots = append(requiredRoots, managedEntryRoots...) input := CoroPlanInput{ Program: ctx.progSSA, requiredRoots: requiredRoots, @@ -1076,7 +1236,9 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { input.EmissionUniverse = ctx.coroSSAEmission input.resolveFunction = ctx.coroEmission.Resolve input.functionBackground = ctx.coroEmission.FunctionBackground + input.foreignNoBlock = ctx.coroEmission.CoroForeignNoBlockCertificate input.intrinsicCallSemantics = ctx.coroEmission.CoroIntrinsicCallSiteSemantics + input.rawFunctionAddressCallArgument = ctx.coroEmission.CoroRawFunctionAddressCallArgument input.demandReferences = ctx.coroEmission.CoroDemandReferences input.loweredCalls = ctx.coroEmission.CoroLoweredCalls input.augmentFunctionIDs = func(config coro.FunctionIDConfig) coro.FunctionIDConfig { @@ -1152,9 +1314,213 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { } ctx.coroProgramBootstraps = bootstraps } + if ctx.buildConf.EnableCoroEntryResolution { + if err := validateCoroUnwindOnlyLoweredCalls(plan, metadata.PanicABI); err != nil { + ctx.coroPlan = nil + ctx.coroPlanDigest = "" + ctx.coroPlanMetadata = coro.PlanDigestMetadata{} + ctx.clCompilation = nil + ctx.coroProgramBootstraps = nil + return fmt.Errorf("validate coroutine unwind-only lowered calls before codegen: %w", err) + } + } + return nil +} + +// validateCoroUnwindOnlyLoweredCalls preserves the legacy panic boundary's +// fail-closed physical contract. Unwind-only edges do not taint an owner's +// normal-return plan, but PanicLegacyABIV0 still emits a direct synchronous +// helper call. Until a panic ABI defines coroutine child unwind propagation, +// every physically emitted unwind-only target must therefore have one bounded +// plain primary body. +func validateCoroUnwindOnlyLoweredCalls(plan *coro.SSAPlan, panicABI string) error { + if plan == nil { + return fmt.Errorf("unwind-only lowered-call validation requires a coroutine plan") + } + for _, owner := range plan.Functions() { + if owner.Function == nil || owner.Plan.Emission == coro.EmitNone { + continue + } + for _, lowered := range plan.LoweredCalls(owner.Function) { + if !lowered.UnwindOnly { + continue + } + if panicABI != coro.PanicLegacyABIV0 { + return fmt.Errorf("lowered call %q in %q is unwind-only, but panic ABI %q has no certified unwind-helper call contract", lowered.LogicalName, owner.Plan.ID, panicABI) + } + certificate := coroLegacyPanicPlainCertificate{ + owner: owner.Function, + logicalName: lowered.LogicalName, + target: lowered.Target, + } + if err := certificate.validate(plan); err != nil { + return fmt.Errorf("unwind-only lowered call %q in %q cannot use its exact %s plain certificate: %w", + lowered.LogicalName, owner.Plan.ID, panicABI, err) + } + } + } + return nil +} + +// coroLegacyPanicPlainCertificate is deliberately an object-identity +// certificate, not a symbol-name exception. owner, logicalName, and target are +// copied from the immutable lowered-call table in SSAPlan. That table is frozen +// by the frontend which physically emits the helper call. +// +// CallUnwind prevents the panic episode from tainting the normal-return effect +// of owner, but it cannot make a coroutine target synchronously callable. The +// legacy ABI therefore also requires the exact target's physically reachable +// managed closure to contain only bounded DirectPlain calls. In particular, a +// terminal panic printer must not turn error.Error, Stringer.String, or another +// user callback into a trusted plain function merely because it is reachable +// only while panicking. +type coroLegacyPanicPlainCertificate struct { + owner *ssa.Function + logicalName string + target *ssa.Function +} + +func (certificate coroLegacyPanicPlainCertificate) validate(plan *coro.SSAPlan) error { + if plan == nil || certificate.owner == nil || certificate.target == nil || certificate.logicalName == "" { + return fmt.Errorf("legacy panic plain certificate is incomplete") + } + matched := false + for _, lowered := range plan.LoweredCalls(certificate.owner) { + if lowered.LogicalName == certificate.logicalName && lowered.Target == certificate.target && lowered.UnwindOnly { + matched = true + break + } + } + if !matched { + return fmt.Errorf("legacy panic plain certificate is not bound to an exact frozen unwind-only target") + } + targetPlan, planned := plan.FunctionPlan(certificate.target) + if !planned { + return fmt.Errorf("legacy panic plain certificate targets an unplanned function") + } + if targetPlan.External != coro.Defined { + return fmt.Errorf("legacy panic plain certificate target %q is not a defined Go body (external=%s)", targetPlan.ID, targetPlan.External) + } + + validator := coroLegacyPanicPlainClosureValidator{ + plan: plan, + validated: make(map[*ssa.Function]bool), + active: make(map[*ssa.Function]bool), + } + return validator.validateFunction(certificate.target, nil) +} + +type coroLegacyPanicPlainClosureValidator struct { + plan *coro.SSAPlan + validated map[*ssa.Function]bool + active map[*ssa.Function]bool +} + +func (validator *coroLegacyPanicPlainClosureValidator) validateFunction(function *ssa.Function, path []string) error { + functionPlan, ok := validator.plan.FunctionPlan(function) + if !ok { + return fmt.Errorf("legacy panic target closure contains an unplanned function") + } + path = append(path, fmt.Sprintf("%s[%s]", function.String(), functionPlan.ID)) + if validator.validated[function] { + return nil + } + if validator.active[function] { + // Recursion is diagnosed below by the fixed-point plan (YieldOnly / + // NeedsPreempt). Avoid hiding that deterministic plan error behind a DFS + // cycle diagnostic. + return nil + } + validator.active[function] = true + defer delete(validator.active, function) + + // Inspect the exact physical managed-call closure before reporting the + // aggregate Effect on this function. This turns an opaque-suspend symptom on + // runtime.Panic into the actionable dynamic edge which caused it. Foreign + // leaves remain governed by their ordinary plan; this code never grants or + // manufactures a foreign-noblock certificate. + if !validator.plan.IgnoresBody(function) { + for _, block := range function.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(ssa.CallInstruction) + if !ok || call.Common() == nil { + continue + } + if _, builtin := call.Common().Value.(*ssa.Builtin); builtin { + continue + } + if validator.plan.ElidesCall(call) { + continue + } + callPlan, planned := validator.plan.CallPlan(call) + if !planned { + return coroLegacyPanicPlainPathError(path, "physical call %q has no coroutine call plan", call.String()) + } + if callPlan.Kind == coro.CallSpawn { + return coroLegacyPanicPlainPathError(path, "spawns asynchronous work at %q", call.String()) + } + if callPlan.Open || callPlan.Rep == coro.Dispatch || call.Common().StaticCallee() == nil || len(callPlan.Targets) != 1 { + kind := "dynamic call" + if call.Common().IsInvoke() { + kind = "dynamic invoke" + if method := call.Common().Method; method != nil { + kind += " " + method.Name() + } + } + return coroLegacyPanicPlainPathError(path, "%s is not a bounded DirectPlain edge at %q", kind, call.String()) + } + target, found := validator.plan.Function(callPlan.Targets[0]) + if !found || target == nil { + return coroLegacyPanicPlainPathError(path, "physical call %q has an unresolved planned target", call.String()) + } + if err := validator.validateFunction(target, path); err != nil { + return err + } + if callPlan.Rep != coro.DirectPlain { + return coroLegacyPanicPlainPathError(path, "physical call %q requires %s", call.String(), callPlan.Rep) + } + } + } + for _, lowered := range validator.plan.LoweredCalls(function) { + if lowered.Target == nil { + return coroLegacyPanicPlainPathError(path, "lowered call %q has no exact target", lowered.LogicalName) + } + if err := validator.validateFunction(lowered.Target, path); err != nil { + return err + } + } + } + + if functionPlan.External != coro.Defined { + // A bodyless foreign declaration is a structural leaf, not a managed + // callback to be pulled into this certificate. Its CallForeign edge still + // contributes WaitForeign to the containing Go function in the ordinary + // fixed point, so accepting it for DFS purposes cannot make that Go body + // pass the bounded-plain check below. This merely lets the diagnostic reach + // a more specific managed/dynamic blocker later in the same panic path. + if functionPlan.Demand != coro.NoDemand && functionPlan.FuncRep == coro.DirectPlain && + functionPlan.Primary == coro.PrimaryExternal && functionPlan.Emission == coro.EmitExternal { + validator.validated[function] = true + return nil + } + return coroLegacyPanicPlainPathError(path, + "foreign leaf has no direct physical entry (external=%s demand=%s effect=%s exec=%s representation=%s primary=%s emission=%s)", + functionPlan.External, functionPlan.Demand, functionPlan.Effect, functionPlan.Exec, functionPlan.FuncRep, functionPlan.Primary, functionPlan.Emission) + } + if functionPlan.Demand == coro.NoDemand || functionPlan.Effect != coro.NoSuspend || + functionPlan.Emission != coro.EmitPlain || functionPlan.FuncRep != coro.DirectPlain || functionPlan.Primary != coro.PrimaryPlain { + return coroLegacyPanicPlainPathError(path, + "target is not one bounded plain Go body (external=%s demand=%s effect=%s exec=%s representation=%s primary=%s emission=%s)", + functionPlan.External, functionPlan.Demand, functionPlan.Effect, functionPlan.Exec, functionPlan.FuncRep, functionPlan.Primary, functionPlan.Emission) + } + validator.validated[function] = true return nil } +func coroLegacyPanicPlainPathError(path []string, format string, args ...any) error { + return fmt.Errorf("legacy panic plain closure %s: %s", strings.Join(path, " -> "), fmt.Sprintf(format, args...)) +} + func activeCoroABIVersion(conf *Config) string { if conf != nil && conf.EnableCoroChildAwait { return coro.PhysicalABIV1 @@ -1165,9 +1531,67 @@ func activeCoroABIVersion(conf *Config) string { return coro.EntryResolutionABIV0 } +// requiredCoroProgramManagedEntryRoots injects the exact main-package +// initializer and main body as managed async-capable roots for the runnable +// startup program. Duplicate builder roots are harmless: AnalyzeSSA joins +// demand by canonical function. Descriptor-only builds keep their historical +// explicit-root contract and legacy native entry. +func requiredCoroProgramManagedEntryRoots(ctx *context) (coro.Roots, error) { + if ctx == nil || ctx.buildConf == nil || !ctx.buildConf.EnableCoroProgramBootstrapRun { + return nil, nil + } + if ctx.coroEmission == nil { + return nil, fmt.Errorf("coroutine managed program roots require a frozen emission universe") + } + publicRuntimeInit, hasPublicRuntimeInit, err := findCoroProgramFunction(ctx, "runtime", "init", "public runtime init") + if err != nil { + return nil, err + } + var roots coro.Roots + if hasPublicRuntimeInit { + roots = append(roots, coro.Root{Function: publicRuntimeInit, Demand: coro.AsyncDemand}) + } + seenPackages := make(map[string]struct{}) + for _, pkg := range ctx.initial { + if pkg == nil || !needLink(pkg, ctx.mode) { + continue + } + if _, duplicate := seenPackages[pkg.ID]; duplicate { + return nil, fmt.Errorf("coroutine managed program roots contain duplicate linked package ID %q", pkg.ID) + } + seenPackages[pkg.ID] = struct{}{} + aPkg := ctx.pkgs[pkg] + if aPkg == nil { + aPkg = ctx.pkgByID[pkg.ID] + } + if aPkg == nil || aPkg.SSA == nil || aPkg.SSA.Pkg == nil || llssa.PathOf(aPkg.SSA.Pkg) != pkg.PkgPath { + return nil, fmt.Errorf("coroutine managed program roots: linked main package %q has no exact SSA package", pkg.ID) + } + for _, name := range []string{"init", "main"} { + original := aPkg.SSA.Func(name) + if original == nil { + return nil, fmt.Errorf("coroutine managed program root %s: exact SSA function is missing", name) + } + fn, ok := ctx.coroEmission.Resolve(original) + if !ok || fn == nil || fn != original { + return nil, fmt.Errorf("coroutine managed program root %s: exact function is absent from the frozen emission universe", name) + } + goBody, err := frozenGoEmittedBody(ctx.coroEmission, fn) + if err != nil { + return nil, fmt.Errorf("classify coroutine managed program root %s: %w", name, err) + } + if !goBody { + return nil, fmt.Errorf("coroutine managed program root %s has no emitted Go body", name) + } + roots = append(roots, coro.Root{Function: fn, Demand: coro.AsyncDemand}) + } + } + return roots, nil +} + func activeCoroSchedulerABIVersion(conf *Config) string { if conf != nil && conf.EnableCoroProgramBootstrapRun { - return coro.SchedulerProgramBootstrapABIV1 + return coro.SchedulerProgramBootstrapABIV2 } if conf != nil && conf.EnableCoroChildAwait { return coro.SchedulerChildAwaitABIV0 @@ -1190,7 +1614,7 @@ func activeCoroFuncRepABIVersion(conf *Config) string { // summary. Their fallback SSA stubs remain ignored; ordinary C declarations // outside this compiler-owned closure stay unknown foreign. func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function]struct{}, []requiredCoroDirectPlainCallArgument, map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate, error) { - if ctx == nil || ctx.buildConf == nil || !ctx.buildConf.EnableCoroProgramBootstrapRun { + if ctx == nil || ctx.buildConf == nil || !ctx.buildConf.EnableCoroChildAwait { return nil, nil, nil, nil, nil } if ctx.coroSSAEmission == nil || ctx.coroEmission == nil { @@ -1200,15 +1624,40 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function if err != nil { return nil, nil, nil, nil, err } - names := []string{ - "init", - coroProgramBeginSymbolV1, - coroProgramRunSymbolV1, - "__llgo_coro_frame_alloc_v1", - "__llgo_coro_frame_publish_v1", - "__llgo_coro_await_prepare_v1", - "__llgo_coro_complete_prepare_v1", - "__llgo_coro_frame_free_v1", + // runtimeLinkRequirements makes the real LLGo runtime package init an + // entry-module call for every active child-await executable. That edge is + // compiler-generated LLVM IR and therefore invisible to the source SSA call + // graph; keep it as an explicit synchronous root even when the runnable + // program-bootstrap gate is disabled. The scheduler driver/hooks below are + // referenced only by the runnable bootstrap path and must not leak into the + // descriptor-only plan. + names := []string{"init"} + demandByName := map[string]coro.Demand{"init": coro.SyncDemand} + plainRootByName := map[string]bool{"init": true} + if ctx.buildConf.EnableCoroProgramBootstrapRun { + // The managed startup program owns runtime.init. Its synchronous Go source + // style is preserved by AsyncDemand propagation: a non-suspending body + // remains one DirectPlain body, while an async-tainted body has one + // DirectCoro primary and is awaited by the compiler bootstrap. + demandByName["init"] = coro.AsyncDemand + plainRootByName["init"] = false + names = append(names, + coroFrameAllocatorBootstrapSymbolV1, + coroProgramBeginSymbolV1, + coroProgramRunSymbolV1, + "__llgo_coro_frame_alloc_v1", + "__llgo_coro_frame_publish_v1", + "__llgo_coro_await_prepare_v1", + "__llgo_coro_preempt_poll_v1", + "__llgo_coro_yield_prepare_v1", + "__llgo_coro_park_prepare_v1", + "__llgo_coro_complete_prepare_v1", + "__llgo_coro_frame_free_v1", + ) + for _, name := range names[1:] { + demandByName[name] = coro.SyncDemand + plainRootByName[name] = true + } } byName := make(map[string]*ssa.Function, len(names)) wanted := make(map[string]struct{}, len(names)) @@ -1240,14 +1689,16 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function if !goBody { return nil, nil, nil, nil, fmt.Errorf("coroutine program bootstrap runtime ABI %q has no emitted Go body in %q", name, llssa.PkgRuntime) } - roots = append(roots, coro.Root{Function: fn, Demand: coro.SyncDemand}) + roots = append(roots, coro.Root{Function: fn, Demand: demandByName[name]}) } plain := make(map[*ssa.Function]struct{}) var directPlain []requiredCoroDirectPlainCallArgument queue := make([]*ssa.Function, 0, len(roots)) for _, root := range roots { - queue = append(queue, root.Function) + if plainRootByName[root.Function.Name()] { + queue = append(queue, root.Function) + } } for head := 0; head < len(queue); head++ { fn := queue[head] @@ -1265,6 +1716,18 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function // are retained here and rejected by requiredPlain classification. continue } + loweredCalls, err := ctx.coroEmission.CoroLoweredCalls(fn) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("classify compiler runtime ABI lowered calls in %q: %w", fn.Name(), err) + } + for _, lowered := range loweredCalls { + if lowered.Target == nil { + return nil, nil, nil, nil, fmt.Errorf("compiler runtime ABI function %q has a nil lowered helper target for %q", fn.Name(), lowered.LogicalName) + } + if _, seen := plain[lowered.Target]; !seen { + queue = append(queue, lowered.Target) + } + } for _, block := range fn.Blocks { for _, instruction := range block.Instrs { call, ok := instruction.(ssa.CallInstruction) @@ -1290,10 +1753,10 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function if err != nil { return nil, nil, nil, nil, fmt.Errorf("classify compiler runtime ABI intrinsic %q in %q: %w", callee.Name(), fn.Name(), err) } - if intrinsic && semantics == cl.CoroIntrinsicCallInlineNoSuspend { - // cl emits the proven no-suspend operation inline in fn; it - // has no callable ABI body and is not a member of the trusted - // runtime plain-function island. + if intrinsic && semantics.ElidesManagedCall() { + // cl emits no call to the intrinsic declaration itself. Any + // managed calls inserted by the operation were queued above + // from its exact frozen lowered-call set. continue } if _, seen := plain[callee]; !seen { @@ -1549,6 +2012,7 @@ func buildCoroPlanDigestMetadata(ctx *context) (coro.PlanDigestMetadata, error) func prepareCoroEmissionUniverse(ctx *context, packages []*aPackage) error { inputs := make([]cl.EmissionPackage, 0, len(packages)) + hasRuntimeABI := false for _, aPkg := range packages { if aPkg == nil || aPkg.Package == nil || aPkg.SSA == nil || llruntime.SkipToBuild(aPkg.PkgPath) { continue @@ -1578,8 +2042,14 @@ func prepareCoroEmissionUniverse(ctx *context, packages []*aPackage) error { Identity: aPkg.ID, MetadataOnly: metadataOnly, }) + hasRuntimeABI = hasRuntimeABI || aPkg.PkgPath == llssa.PkgRuntime } - emission, err := cl.PrepareEmissionUniverse(ctx.prog, ctx.patches, inputs) + emission, err := cl.PrepareEmissionUniverseWithOptions(ctx.prog, ctx.patches, inputs, cl.EmissionUniverseOptions{ + // Active archive-producing entry resolution with the real runtime input + // must freeze every hidden compiler/runtime ABI edge. Isolated plan tests + // and report-only builds preserve the legacy incomplete-package behavior. + CompleteRuntimeABI: hasRuntimeABI && ctx.buildConf != nil && ctx.buildConf.EnableCoroEntryResolution, + }) if err != nil { return err } @@ -2260,6 +2730,10 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa if coroBootstrap == nil { return fmt.Errorf("coroutine program bootstrap: no pre-codegen table was frozen for linked package %q", pkg.ID) } + coroBootstrap, err = bindCoroProgramBootstrapV2(coroBootstrap, linkedOrder) + if err != nil { + return fmt.Errorf("bind coroutine program bootstrap: %w", err) + } } coroManifestHash, err = coroProgramManifestHashV1(ctx, coroRootAnchors, coroBootstrap) if err != nil { diff --git a/internal/build/collect.go b/internal/build/collect.go index ad3bc9ab10..b6b282592f 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -109,6 +109,7 @@ func (c *context) collectCommonInputs(m *manifestBuilder) { m.common.BuildTags = strings.Split(c.buildConf.Tags, ",") } m.common.Target = c.buildConf.Target + m.common.RuntimeGC = c.crossCompile.GC if c.hasNonDefaultLLVMConfig() { m.common.LLVMCPU = c.crossCompile.CPU m.common.LLVMFeatures = c.crossCompile.Features diff --git a/internal/build/coro_bootstrap.go b/internal/build/coro_bootstrap.go index e5cf91e6cc..166e7792f9 100644 --- a/internal/build/coro_bootstrap.go +++ b/internal/build/coro_bootstrap.go @@ -22,19 +22,26 @@ import ( "encoding/hex" "fmt" "go/types" + "sort" "strconv" "github.com/goplus/llgo/internal/coro" "github.com/goplus/llgo/internal/packages" llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" ) const ( - coroProgramBootstrapVersionV1 uint32 = 1 - coroProgramBootstrapFactorySymbolV1 = "__llgo_coro_program_bootstrap_factory_v1" - coroProgramBootstrapFrameDescriptorPrefixV1 = "__llgo_coro_program_bootstrap_frame_descriptor_v1." - coroProgramBeginSymbolV1 = "__llgo_coro_program_begin_v1" - coroProgramRunSymbolV1 = "__llgo_coro_program_run_v1" + coroProgramBootstrapVersionV1 uint32 = 1 + coroProgramBootstrapVersionV2 uint32 = 2 + coroProgramBootstrapFactorySymbolV1 = "__llgo_coro_program_bootstrap_factory_v1" + coroProgramBootstrapFactorySymbolV2 = "__llgo_coro_program_bootstrap_factory_v2" + coroProgramBootstrapFrameDescriptorPrefixV1 = "__llgo_coro_program_bootstrap_frame_descriptor_v1." + coroProgramBootstrapFrameDescriptorPrefixV2 = "__llgo_coro_program_bootstrap_frame_descriptor_v2." + coroProgramPublicRuntimeNoopSymbolV2 = "__llgo_coro_public_runtime_init_noop_v2" + coroProgramPublicRuntimeNoopIDV2 coro.FunctionID = "llgo.bootstrap.v2.public-runtime-init.noop" + coroProgramBeginSymbolV1 = "__llgo_coro_program_begin_v1" + coroProgramRunSymbolV1 = "__llgo_coro_program_run_v1" // Step kinds and semantic roles are part of the cross-target bootstrap ABI. // Keep these numeric values synchronized with ssa and runtime/internal/coro. @@ -42,21 +49,41 @@ const ( coroProgramStepCoroRootV1 uint32 = 2 coroProgramStepRoleInitV1 uint32 = 1 coroProgramStepRoleMainV1 uint32 = 2 + + coroProgramStepRoleRuntimeInitV2 uint32 = 1 + coroProgramStepRoleABIInitV2 uint32 = 2 + coroProgramStepRolePublicRuntimeInitV2 uint32 = 4 + coroProgramStepRolePackageInitV2 uint32 = 8 + coroProgramStepRoleMainV2 uint32 = 16 ) type coroProgramBootstrapStepV1 struct { Kind uint32 Role uint32 FunctionID coro.FunctionID - Target string - Aux uint64 + // Target is the exact callable symbol. For CoroRoot it is the function's + // unique physical coroutine primary and is used by the compiler-owned + // bootstrap; CatalogTarget is the linked package anchor validated by the + // runtime startup table. + Target string + Owner string + CatalogTarget string + Aux uint64 } type coroProgramBootstrapV1 struct { + Version uint32 StepHash [16]byte Steps []coroProgramBootstrapStepV1 } +func (b *coroProgramBootstrapV1) abiVersion() uint32 { + if b == nil || b.Version == 0 { + return coroProgramBootstrapVersionV1 + } + return b.Version +} + func validateCoroProgramBootstrapConfig(conf *Config) error { if conf == nil { return nil @@ -93,7 +120,13 @@ func prepareCoroProgramBootstrapsV1(ctx *context) (map[string]*coroProgramBootst if _, exists := bootstraps[pkg.ID]; exists { return nil, fmt.Errorf("duplicate linked main package ID %q", pkg.ID) } - bootstrap, err := selectCoroProgramBootstrapV1(ctx, pkg) + var bootstrap *coroProgramBootstrapV1 + var err error + if ctx.buildConf.EnableCoroProgramBootstrapRun { + bootstrap, err = selectCoroProgramBootstrapV2(ctx, pkg) + } else { + bootstrap, err = selectCoroProgramBootstrapV1(ctx, pkg) + } if err != nil { return nil, fmt.Errorf("package %q: %w", pkg.ID, err) } @@ -158,6 +191,296 @@ func selectCoroProgramBootstrapV1(ctx *context, pkg *packages.Package) (*coroPro return &coroProgramBootstrapV1{StepHash: hash, Steps: steps}, nil } +// selectCoroProgramBootstrapV2 freezes the managed five-stage startup program: +// internal runtime init, compiler ABI init, public runtime init, main-package +// init, and main. Go bodies retain exactly one primary selected by the plan; +// compiler-owned stages are bounded direct-plain calls. +func selectCoroProgramBootstrapV2(ctx *context, pkg *packages.Package) (*coroProgramBootstrapV1, error) { + if ctx == nil || ctx.buildConf == nil || !ctx.buildConf.EnableCoroProgramBootstrapRun { + return nil, nil + } + if err := validateCoroProgramBootstrapConfig(ctx.buildConf); err != nil { + return nil, err + } + if pkg == nil || ctx.prog == nil || ctx.coroEmission == nil || ctx.coroPlan == nil { + return nil, fmt.Errorf("coroutine program bootstrap v2 requires a linked main package, LLVM program, frozen emission universe, and plan") + } + aPkg := ctx.pkgs[pkg] + if aPkg == nil { + aPkg = ctx.pkgByID[pkg.ID] + } + if aPkg == nil || aPkg.Package == nil || aPkg.SSA == nil || aPkg.SSA.Pkg == nil || + aPkg.ID != pkg.ID || aPkg.PkgPath != pkg.PkgPath || llssa.PathOf(aPkg.SSA.Pkg) != pkg.PkgPath { + return nil, fmt.Errorf("coroutine program bootstrap v2: linked main package %q has no exact SSA package", pkg.ID) + } + + runtimeInit, err := exactCoroRuntimeABIFunction(ctx, "init") + if err != nil { + return nil, err + } + publicRuntimeInit, hasPublicRuntimeInit, err := findCoroProgramFunction(ctx, "runtime", "init", "public runtime init") + if err != nil { + return nil, err + } + mainInit := aPkg.SSA.Func("init") + mainMain := aPkg.SSA.Func("main") + steps := make([]coroProgramBootstrapStepV1, 0, 5) + for _, spec := range []struct { + fn *ssa.Function + target string + owner string + label string + role uint32 + }{ + {runtimeInit, llssa.PkgRuntime + ".init", llssa.PkgRuntime, "internal runtime init", coroProgramStepRoleRuntimeInitV2}, + {mainInit, aPkg.PkgPath + ".init", aPkg.PkgPath, "main package init", coroProgramStepRolePackageInitV2}, + {mainMain, aPkg.PkgPath + ".main", aPkg.PkgPath, "main", coroProgramStepRoleMainV2}, + } { + step, err := selectCoroProgramManagedStepV2(ctx, spec.fn, spec.target, spec.owner, spec.label, spec.role) + if err != nil { + return nil, err + } + steps = append(steps, step) + } + publicRuntimeStep := coroProgramBootstrapStepV1{ + Kind: coroProgramStepDirectPlainV1, + Role: coroProgramStepRolePublicRuntimeInitV2, + FunctionID: coroProgramPublicRuntimeNoopIDV2, + Target: coroProgramPublicRuntimeNoopSymbolV2, + } + if hasPublicRuntimeInit { + publicRuntimeStep, err = selectCoroProgramManagedStepV2( + ctx, publicRuntimeInit, "runtime.init", "runtime", "public runtime init", coroProgramStepRolePublicRuntimeInitV2, + ) + if err != nil { + return nil, err + } + } + // Insert the compiler-owned ABI stage between the internal and public + // runtime initializers. It always exists in the entry module; profiles with + // no work receive a canonical no-op body. Public runtime initialization is + // an exact managed Go body above, never an assumed plain weak stub. + steps = append(steps[:1], append([]coroProgramBootstrapStepV1{ + { + Kind: coroProgramStepDirectPlainV1, + Role: coroProgramStepRoleABIInitV2, + FunctionID: "llgo.bootstrap.v2.compiler-abi-init", + Target: "init$abitypes", + }, + publicRuntimeStep, + }, steps[1:]...)...) + hash, err := coroProgramBootstrapHash(ctx, coroProgramBootstrapVersionV2, steps) + if err != nil { + return nil, err + } + return &coroProgramBootstrapV1{Version: coroProgramBootstrapVersionV2, StepHash: hash, Steps: steps}, nil +} + +func exactCoroRuntimeABIFunction(ctx *context, name string) (*ssa.Function, error) { + return exactCoroProgramFunction(ctx, llssa.PkgRuntime, name, "internal runtime ABI") +} + +// exactCoroProgramFunction selects one canonical emitted top-level Go body by +// package identity. It is used for startup stages whose source may come from a +// patch package (notably the public standard-library runtime package), so the +// selection is made from the frozen emission universe rather than from an +// import/package-name guess. +func exactCoroProgramFunction(ctx *context, pkgPath, name, label string) (*ssa.Function, error) { + fn, ok, err := findCoroProgramFunction(ctx, pkgPath, name, label) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("coroutine program bootstrap %s %q has no emitted Go body in %q", label, name, pkgPath) + } + return fn, nil +} + +// findCoroProgramFunction is exactCoroProgramFunction with an explicit absent +// result. Absence is valid only for optional startup packages such as the +// public standard-library runtime facade; ambiguity or a selected non-Go body +// still fails closed. +func findCoroProgramFunction(ctx *context, pkgPath, name, label string) (*ssa.Function, bool, error) { + if ctx == nil || ctx.coroSSAEmission == nil || ctx.coroEmission == nil { + return nil, false, fmt.Errorf("coroutine program bootstrap %s %q requires a complete frozen emission universe", label, name) + } + var found *ssa.Function + for _, fn := range ctx.coroSSAEmission.Functions() { + if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil || llssa.PathOf(fn.Pkg.Pkg) != pkgPath || fn.Name() != name { + continue + } + if found != nil && found != fn { + return nil, false, fmt.Errorf("coroutine program bootstrap %s %q has multiple canonical SSA bodies in %q", label, name, pkgPath) + } + found = fn + } + if found == nil { + return nil, false, nil + } + goBody, err := frozenGoEmittedBody(ctx.coroEmission, found) + if err != nil { + return nil, false, fmt.Errorf("classify coroutine program bootstrap %s %q: %w", label, name, err) + } + if !goBody { + return nil, false, fmt.Errorf("coroutine program bootstrap %s %q selected a non-Go body in %q", label, name, pkgPath) + } + return found, true, nil +} + +func selectCoroProgramManagedStepV2( + ctx *context, original *ssa.Function, target, owner, label string, role uint32, +) (coroProgramBootstrapStepV1, error) { + if original == nil { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: exact SSA function is missing", label) + } + fn, ok := ctx.coroEmission.Resolve(original) + if !ok || fn == nil || fn != original { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: exact function is absent from the frozen emission universe", label) + } + if fn.Pkg == nil || fn.Pkg.Pkg == nil || llssa.PathOf(fn.Pkg.Pkg) != owner || fn.Parent() != nil || fn.Origin() != nil || len(fn.TypeArgs()) != 0 { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: frozen target is not the exact top-level owner function", label) + } + if link, exists := ctx.prog.Linkname(target); exists && link != target { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: physical symbol is redirected from %q to %q", label, target, link) + } + sig := fn.Signature + if sig == nil || sig.Recv() != nil || sig.Params().Len() != 0 || sig.Results().Len() != 0 || sig.Variadic() || + typeParamLen(sig.TypeParams()) != 0 || typeParamLen(sig.RecvTypeParams()) != 0 || len(fn.FreeVars) != 0 { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: target must have the exact func() signature", label) + } + + rootID := coro.FunctionID("") + rootDemand := coro.NoDemand + for _, root := range ctx.coroPlan.Roots() { + if root.Function != fn { + continue + } + if rootID != "" { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: duplicate explicit plan roots", label) + } + rootID, rootDemand = root.ID, root.Demand + } + if rootID == "" || !rootDemand.Contains(coro.AsyncDemand) { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: target is not an explicit async-capable plan root", label) + } + plan, ok := ctx.coroPlan.FunctionPlan(fn) + if !ok || plan.ID != rootID || plan.External != coro.Defined || !plan.Demand.Contains(coro.AsyncDemand) { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: exact defined async-capable function plan is missing", label) + } + + switch plan.Emission { + case coro.EmitPlain: + if plan.FuncRep != coro.DirectPlain || plan.Primary != coro.PrimaryPlain || plan.Effect != coro.NoSuspend || plan.Exec.Contains(coro.NeedsPreempt) { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: plain target %q has incompatible plan (demand=%s rep=%s primary=%s effect=%s exec=%s)", + label, plan.ID, plan.Demand, plan.FuncRep, plan.Primary, plan.Effect, plan.Exec) + } + // IRQUnsafe is an entry-context restriction, not a request for another + // physical body. The program bootstrap runs as an ordinary G on the + // executor, never as an interrupt callback, so a bounded plain stage may + // retain this flag. ThreadAffine remains rejected until the bootstrap G has + // an explicit locked-M/pinned-P contract. + const supportedPlain = coro.MayUnwind | coro.NeedsCleanupFrame | coro.IRQUnsafe + if unsupported := plan.Exec &^ supportedPlain; unsupported != 0 { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: plain target %q has unsupported execution constraints %s", label, plan.ID, unsupported) + } + return coroProgramBootstrapStepV1{ + Kind: coroProgramStepDirectPlainV1, Role: role, FunctionID: plan.ID, Target: target, + }, nil + + case coro.EmitCoroutine: + if rootDemand != coro.AsyncDemand || plan.Demand != coro.AsyncDemand || plan.FuncRep != coro.DirectCoro || plan.Primary != coro.PrimaryCoroutine || !plan.Effect.MaySuspend() { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: coroutine target %q is not one async-only direct coroutine (root=%s demand=%s rep=%s primary=%s effect=%s)", + label, plan.ID, rootDemand, plan.Demand, plan.FuncRep, plan.Primary, plan.Effect) + } + if unsupported := plan.Exec &^ (coro.MayUnwind | coro.NeedsPreempt | coro.IRQUnsafe); unsupported != 0 { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: coroutine target %q has unsupported execution constraints %s", label, plan.ID, unsupported) + } + index, err := coroProgramRootDescriptorIndexV2(ctx.coroPlan, fn) + if err != nil { + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: %w", label, err) + } + return coroProgramBootstrapStepV1{ + Kind: coroProgramStepCoroRootV1, + Role: role, + FunctionID: plan.ID, + Target: target + "$coro", + Owner: owner, + Aux: index, + }, nil + + default: + return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: target %q has unsupported emission %s", label, plan.ID, plan.Emission) + } +} + +func coroProgramRootDescriptorIndexV2(plan *coro.SSAPlan, target *ssa.Function) (uint64, error) { + if plan == nil || target == nil || target.Pkg == nil { + return 0, fmt.Errorf("coroutine root descriptor index requires an exact owned target") + } + type rootEntry struct { + id coro.FunctionID + fn *ssa.Function + } + var entries []rootEntry + for _, root := range plan.Roots() { + fnPlan, ok := plan.FunctionPlan(root.Function) + if !ok || root.Function == nil || root.Function.Pkg != target.Pkg || fnPlan.Emission != coro.EmitCoroutine { + continue + } + entries = append(entries, rootEntry{id: root.ID, fn: root.Function}) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].id < entries[j].id }) + for index, entry := range entries { + if entry.fn == target { + return uint64(index), nil + } + } + return 0, fmt.Errorf("coroutine target is absent from its owner's explicit root descriptor order") +} + +// bindCoroProgramBootstrapV2 resolves semantic coroutine owners to the exact +// cache-visible package anchors produced by cl. It returns a copy so the +// pre-codegen semantic table and hash remain immutable; the final manifest hash +// additionally covers the complete sorted anchor catalog. +func bindCoroProgramBootstrapV2(bootstrap *coroProgramBootstrapV1, linked []Package) (*coroProgramBootstrapV1, error) { + if bootstrap == nil || bootstrap.abiVersion() != coroProgramBootstrapVersionV2 { + return bootstrap, nil + } + anchors := make(map[string]string) + for _, pkg := range linked { + if pkg == nil || pkg.PkgPath == "" || pkg.CoroRootAnchorV1 == "" { + continue + } + if !validCoroRootPackageAnchorV1(pkg.CoroRootAnchorV1) { + return nil, fmt.Errorf("package %s has invalid coroutine root anchor %q", pkg.PkgPath, pkg.CoroRootAnchorV1) + } + if previous, duplicate := anchors[pkg.PkgPath]; duplicate && previous != pkg.CoroRootAnchorV1 { + return nil, fmt.Errorf("package %s has conflicting coroutine root anchors %q and %q", pkg.PkgPath, previous, pkg.CoroRootAnchorV1) + } + anchors[pkg.PkgPath] = pkg.CoroRootAnchorV1 + } + bound := *bootstrap + bound.Steps = append([]coroProgramBootstrapStepV1(nil), bootstrap.Steps...) + for index := range bound.Steps { + step := &bound.Steps[index] + switch step.Kind { + case coroProgramStepDirectPlainV1: + if step.Owner != "" || step.CatalogTarget != "" || step.Aux != 0 { + return nil, fmt.Errorf("coroutine program bootstrap v2 direct step %d has catalog state", index) + } + case coroProgramStepCoroRootV1: + anchor := anchors[step.Owner] + if anchor == "" { + return nil, fmt.Errorf("coroutine program bootstrap v2 step %d owner %q has no linked root anchor", index, step.Owner) + } + step.CatalogTarget = anchor + default: + return nil, fmt.Errorf("coroutine program bootstrap v2 step %d has invalid kind %d", index, step.Kind) + } + } + return &bound, nil +} + func selectCoroProgramPlainStepV1(ctx *context, aPkg *aPackage, name string, role uint32) (coroProgramBootstrapStepV1, error) { want := aPkg.PkgPath + "." + name if name == "init" { @@ -247,6 +570,10 @@ func typeParamLen(list *types.TypeParamList) int { } func coroProgramBootstrapHashV1(ctx *context, steps []coroProgramBootstrapStepV1) ([16]byte, error) { + return coroProgramBootstrapHash(ctx, coroProgramBootstrapVersionV1, steps) +} + +func coroProgramBootstrapHash(ctx *context, version uint32, steps []coroProgramBootstrapStepV1) ([16]byte, error) { if ctx == nil || ctx.prog == nil || ctx.buildConf == nil || ctx.coroPlan == nil { return [16]byte{}, fmt.Errorf("coroutine program bootstrap hash requires a complete build context and plan") } @@ -266,15 +593,22 @@ func coroProgramBootstrapHashV1(ctx *context, steps []coroProgramBootstrapStepV1 h.Write(length[:]) h.Write([]byte(value)) } - write("llgo.coro.program-bootstrap.v1") - write(strconv.FormatUint(uint64(coroProgramBootstrapVersionV1), 10)) + if version != coroProgramBootstrapVersionV1 && version != coroProgramBootstrapVersionV2 { + return [16]byte{}, fmt.Errorf("coroutine program bootstrap hash has unsupported version %d", version) + } + write("llgo.coro.program-bootstrap.v" + strconv.FormatUint(uint64(version), 10)) + write(strconv.FormatUint(uint64(version), 10)) write("flags=0") write("step={kind:u32,flags:u32,target:ptr,aux:uintptr}") write("bootstrap={version:u32,flags:u32,hash-lo:u64,hash-hi:u64,step-count:uintptr,steps:ptr,factory:ptr}") write("direct-plain=" + strconv.FormatUint(uint64(coroProgramStepDirectPlainV1), 10)) write("coro-root=" + strconv.FormatUint(uint64(coroProgramStepCoroRootV1), 10)) if ctx.buildConf.EnableCoroProgramBootstrapRun { - write("factory=compiler-direct-plain-v1:" + coroProgramBootstrapFactorySymbolV1) + factory := coroProgramBootstrapFactorySymbolV1 + if version == coroProgramBootstrapVersionV2 { + factory = coroProgramBootstrapFactorySymbolV2 + } + write("factory=compiler-static-mixed-v" + strconv.FormatUint(uint64(version), 10) + ":" + factory) write("driver=runtime-static-single-p-v1:" + coroProgramBeginSymbolV1 + ":" + coroProgramRunSymbolV1) write("header=physical-abi-v1") } else { @@ -299,6 +633,7 @@ func coroProgramBootstrapHashV1(ctx *context, steps []coroProgramBootstrapStepV1 write(strconv.FormatUint(uint64(step.Role), 10)) write(string(step.FunctionID)) write(step.Target) + write(step.Owner) write(strconv.FormatUint(step.Aux, 10)) } sum := h.Sum(nil) diff --git a/internal/build/coro_bootstrap_factory.go b/internal/build/coro_bootstrap_factory.go index 546b1516ce..2c17df0e8e 100644 --- a/internal/build/coro_bootstrap_factory.go +++ b/internal/build/coro_bootstrap_factory.go @@ -32,9 +32,11 @@ const ( coroProgramFrameFreeHookV1 = "__llgo_coro_frame_free_v1" coroProgramPhysicalABIVersionV1 = 1 coroProgramSuspendNoneV1 = 0 + coroProgramSuspendCallV1 = 1 coroProgramSuspendFrameCompleteV1 = 2 coroProgramLifecycleInitialV1 = 1 coroProgramLifecycleActiveV1 = 2 + coroProgramLifecycleSuspendedV1 = 3 coroProgramLifecycleFinalV1 = 4 ) @@ -50,6 +52,11 @@ const ( coroProgramHeaderFlagsV1 ) +type coroProgramBootstrapFactoryTargetV2 struct { + Plain llssa.Function + Anchor llssa.Expr +} + // emitCoroProgramBootstrapFactoryV1 defines the compiler-owned program-root // coroutine. The caller supplies the exact two target declarations used by the // already validated bootstrap table; the factory deliberately does not look up @@ -159,6 +166,194 @@ func emitCoroProgramBootstrapFactoryV1( return factory } +// emitCoroProgramBootstrapFactoryV2 defines the compiler-owned heterogeneous +// startup coroutine. DirectPlain steps are statically called. CoroRoot steps +// load the exact validated descriptor factory from their bound package +// anchor/index, create an initial-suspended child, and reuse the ordinary v1 +// parent/await scheduler handoff. The runtime never chooses or invokes a user +// function pointer; the compiler emits this fixed five-stage program. +func emitCoroProgramBootstrapFactoryV2( + pkg llssa.Package, + bootstrap *coroProgramBootstrapV1, + targets []coroProgramBootstrapFactoryTargetV2, + finalHash [16]byte, +) llssa.Function { + validateCoroProgramBootstrapFactoryV2(pkg, bootstrap, targets) + + prog := pkg.Prog + pointer := types.Typ[types.UnsafePointer] + factory := pkg.NewFunc(coroProgramBootstrapFactorySymbolV2, newSignature( + []types.Type{pointer, pointer, pointer}, + []types.Type{pointer}, + ), llssa.InC) + if factory.HasBody() { + panic(fmt.Sprintf("coroutine program bootstrap factory symbol %q already has a body", coroProgramBootstrapFactorySymbolV2)) + } + factoryValue := pkg.Module().NamedFunction(coroProgramBootstrapFactorySymbolV2) + factoryValue.SetVisibility(llvm.HiddenVisibility) + + emptyPayload := prog.Struct() + descriptor := pkg.NewCoroFrameDescriptor( + coroProgramBootstrapFrameDescriptorPrefixV2+hex.EncodeToString(finalHash[:]), + llssa.CoroFrameDescriptorOptions{ + Version: coroProgramPhysicalABIVersionV1, + ABIHash: finalHash, + Result: emptyPayload, + }, + ) + + b := factory.MakeBody(1) + g := factory.Param(0) + out := factory.Param(1) + null := prog.Nil(prog.VoidPtr()) + descriptorPointer := b.Convert(prog.VoidPtr(), descriptor) + headerType := coroProgramBootstrapHeaderTypeV1(prog) + header := b.AllocaT(headerType) + + alloc := pkg.NewFunc(coroProgramFrameAllocHookV1, newSignature( + []types.Type{pointer, types.Typ[types.Uintptr], types.Typ[types.Uintptr], pointer}, + []types.Type{pointer}, + ), llssa.InC) + publish := pkg.NewFunc(coroProgramFramePublishHookV1, newSignature( + []types.Type{pointer, pointer, pointer, pointer}, nil, + ), llssa.InC) + await := pkg.NewFunc("__llgo_coro_await_prepare_v1", newSignature( + []types.Type{pointer, pointer, pointer}, nil, + ), llssa.InC) + complete := pkg.NewFunc(coroProgramCompletePrepareHookV1, newSignature( + []types.Type{pointer, pointer, pointer}, nil, + ), llssa.InC) + free := pkg.NewFunc(coroProgramFrameFreeHookV1, newSignature( + []types.Type{pointer, pointer, types.Typ[types.Uintptr], types.Typ[types.Uintptr], pointer}, nil, + ), llssa.InC) + + frame := llssa.CoroFrameOps{ + Alloc: func(b llssa.Builder, size, align llssa.Expr) llssa.Expr { + return b.Call(alloc.Expr, g, size, align, descriptorPointer) + }, + Free: func(b llssa.Builder, storage, size, align llssa.Expr) { + b.Call(free.Expr, g, storage, size, align, descriptorPointer) + }, + } + coroBuilder := b.BeginCoro(llssa.CoroOptions{ + Promise: header, + Frame: frame, + BeforeInitialSuspend: func(b llssa.Builder, handle, storage llssa.Expr) { + values := []llssa.Expr{ + g, + null, + descriptorPointer, + null, + out, + prog.IntVal(coroProgramSuspendNoneV1, prog.Uint16()), + prog.IntVal(coroProgramLifecycleInitialV1, prog.Uint16()), + prog.IntVal(0, prog.Uint32()), + prog.IntVal(0, prog.Uint32()), + } + for index, value := range values { + b.Store(b.FieldAddr(header, index), value) + } + b.Call(publish.Expr, g, handle, b.Convert(prog.VoidPtr(), header), storage) + }, + }) + + b.SetBlock(coroBuilder.InitialResumeBlock()) + b.Store(b.FieldAddr(header, coroProgramHeaderSuspendReasonV1), prog.IntVal(coroProgramSuspendNoneV1, prog.Uint16())) + b.Store(b.FieldAddr(header, coroProgramHeaderLifecycleV1), prog.IntVal(coroProgramLifecycleActiveV1, prog.Uint16())) + + rootFactorySig := newSignature( + []types.Type{pointer, pointer, pointer}, + []types.Type{pointer}, + ) + // A signature used as a value is llssa's callable vkFuncPtr shape. FuncDecl + // is the declaration/function type used by statically named functions and + // cannot represent the loaded opaque pointer here. + rootFactoryType := prog.Type(rootFactorySig, llssa.InC) + rootDescriptorType := prog.Struct( + prog.Uint32(), prog.Uint32(), prog.Uint64(), prog.Uint64(), + prog.VoidPtr(), + prog.Uintptr(), prog.Uintptr(), prog.Uintptr(), prog.Uintptr(), + ) + for index, step := range bootstrap.Steps { + target := targets[index] + switch step.Kind { + case coroProgramStepDirectPlainV1: + b.Call(target.Plain.Expr) + case coroProgramStepCoroRootV1: + entries := b.Load(b.FieldAddr(target.Anchor, 5)) + entryPointer := b.Convert(prog.Pointer(prog.VoidPtr()), entries) + descriptorRaw := b.Load(b.Advance(entryPointer, prog.IntVal(step.Aux, prog.Uintptr()))) + rootDescriptor := b.Convert(prog.Pointer(rootDescriptorType), descriptorRaw) + rootFactoryRaw := b.Load(b.FieldAddr(rootDescriptor, 4)) + // LLVM uses opaque pointers, but llssa still needs the callable + // declaration kind/signature on the expression. This is a pure type + // retag, not a pointer-to-function Go conversion (which would leave + // Builder.Call with a non-callable vkPtr expression). + rootFactory := b.ChangeType(rootFactoryType, rootFactoryRaw) + child := b.Call(rootFactory, g, null, null) + childHeader := b.CoroPromise(child, headerType) + b.Store(b.FieldAddr(childHeader, coroProgramHeaderParentV1), coroBuilder.Handle()) + stateID := uint64(index + 1) + b.Store(b.FieldAddr(header, coroProgramHeaderSuspendReasonV1), prog.IntVal(coroProgramSuspendCallV1, prog.Uint16())) + b.Store(b.FieldAddr(header, coroProgramHeaderLifecycleV1), prog.IntVal(coroProgramLifecycleSuspendedV1, prog.Uint16())) + b.Store(b.FieldAddr(header, coroProgramHeaderStateIDV1), prog.IntVal(stateID, prog.Uint32())) + b.Call(await.Expr, g, coroBuilder.Handle(), child) + coroBuilder.SuspendCurrentBlock() + b.Store(b.FieldAddr(header, coroProgramHeaderSuspendReasonV1), prog.IntVal(coroProgramSuspendNoneV1, prog.Uint16())) + b.Store(b.FieldAddr(header, coroProgramHeaderLifecycleV1), prog.IntVal(coroProgramLifecycleActiveV1, prog.Uint16())) + } + } + + b.Store(b.FieldAddr(header, coroProgramHeaderSuspendReasonV1), prog.IntVal(coroProgramSuspendFrameCompleteV1, prog.Uint16())) + b.Store(b.FieldAddr(header, coroProgramHeaderLifecycleV1), prog.IntVal(coroProgramLifecycleFinalV1, prog.Uint16())) + b.Store(b.FieldAddr(header, coroProgramHeaderStateIDV1), prog.IntVal(uint64(len(bootstrap.Steps)+1), prog.Uint32())) + b.Call(complete.Expr, g, coroBuilder.Handle(), b.Convert(prog.VoidPtr(), header)) + coroBuilder.Finish() + b.Dispose() + return factory +} + +func validateCoroProgramBootstrapFactoryV2( + pkg llssa.Package, bootstrap *coroProgramBootstrapV1, targets []coroProgramBootstrapFactoryTargetV2, +) { + if pkg == nil || pkg.Prog == nil { + panic("coroutine program bootstrap v2 factory requires an LLVM package") + } + if bootstrap == nil || bootstrap.abiVersion() != coroProgramBootstrapVersionV2 || len(bootstrap.Steps) != 5 || len(targets) != 5 { + panic("coroutine program bootstrap v2 factory requires exactly five validated steps") + } + roles := [...]uint32{ + coroProgramStepRoleRuntimeInitV2, + coroProgramStepRoleABIInitV2, + coroProgramStepRolePublicRuntimeInitV2, + coroProgramStepRolePackageInitV2, + coroProgramStepRoleMainV2, + } + for index, step := range bootstrap.Steps { + target := targets[index] + if step.Role != roles[index] || step.FunctionID == "" || step.Target == "" { + panic(fmt.Sprintf("coroutine program bootstrap v2 factory step %d has noncanonical identity or role", index)) + } + switch step.Kind { + case coroProgramStepDirectPlainV1: + if step.Owner != "" || step.CatalogTarget != "" || step.Aux != 0 || target.Plain == nil || !target.Anchor.IsNil() || + target.Plain.Pkg != pkg || target.Plain.Name() != step.Target { + panic(fmt.Sprintf("coroutine program bootstrap v2 direct step %d target does not match %q", index, step.Target)) + } + sig, ok := target.Plain.RawType().(*types.Signature) + if !ok || sig.Recv() != nil || sig.Variadic() || sig.Params().Len() != 0 || sig.Results().Len() != 0 { + panic(fmt.Sprintf("coroutine program bootstrap v2 direct step %d target %q does not have void() C ABI", index, step.Target)) + } + case coroProgramStepCoroRootV1: + if step.Owner == "" || step.CatalogTarget == "" || target.Plain != nil || target.Anchor.IsNil() || target.Anchor.Name() != step.CatalogTarget { + panic(fmt.Sprintf("coroutine program bootstrap v2 coroutine step %d anchor does not match %q", index, step.CatalogTarget)) + } + default: + panic(fmt.Sprintf("coroutine program bootstrap v2 factory step %d has invalid kind %d", index, step.Kind)) + } + } +} + // coroProgramBootstrapHeaderTypeV1 must remain field-for-field identical to // runtime/internal/coro.HeaderV1 and cl's physical coroutine header. func coroProgramBootstrapHeaderTypeV1(prog llssa.Program) llssa.Type { diff --git a/internal/build/coro_bootstrap_factory_test.go b/internal/build/coro_bootstrap_factory_test.go index ba8fe4f244..5cfb554460 100644 --- a/internal/build/coro_bootstrap_factory_test.go +++ b/internal/build/coro_bootstrap_factory_test.go @@ -17,6 +17,7 @@ package build import ( + "go/types" "regexp" "strings" "testing" @@ -93,6 +94,71 @@ func TestCoroProgramBootstrapFactoryV1NativeAndWasm(t *testing.T) { } } +func TestCoroProgramBootstrapFactoryV2MixedNativeAndWasm(t *testing.T) { + llssa.Initialize(llssa.InitAll) + tests := []struct { + name string + target *llssa.Target + uintptrIR string + }{ + {name: "native", uintptrIR: "i64"}, + {name: "wasm", target: &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}, uintptrIR: "i32"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + prog := llssa.NewProgram(test.target) + defer prog.Dispose() + pkg := prog.NewPackage("entry", "entry") + defer pkg.Module().Dispose() + + bootstrap, targets, tableSteps, finalHash := newCoroProgramBootstrapFactoryFixtureV2(pkg) + factory := emitCoroProgramBootstrapFactoryV2(pkg, bootstrap, targets, finalHash) + pkg.NewCoroProgramBootstrap("__llgo_test_program_bootstrap_v2", llssa.CoroProgramBootstrapOptions{ + Version: coroProgramBootstrapVersionV2, + ABIHash: finalHash, + Steps: tableSteps, + Factory: factory.Expr, + }) + + mod := pkg.Module() + mod.SetDataLayout(prog.DataLayout()) + mod.SetTarget(prog.TargetSpec().Triple) + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify mixed v2 bootstrap factory before CoroSplit: %v\n%s", err, mod.String()) + } + pre := mod.String() + assertCoroProgramBootstrapFactoryPresplitV2(t, pre, test.uintptrIR) + + options := llvm.NewPassBuilderOptions() + options.SetVerifyEach(true) + if err := mod.RunPasses("coro-early,cgscc(coro-split),coro-cleanup", prog.TargetMachine(), options); err != nil { + options.Dispose() + t.Fatalf("CoroSplit mixed v2 bootstrap factory: %v\n%s", err, mod.String()) + } + options.Dispose() + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify mixed v2 bootstrap factory after CoroSplit: %v\n%s", err, mod.String()) + } + post := mod.String() + for _, suffix := range []string{".resume", ".destroy"} { + if mod.NamedFunction(coroProgramBootstrapFactorySymbolV2 + suffix).IsNil() { + t.Fatalf("CoroSplit did not create mixed v2 bootstrap factory%s:\n%s", suffix, post) + } + } + for _, intrinsic := range []string{"llvm.coro.id", "llvm.coro.begin", "llvm.coro.suspend"} { + if regexp.MustCompile(`call [^\n]*@` + regexp.QuoteMeta(intrinsic) + `\b`).MatchString(post) { + t.Fatalf("post-split mixed v2 bootstrap still calls %s:\n%s", intrinsic, post) + } + } + object, err := prog.TargetMachine().EmitToMemoryBuffer(mod, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit mixed v2 bootstrap factory object: %v\n%s", err, post) + } + object.Dispose() + }) + } +} + func TestCoroProgramBootstrapFactoryV1RejectsNonCanonicalInputs(t *testing.T) { llssa.Initialize(llssa.InitAll) tests := []struct { @@ -166,6 +232,90 @@ func newCoroProgramBootstrapFactoryFixtureV1( return bootstrap, targets, finalHash } +func newCoroProgramBootstrapFactoryFixtureV2( + pkg llssa.Package, +) (*coroProgramBootstrapV1, []coroProgramBootstrapFactoryTargetV2, []llssa.CoroProgramStep, [16]byte) { + prog := pkg.Prog + pointer := types.Typ[types.UnsafePointer] + rootFactorySig := newSignature( + []types.Type{pointer, pointer, pointer}, + []types.Type{pointer}, + ) + rootFactories := [2]llssa.Function{ + pkg.NewFunc("example.com/runtime.init$coro.factory", rootFactorySig, llssa.InC), + pkg.NewFunc("example.com/program.init$coro.factory", rootFactorySig, llssa.InC), + } + emptyPayload := prog.Struct() + descriptors := [2]llssa.Expr{ + pkg.NewCoroRootFactoryDescriptor("example.com/runtime.init$coro.descriptor", llssa.CoroRootFactoryDescriptorOptions{ + Version: coroProgramPhysicalABIVersionV1, + Factory: rootFactories[0].Expr, + Startup: emptyPayload, + Result: emptyPayload, + }), + pkg.NewCoroRootFactoryDescriptor("example.com/program.init$coro.descriptor", llssa.CoroRootFactoryDescriptorOptions{ + Version: coroProgramPhysicalABIVersionV1, + Factory: rootFactories[1].Expr, + Startup: emptyPayload, + Result: emptyPayload, + }), + } + const anchorName = "__llgo_coro_root_package_v1.0123456789abcdef0123456789abcdef" + anchor := pkg.NewCoroRootPackageAnchor(anchorName, llssa.CoroRootPackageAnchorOptions{ + Version: coroProgramPhysicalABIVersionV1, + Descriptors: descriptors[:], + }) + plains := [3]llssa.Function{ + declareNoArgFunc(pkg, "init$abitypes"), + declareNoArgFunc(pkg, "runtime.init"), + declareNoArgFunc(pkg, "example.com/program.main"), + } + steps := []coroProgramBootstrapStepV1{ + { + Kind: coroProgramStepCoroRootV1, Role: coroProgramStepRoleRuntimeInitV2, + FunctionID: "runtime-init-id", Target: "example.com/runtime.init$coro", + Owner: "example.com/runtime", CatalogTarget: anchorName, Aux: 0, + }, + { + Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleABIInitV2, + FunctionID: "abi-init-id", Target: plains[0].Name(), + }, + { + Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRolePublicRuntimeInitV2, + FunctionID: "public-runtime-init-id", Target: plains[1].Name(), + }, + { + Kind: coroProgramStepCoroRootV1, Role: coroProgramStepRolePackageInitV2, + FunctionID: "package-init-id", Target: "example.com/program.init$coro", + Owner: "example.com/program", CatalogTarget: anchorName, Aux: 1, + }, + { + Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleMainV2, + FunctionID: "main-id", Target: plains[2].Name(), + }, + } + bootstrap := &coroProgramBootstrapV1{Version: coroProgramBootstrapVersionV2, Steps: steps} + targets := []coroProgramBootstrapFactoryTargetV2{ + {Anchor: anchor}, + {Plain: plains[0]}, + {Plain: plains[1]}, + {Anchor: anchor}, + {Plain: plains[2]}, + } + tableSteps := []llssa.CoroProgramStep{ + {Kind: llssa.CoroProgramStepCoroRoot, Flags: steps[0].Role, Target: anchor, Aux: steps[0].Aux}, + {Kind: llssa.CoroProgramStepDirectPlain, Flags: steps[1].Role, Target: plains[0].Expr}, + {Kind: llssa.CoroProgramStepDirectPlain, Flags: steps[2].Role, Target: plains[1].Expr}, + {Kind: llssa.CoroProgramStepCoroRoot, Flags: steps[3].Role, Target: anchor, Aux: steps[3].Aux}, + {Kind: llssa.CoroProgramStepDirectPlain, Flags: steps[4].Role, Target: plains[2].Expr}, + } + var finalHash [16]byte + for index := range finalHash { + finalHash[index] = byte(index + 1) + } + return bootstrap, targets, tableSteps, finalHash +} + func assertCoroProgramBootstrapFactoryPresplitV1(t *testing.T, ir, uintptrIR string) { t.Helper() descriptorLine := irLineWithPrefix(ir, "@"+coroProgramBootstrapFrameDescriptorPrefixV1) @@ -226,6 +376,56 @@ func assertCoroProgramBootstrapFactoryPresplitV1(t *testing.T, ir, uintptrIR str } } +func assertCoroProgramBootstrapFactoryPresplitV2(t *testing.T, ir, uintptrIR string) { + t.Helper() + descriptorLine := irLineWithPrefix(ir, "@"+coroProgramBootstrapFrameDescriptorPrefixV2) + if descriptorLine == "" { + t.Fatalf("mixed v2 bootstrap frame descriptor is missing:\n%s", ir) + } + for _, want := range []string{ + "i32 1, i32 0", + "i64 72623859790382856, i64 651345242494996240", + uintptrIR + " 0, " + uintptrIR + " 1", + } { + if !strings.Contains(descriptorLine, want) { + t.Fatalf("mixed v2 bootstrap frame descriptor missing %q: %s", want, descriptorLine) + } + } + bootstrapLine := irLineWithPrefix(ir, "@__llgo_test_program_bootstrap_v2 =") + if bootstrapLine == "" || !strings.Contains(bootstrapLine, "i32 2") || + !strings.Contains(bootstrapLine, "ptr @"+coroProgramBootstrapFactorySymbolV2) { + t.Fatalf("mixed v2 bootstrap descriptor does not publish version/factory: %s\n%s", bootstrapLine, ir) + } + + body := llvmFunctionIRV1(ir, coroProgramBootstrapFactorySymbolV2) + if body == "" { + t.Fatalf("mixed v2 bootstrap factory body is missing:\n%s", ir) + } + if got := strings.Count(body, "call void @__llgo_coro_await_prepare_v1"); got != 2 { + t.Fatalf("mixed v2 bootstrap await calls = %d, want 2:\n%s", got, body) + } + if got := strings.Count(body, "call ptr %"); got != 2 { + t.Fatalf("mixed v2 bootstrap indirect child factory calls = %d, want 2:\n%s", got, body) + } + assertInOrder(t, body, + "call void @"+coroProgramFramePublishHookV1, + "call i8 @llvm.coro.suspend", + "store i16 2", + "call ptr %", + "store i16 1", + "store i16 3", + "call void @__llgo_coro_await_prepare_v1", + "call i8 @llvm.coro.suspend", + "call void @\"init$abitypes\"()", + "call void @runtime.init()", + "call ptr %", + "call void @__llgo_coro_await_prepare_v1", + "call i8 @llvm.coro.suspend", + "call void @\"example.com/program.main\"()", + "call void @"+coroProgramCompletePrepareHookV1, + ) +} + func llvmFunctionIRV1(ir, name string) string { quoted := "@" + name + "(" start := strings.Index(ir, quoted) diff --git a/internal/build/coro_bootstrap_test.go b/internal/build/coro_bootstrap_test.go index 0d956205d9..feef721588 100644 --- a/internal/build/coro_bootstrap_test.go +++ b/internal/build/coro_bootstrap_test.go @@ -25,6 +25,7 @@ import ( "go/parser" "go/token" "go/types" + "sort" "strings" "testing" @@ -176,34 +177,282 @@ func TestSelectCoroProgramBootstrapV1RejectsPatchedInitSymbol(t *testing.T) { } } -func TestCoroProgramBootstrapRejectsInvalidRootsBeforePackageCodegen(t *testing.T) { - conf := NewDefaultConf(ModeGen) - conf.EnableCoroEntryResolution = true - conf.EnableCoroPhysicalABI = true - conf.EnableCoroChildAwait = true - conf.EnableCoroProgramBootstrapABI = true - moduleCalls := 0 - conf.ModuleHook = func(Package) { moduleCalls++ } - conf.CoroPlanBuilder = func(input CoroPlanInput) (*coro.SSAPlan, error) { - mainFn, err := findSingleSSAMain(input.Program) - if err != nil { - return nil, err +func TestSelectCoroProgramBootstrapV2ExactMixedFiveStageProgram(t *testing.T) { + fixture := newCoroBootstrapV2TestContext(t) + bootstrap, err := selectCoroProgramBootstrapV2(fixture.ctx, fixture.mainPackage) + if err != nil { + t.Fatal(err) + } + if bootstrap == nil || bootstrap.Version != coroProgramBootstrapVersionV2 || len(bootstrap.Steps) != 5 { + t.Fatalf("v2 bootstrap = %+v, want version 2 and five steps", bootstrap) + } + if frozen := fixture.ctx.coroProgramBootstraps[fixture.mainPackage.ID]; frozen == nil || + frozen.StepHash != bootstrap.StepHash { + t.Fatalf("pre-codegen frozen bootstrap = %+v, want stable selection hash %x", frozen, bootstrap.StepHash) + } + + runtimeIndex := expectedCoroBootstrapV2DescriptorIndex(t, fixture.ctx.coroPlan, fixture.runtimeInit) + publicRuntimeIndex := expectedCoroBootstrapV2DescriptorIndex(t, fixture.ctx.coroPlan, fixture.publicRuntimeInit) + mainInitIndex := expectedCoroBootstrapV2DescriptorIndex(t, fixture.ctx.coroPlan, fixture.mainInit) + wants := []struct { + kind uint32 + role uint32 + target string + owner string + aux uint64 + }{ + { + kind: coroProgramStepCoroRootV1, role: coroProgramStepRoleRuntimeInitV2, + target: llssa.PkgRuntime + ".init$coro", owner: llssa.PkgRuntime, aux: runtimeIndex, + }, + { + kind: coroProgramStepDirectPlainV1, role: coroProgramStepRoleABIInitV2, + target: "init$abitypes", + }, + { + kind: coroProgramStepCoroRootV1, role: coroProgramStepRolePublicRuntimeInitV2, + target: "runtime.init$coro", owner: "runtime", aux: publicRuntimeIndex, + }, + { + kind: coroProgramStepCoroRootV1, role: coroProgramStepRolePackageInitV2, + target: fixture.mainPackage.PkgPath + ".init$coro", owner: fixture.mainPackage.PkgPath, aux: mainInitIndex, + }, + { + kind: coroProgramStepDirectPlainV1, role: coroProgramStepRoleMainV2, + target: fixture.mainPackage.PkgPath + ".main", + }, + } + for index, want := range wants { + got := bootstrap.Steps[index] + if got.Kind != want.kind || got.Role != want.role || got.Target != want.target || + got.Owner != want.owner || got.Aux != want.aux || got.FunctionID == "" || got.CatalogTarget != "" { + t.Fatalf("v2 step %d = %+v, want kind=%d role=%d target=%q owner=%q aux=%d, nonempty ID and unbound catalog", + index, got, want.kind, want.role, want.target, want.owner, want.aux) } - // Deliberately omit the synthetic main-package init root. It may still - // exist in the plan, but the startup ABI requires an explicit root. - return input.Analyze(coro.Roots{{Function: mainFn, Demand: coro.AsyncDemand}}, coro.SSAConfig{ - MaxPlainInstructions: -1, + } + + for _, check := range []struct { + name string + fn *ssa.Function + kind uint32 + }{ + {name: "runtime init", fn: fixture.runtimeInit, kind: coroProgramStepCoroRootV1}, + {name: "public runtime init", fn: fixture.publicRuntimeInit, kind: coroProgramStepCoroRootV1}, + {name: "main package init", fn: fixture.mainInit, kind: coroProgramStepCoroRootV1}, + {name: "main", fn: fixture.mainMain, kind: coroProgramStepDirectPlainV1}, + } { + plan, ok := fixture.ctx.coroPlan.FunctionPlan(check.fn) + if !ok { + t.Fatalf("%s has no exact function plan", check.name) + } + if check.kind == coroProgramStepCoroRootV1 { + if plan.Emission != coro.EmitCoroutine || plan.FuncRep != coro.DirectCoro || plan.Primary != coro.PrimaryCoroutine { + t.Fatalf("%s plan = %+v, want one direct coroutine primary", check.name, plan) + } + } else if plan.Emission != coro.EmitPlain || plan.FuncRep != coro.DirectPlain || plan.Primary != coro.PrimaryPlain { + t.Fatalf("%s plan = %+v, want one direct plain primary", check.name, plan) + } + } +} + +func TestSelectCoroProgramBootstrapV2UsesOwnedNoopWhenPublicRuntimeIsAbsent(t *testing.T) { + fixture := newCoroBootstrapV2TestContextWithPublicRuntime(t, false) + bootstrap, err := selectCoroProgramBootstrapV2(fixture.ctx, fixture.mainPackage) + if err != nil { + t.Fatal(err) + } + if bootstrap == nil || len(bootstrap.Steps) != 5 { + t.Fatalf("v2 bootstrap = %+v, want five fixed roles", bootstrap) + } + step := bootstrap.Steps[2] + if step.Kind != coroProgramStepDirectPlainV1 || step.Role != coroProgramStepRolePublicRuntimeInitV2 || + step.FunctionID != coroProgramPublicRuntimeNoopIDV2 || step.Target != coroProgramPublicRuntimeNoopSymbolV2 || + step.Owner != "" || step.CatalogTarget != "" || step.Aux != 0 { + t.Fatalf("absent public runtime step = %+v, want compiler-owned canonical no-op", step) + } + for _, root := range fixture.ctx.coroPlan.Roots() { + if root.Function != nil && root.Function.Pkg != nil && root.Function.Pkg.Pkg != nil && + llssa.PathOf(root.Function.Pkg.Pkg) == "runtime" { + t.Fatalf("absent public runtime created a guessed managed root: %+v", root) + } + } +} + +func TestSelectCoroProgramBootstrapV2AllowsIRQUnsafeOnOrdinaryG(t *testing.T) { + fixture := newCoroBootstrapV2TestContext(t) + step, err := selectCoroProgramManagedStepV2( + fixture.ctx, + fixture.irqRuntimeRoot, + llssa.PkgRuntime+".irqRuntimeRoot", + llssa.PkgRuntime, + "IRQ-unsafe bounded startup fixture", + coroProgramStepRoleRuntimeInitV2, + ) + if err != nil { + t.Fatal(err) + } + plan, ok := fixture.ctx.coroPlan.FunctionPlan(fixture.irqRuntimeRoot) + if !ok || plan.Effect != coro.NoSuspend || !plan.Exec.Contains(coro.IRQUnsafe) || plan.Exec.Contains(coro.ThreadAffine) { + t.Fatalf("IRQ-unsafe fixture plan = %+v, present=%t", plan, ok) + } + if step.Kind != coroProgramStepDirectPlainV1 || step.Target != llssa.PkgRuntime+".irqRuntimeRoot" || step.Owner != "" { + t.Fatalf("IRQ-unsafe ordinary-G step = %+v, want direct plain", step) + } +} + +func TestBindCoroProgramBootstrapV2OwnersAndAnchors(t *testing.T) { + fixture := newCoroBootstrapV2TestContext(t) + semantic := fixture.ctx.coroProgramBootstraps[fixture.mainPackage.ID] + const ( + runtimeAnchor = coroRootPackageAnchorPrefixV1 + "11111111111111111111111111111111" + publicRuntimeAnchor = coroRootPackageAnchorPrefixV1 + "22222222222222222222222222222222" + mainAnchor = coroRootPackageAnchorPrefixV1 + "33333333333333333333333333333333" + ) + linked := []Package{ + coroBootstrapV2LinkedPackage(llssa.PkgRuntime, runtimeAnchor), + coroBootstrapV2LinkedPackage("runtime", publicRuntimeAnchor), + coroBootstrapV2LinkedPackage(fixture.mainPackage.PkgPath, mainAnchor), + } + bound, err := bindCoroProgramBootstrapV2(semantic, linked) + if err != nil { + t.Fatal(err) + } + if bound == semantic || &bound.Steps[0] == &semantic.Steps[0] { + t.Fatal("v2 binding mutated or aliased the immutable semantic bootstrap") + } + for index, step := range semantic.Steps { + if step.CatalogTarget != "" { + t.Fatalf("semantic step %d was modified by binding: %+v", index, step) + } + } + for index, step := range bound.Steps { + switch step.Owner { + case llssa.PkgRuntime: + if step.CatalogTarget != runtimeAnchor { + t.Fatalf("runtime-owned step %d bound to %q, want %q", index, step.CatalogTarget, runtimeAnchor) + } + case "runtime": + if step.CatalogTarget != publicRuntimeAnchor { + t.Fatalf("public-runtime-owned step %d bound to %q, want %q", index, step.CatalogTarget, publicRuntimeAnchor) + } + case fixture.mainPackage.PkgPath: + if step.CatalogTarget != mainAnchor { + t.Fatalf("main-owned step %d bound to %q, want %q", index, step.CatalogTarget, mainAnchor) + } + default: + if step.Kind != coroProgramStepDirectPlainV1 || step.CatalogTarget != "" { + t.Fatalf("compiler-owned step %d acquired catalog state: %+v", index, step) + } + } + } +} + +func TestBindCoroProgramBootstrapV2RejectsMissingConflictingAndInvalidAnchors(t *testing.T) { + fixture := newCoroBootstrapV2TestContext(t) + semantic := fixture.ctx.coroProgramBootstraps[fixture.mainPackage.ID] + const ( + anchorA = coroRootPackageAnchorPrefixV1 + "11111111111111111111111111111111" + anchorB = coroRootPackageAnchorPrefixV1 + "22222222222222222222222222222222" + anchorC = coroRootPackageAnchorPrefixV1 + "33333333333333333333333333333333" + ) + tests := []struct { + name string + linked []Package + want string + }{ + { + name: "missing main owner", + linked: []Package{ + coroBootstrapV2LinkedPackage(llssa.PkgRuntime, anchorA), + coroBootstrapV2LinkedPackage("runtime", anchorB), + }, + want: `owner "example.com/bootstrapv2" has no linked root anchor`, + }, + { + name: "conflicting runtime owner", + linked: []Package{ + coroBootstrapV2LinkedPackage(llssa.PkgRuntime, anchorA), + coroBootstrapV2LinkedPackage(llssa.PkgRuntime, anchorB), + coroBootstrapV2LinkedPackage("runtime", anchorB), + coroBootstrapV2LinkedPackage(fixture.mainPackage.PkgPath, anchorC), + }, + want: "conflicting coroutine root anchors", + }, + { + name: "invalid runtime anchor", + linked: []Package{ + coroBootstrapV2LinkedPackage(llssa.PkgRuntime, "invalid"), + coroBootstrapV2LinkedPackage("runtime", anchorB), + coroBootstrapV2LinkedPackage(fixture.mainPackage.PkgPath, anchorC), + }, + want: "invalid coroutine root anchor", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + bound, err := bindCoroProgramBootstrapV2(semantic, test.linked) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("bind result = %+v, %v; want error containing %q", bound, err, test.want) + } + for index, step := range semantic.Steps { + if step.CatalogTarget != "" { + t.Fatalf("failed binding modified semantic step %d: %+v", index, step) + } + } }) } - pkgs, err := Do([]string{"../../cl/_testgo/print"}, conf) - if err == nil || !strings.Contains(err.Error(), "init: target is not an explicit plan root") { - t.Fatalf("Do error = %v, want missing explicit init-root rejection", err) +} + +func TestCoroProgramManifestHashV1CoversV2OwnerAnchorBinding(t *testing.T) { + fixture := newCoroBootstrapV2TestContext(t) + semantic := fixture.ctx.coroProgramBootstraps[fixture.mainPackage.ID] + const ( + anchorA = coroRootPackageAnchorPrefixV1 + "11111111111111111111111111111111" + anchorB = coroRootPackageAnchorPrefixV1 + "22222222222222222222222222222222" + anchorC = coroRootPackageAnchorPrefixV1 + "33333333333333333333333333333333" + ) + firstLinked := []Package{ + coroBootstrapV2LinkedPackage(llssa.PkgRuntime, anchorA), + coroBootstrapV2LinkedPackage("runtime", anchorB), + coroBootstrapV2LinkedPackage(fixture.mainPackage.PkgPath, anchorC), + } + secondLinked := []Package{ + coroBootstrapV2LinkedPackage(llssa.PkgRuntime, anchorC), + coroBootstrapV2LinkedPackage("runtime", anchorB), + coroBootstrapV2LinkedPackage(fixture.mainPackage.PkgPath, anchorA), + } + first, err := bindCoroProgramBootstrapV2(semantic, firstLinked) + if err != nil { + t.Fatal(err) + } + second, err := bindCoroProgramBootstrapV2(semantic, secondLinked) + if err != nil { + t.Fatal(err) + } + firstCatalog, err := collectLinkedCoroRootAnchors(firstLinked) + if err != nil { + t.Fatal(err) + } + secondCatalog, err := collectLinkedCoroRootAnchors(secondLinked) + if err != nil { + t.Fatal(err) + } + if strings.Join(firstCatalog, "\x00") != strings.Join(secondCatalog, "\x00") { + t.Fatalf("test did not preserve the same sorted anchor catalog: %q != %q", firstCatalog, secondCatalog) } - if len(pkgs) != 0 { - t.Fatalf("Do packages = %+v, want none", pkgs) + if first.StepHash != second.StepHash || first.StepHash != semantic.StepHash { + t.Fatalf("binding changed semantic StepHash: %x, %x, want %x", first.StepHash, second.StepHash, semantic.StepHash) } - if moduleCalls != 0 { - t.Fatalf("ModuleHook calls = %d, want zero before-codegen rejection", moduleCalls) + firstHash, err := coroProgramManifestHashV1(fixture.ctx, firstCatalog, first) + if err != nil { + t.Fatal(err) + } + secondHash, err := coroProgramManifestHashV1(fixture.ctx, secondCatalog, second) + if err != nil { + t.Fatal(err) + } + if firstHash == secondHash { + t.Fatalf("final manifest hash ignored owner-to-CatalogTarget binding: %x", firstHash) } } @@ -265,6 +514,205 @@ func TestCoroProgramBootstrapHashV1StableAndStepComplete(t *testing.T) { } } +type coroBootstrapV2TestFixture struct { + ctx *context + mainPackage *packages.Package + runtimeInit *ssa.Function + irqRuntimeRoot *ssa.Function + publicRuntimeInit *ssa.Function + mainInit *ssa.Function + mainMain *ssa.Function +} + +func newCoroBootstrapV2TestContext(t *testing.T) coroBootstrapV2TestFixture { + return newCoroBootstrapV2TestContextWithPublicRuntime(t, true) +} + +func newCoroBootstrapV2TestContextWithPublicRuntime(t *testing.T, includePublicRuntime bool) coroBootstrapV2TestFixture { + t.Helper() + fset := token.NewFileSet() + ssaProg := ssa.NewProgram(fset, ssa.SanityCheckFunctions|ssa.InstantiateGenerics) + runtimeSSA, runtimeFiles, _, _ := createCoroBootstrapV2SSAPackage(t, ssaProg, fset, llssa.PkgRuntime, `package runtime +func aRuntimeRoot() {} +func irqRuntimeRoot() {} +func zRuntimeRoot() {} +`) + var publicRuntimeSSA *ssa.Package + var publicRuntimeFiles []*ast.File + if includePublicRuntime { + publicRuntimeSSA, publicRuntimeFiles, _, _ = createCoroBootstrapV2SSAPackage(t, ssaProg, fset, "runtime", `package runtime +func publicRuntimeBody() {} +`) + } + mainSSA, mainFiles, mainTypes, mainInfo := createCoroBootstrapV2SSAPackage(t, ssaProg, fset, "example.com/bootstrapv2", `package main +func aMainRoot() {} +func zMainRoot() {} +func main() {} +`) + ssaProg.Build() + + prog := llssa.NewProgram(nil) + t.Cleanup(prog.Dispose) + emissionInputs := []cl.EmissionPackage{ + {SSA: runtimeSSA, Files: runtimeFiles, Identity: llssa.PkgRuntime}, + {SSA: mainSSA, Files: mainFiles, Identity: "example.com/bootstrapv2"}, + } + if includePublicRuntime { + emissionInputs = append(emissionInputs[:1], append([]cl.EmissionPackage{ + {SSA: publicRuntimeSSA, Files: publicRuntimeFiles, Identity: "runtime"}, + }, emissionInputs[1:]...)...) + } + emission, err := cl.PrepareEmissionUniverse(prog, nil, emissionInputs) + if err != nil { + t.Fatal(err) + } + ssaEmission, err := coro.NewSSAEmissionUniverse(ssaProg, emission.Functions()) + if err != nil { + t.Fatal(err) + } + mainPackage := &packages.Package{ + ID: "example.com/bootstrapv2", PkgPath: "example.com/bootstrapv2", Name: "main", + Types: mainTypes, TypesInfo: mainInfo, Syntax: mainFiles, + } + aMain := &aPackage{Package: mainPackage, SSA: mainSSA} + runtimeInit := runtimeSSA.Func("init") + irqRuntimeRoot := runtimeSSA.Func("irqRuntimeRoot") + var publicRuntimeInit *ssa.Function + if publicRuntimeSSA != nil { + publicRuntimeInit = publicRuntimeSSA.Func("init") + } + mainInit := mainSSA.Func("init") + mainMain := mainSSA.Func("main") + suspending := map[*ssa.Function]bool{ + runtimeInit: true, + runtimeSSA.Func("aRuntimeRoot"): true, + runtimeSSA.Func("zRuntimeRoot"): true, + mainInit: true, + mainSSA.Func("aMainRoot"): true, + mainSSA.Func("zMainRoot"): true, + } + if publicRuntimeInit != nil { + suspending[publicRuntimeInit] = true + } + conf := &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + EnableCoroEntryResolution: true, + EnableCoroPhysicalABI: true, + EnableCoroChildAwait: true, + EnableCoroProgramBootstrapABI: true, + EnableCoroProgramBootstrapRun: true, + } + conf.CoroPlanBuilder = func(input CoroPlanInput) (*coro.SSAPlan, error) { + // Deliberately provide roots in reverse name order. Descriptor Aux must + // follow the canonical same-package FunctionID order, never this input + // order or whole-program package order. + roots := coro.Roots{ + {Function: mainSSA.Func("zMainRoot"), Demand: coro.AsyncDemand}, + {Function: mainSSA.Func("aMainRoot"), Demand: coro.AsyncDemand}, + {Function: runtimeSSA.Func("zRuntimeRoot"), Demand: coro.AsyncDemand}, + {Function: irqRuntimeRoot, Demand: coro.AsyncDemand}, + {Function: runtimeInit, Demand: coro.AsyncDemand}, + {Function: runtimeSSA.Func("aRuntimeRoot"), Demand: coro.AsyncDemand}, + } + return input.Analyze(roots, coro.SSAConfig{ + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == irqRuntimeRoot { + return coro.SSAFunctionPolicy{Exec: coro.IRQUnsafe}, nil + } + if suspending[fn] { + return coro.SSAFunctionPolicy{Effect: coro.MayPark}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + }) + } + ctx := &context{ + progSSA: ssaProg, + prog: prog, + patches: make(cl.Patches), + initial: []*packages.Package{mainPackage}, + pkgs: map[*packages.Package]Package{mainPackage: aMain}, + pkgByID: map[string]Package{mainPackage.ID: aMain}, + mode: ModeBuild, + buildConf: conf, + coroEmission: emission, + coroSSAEmission: ssaEmission, + coroPlanMetadata: coro.PlanDigestMetadata{}, + } + if err := buildCoroPlan(ctx); err != nil { + t.Fatalf("build v2 coroutine bootstrap test plan: %v", err) + } + return coroBootstrapV2TestFixture{ + ctx: ctx, mainPackage: mainPackage, + runtimeInit: runtimeInit, irqRuntimeRoot: irqRuntimeRoot, publicRuntimeInit: publicRuntimeInit, + mainInit: mainInit, mainMain: mainMain, + } +} + +func createCoroBootstrapV2SSAPackage( + t *testing.T, prog *ssa.Program, fset *token.FileSet, pkgPath, source string, +) (*ssa.Package, []*ast.File, *types.Package, *types.Info) { + t.Helper() + file, err := parser.ParseFile(fset, pkgPath+".go", source, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + files := []*ast.File{file} + info := &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + Implicits: make(map[ast.Node]types.Object), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + Scopes: make(map[ast.Node]*types.Scope), + } + typesPkg, err := (&types.Config{}).Check(pkgPath, fset, files, info) + if err != nil { + t.Fatal(err) + } + return prog.CreatePackage(typesPkg, files, info, true), files, typesPkg, info +} + +func expectedCoroBootstrapV2DescriptorIndex(t *testing.T, plan *coro.SSAPlan, target *ssa.Function) uint64 { + t.Helper() + var ids []coro.FunctionID + for _, root := range plan.Roots() { + fnPlan, ok := plan.FunctionPlan(root.Function) + if ok && root.Function.Pkg == target.Pkg && fnPlan.Emission == coro.EmitCoroutine { + ids = append(ids, root.ID) + } + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + targetPlan, ok := plan.FunctionPlan(target) + if !ok { + t.Fatalf("descriptor target %q has no function plan", target.Name()) + } + for index, id := range ids { + if id == targetPlan.ID { + got, err := coroProgramRootDescriptorIndexV2(plan, target) + if err != nil { + t.Fatal(err) + } + if got != uint64(index) { + t.Fatalf("descriptor index for %q = %d, want FunctionID-sorted index %d in %q", target.Name(), got, index, ids) + } + return uint64(index) + } + } + t.Fatalf("descriptor target %q ID %q is absent from sorted coroutine roots %q", target.Name(), targetPlan.ID, ids) + return 0 +} + +func coroBootstrapV2LinkedPackage(pkgPath, anchor string) Package { + return &aPackage{ + Package: &packages.Package{ID: pkgPath, PkgPath: pkgPath}, + CoroRootAnchorV1: anchor, + } +} + func newCoroBootstrapTestContext(t *testing.T, target *llssa.Target, spec coroBootstrapTestPlan) (*context, *packages.Package) { t.Helper() ctx, pkg, err := buildCoroBootstrapTestContext(t, target, spec) diff --git a/internal/build/coro_foreign_noblock_test.go b/internal/build/coro_foreign_noblock_test.go new file mode 100644 index 0000000000..b65f865477 --- /dev/null +++ b/internal/build/coro_foreign_noblock_test.go @@ -0,0 +1,157 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "go/ast" + "strings" + "testing" + + "github.com/goplus/llgo/cl" + "github.com/goplus/llgo/internal/coro" + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +func TestCoroPlanInputUsesOnlyFrozenForeignNoBlockCertificate(t *testing.T) { + ssaPkg, files := buildCoroPlanTestPackage(t, "example.com/noblock", `package noblock +//llgo:coro noblock +//go:linkname Safe C.safe_exact +func Safe(int) int +//go:linkname Memcpy C.memcpy +func Memcpy(uintptr) +func SafeCaller() int { return Safe(1) } +func OrdinaryCaller(n uintptr) { Memcpy(n) } +`, nil) + prog := llssa.NewProgram(nil) + defer prog.Dispose() + emission, err := cl.PrepareEmissionUniverse(prog, nil, []cl.EmissionPackage{{ + SSA: ssaPkg, Files: []*ast.File{files[0]}, Identity: "example.com/noblock", + }}) + if err != nil { + t.Fatal(err) + } + ssaEmission, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, emission.Functions()) + if err != nil { + t.Fatal(err) + } + input := CoroPlanInput{ + Program: ssaPkg.Prog, + EmissionUniverse: ssaEmission, + resolveFunction: emission.Resolve, + functionBackground: emission.FunctionBackground, + foreignNoBlock: emission.CoroForeignNoBlockCertificate, + } + functionIDs := emission.FunctionIDConfig() + functionIDs.CoroABI = coro.EntryResolutionABIV0 + functionIDs.SchedulerABI = coro.SchedulerNoneABIV0 + functionIDs.ArchiveReady = true + roots := coro.Roots{ + {Function: ssaPkg.Func("SafeCaller"), Demand: coro.SyncDemand}, + {Function: ssaPkg.Func("OrdinaryCaller"), Demand: coro.SyncDemand}, + } + analyze := func(in CoroPlanInput, classify func(*ssa.Function) (coro.SSAFunctionPolicy, error)) (*coro.SSAPlan, error) { + return in.Analyze(roots, coro.SSAConfig{ + MaxPlainInstructions: -1, + FunctionIDs: functionIDs, + ClassifyFunction: classify, + }) + } + plan, err := analyze(input, nil) + if err != nil { + t.Fatal(err) + } + safe := ssaPkg.Func("Safe") + certificate, ok := plan.ForeignNoBlockCertificate(safe) + if !ok || certificate == "" { + t.Fatal("certified C declaration lost its exact proof in SSAPlan") + } + safePlan, _ := plan.FunctionPlan(safe) + if safePlan.External != coro.ExternalKnown || safePlan.Effect != coro.NoSuspend || safePlan.Exec != coro.IRQUnsafe || + safePlan.Exec.Contains(coro.BlockForeign) || safePlan.Emission != coro.EmitExternal { + t.Fatalf("certified Safe plan = %+v; want external-known/no-suspend/irq-unsafe without block-foreign", safePlan) + } + safeCaller, _ := plan.FunctionPlan(ssaPkg.Func("SafeCaller")) + if safeCaller.Effect != coro.NoSuspend || !safeCaller.Exec.Contains(coro.IRQUnsafe) || safeCaller.Exec.Contains(coro.BlockForeign) { + t.Fatalf("SafeCaller plan = %+v; want direct bounded foreign call with retained IRQUnsafe", safeCaller) + } + ordinary, _ := plan.FunctionPlan(ssaPkg.Func("Memcpy")) + ordinaryCaller, _ := plan.FunctionPlan(ssaPkg.Func("OrdinaryCaller")) + if ordinary.External != coro.ExternalUnknownForeign || !ordinary.Exec.Contains(coro.BlockForeign|coro.IRQUnsafe) || + !ordinaryCaller.Effect.Contains(coro.WaitForeign) { + t.Fatalf("ordinary foreign plans = leaf:%+v caller:%+v; want default fail-closed boundary", ordinary, ordinaryCaller) + } + + _, err = analyze(input, func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == safe { + return coro.SSAFunctionPolicy{ForeignNoBlockCertificate: "forged"}, nil + } + return coro.SSAFunctionPolicy{}, nil + }) + if err == nil || !strings.Contains(err.Error(), "conflicts with the frozen frontend proof") { + t.Fatalf("conflicting builder certificate error = %v; want fail-closed mismatch", err) + } + forgedInput := input + forgedInput.foreignNoBlock = nil + _, err = analyze(forgedInput, func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == safe { + return coro.SSAFunctionPolicy{ForeignNoBlockCertificate: certificate}, nil + } + return coro.SSAFunctionPolicy{}, nil + }) + if err == nil || !strings.Contains(err.Error(), "without exact frozen frontend noblock metadata") { + t.Fatalf("unfrozen builder certificate error = %v; want fail-closed rejection", err) + } + + // Build the same effective function/call plan without retaining the source + // proof. The private certificate must still change the archive cache digest. + uncertifiedInput := input + uncertifiedInput.foreignNoBlock = nil + uncertified, err := analyze(uncertifiedInput, func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == safe { + return coro.SSAFunctionPolicy{ + IgnoreBody: true, External: coro.ExternalKnown, OverrideExternal: true, Exec: coro.IRQUnsafe, + }, nil + } + return coro.SSAFunctionPolicy{}, nil + }) + if err != nil { + t.Fatal(err) + } + if got, _ := uncertified.FunctionPlan(safe); got != safePlan { + t.Fatalf("uncertified effective Safe plan = %+v, want same %+v", got, safePlan) + } + metadata := coro.PlanDigestMetadata{ + CoroABI: coro.EntryResolutionABIV0, SchedulerABI: coro.SchedulerNoneABIV0, + PanicABI: coro.PanicLegacyABIV0, FuncRepABI: coro.FuncRepABIV0, + TargetTriple: "x86_64-unknown-linux-gnu", PointerBits: 64, + Endianness: "little", DataLayout: "e-p:64:64", + } + certifiedDigest, err := plan.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + uncertifiedDigest, err := uncertified.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if certifiedDigest == uncertifiedDigest { + t.Fatalf("foreign noblock certificate did not change archive plan digest %q", certifiedDigest) + } +} diff --git a/internal/build/coro_funcaddr_test.go b/internal/build/coro_funcaddr_test.go new file mode 100644 index 0000000000..ae2f5df4df --- /dev/null +++ b/internal/build/coro_funcaddr_test.go @@ -0,0 +1,209 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "fmt" + "go/ast" + "strings" + "testing" + + "github.com/goplus/llgo/cl" + "github.com/goplus/llgo/internal/coro" + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +func TestCoroFuncAddrUsesExactRawAddressConsumer(t *testing.T) { + ssaPkg, files := buildCoroPlanTestPackage(t, "example.com/coro/funcaddr", `package funcaddr +import "unsafe" +//llgo:link Func llgo.funcAddr +func Func(any) unsafe.Pointer +func target() {} +func root() unsafe.Pointer { return Func(target) } +`, nil) + plan, emission, call, err := analyzeCoroFuncAddrTest(t, ssaPkg, files) + if err != nil { + t.Fatal(err) + } + semantics, intrinsic, err := emission.CoroIntrinsicCallSiteSemantics(call) + if err != nil || !intrinsic || semantics != cl.CoroIntrinsicCallInlineNoSuspend { + t.Fatalf("funcAddr semantics = %v, %v, %v; want inline-no-suspend, true, nil", semantics, intrinsic, err) + } + if !plan.ElidesCall(call) || !plan.RawFunctionAddressArgument(call, 0) { + t.Fatalf("funcAddr plan elided=%t raw-argument=%t; want both true", plan.ElidesCall(call), plan.RawFunctionAddressArgument(call, 0)) + } + if _, ok := plan.CallPlan(call); ok { + t.Fatal("funcAddr intrinsic declaration unexpectedly retained a CallPlan") + } + target := ssaPkg.Func("target") + targetPlan, ok := plan.FunctionPlan(target) + if !ok || targetPlan.FuncRep != coro.DirectPlain { + t.Fatalf("raw-only funcAddr target plan = %+v, %v; want direct-plain without dispatch", targetPlan, ok) + } + valuePlan, ok := plan.ValuePlan(target) + if !ok || len(valuePlan.Funcs) != 1 || valuePlan.Funcs[0].Rep != coro.DirectPlain { + t.Fatalf("raw-only funcAddr target value plan = %+v, %v; want direct-plain", valuePlan, ok) + } +} + +func TestCoroFuncAddrRawAddressFactIsConsumerScoped(t *testing.T) { + ssaPkg, files := buildCoroPlanTestPackage(t, "example.com/coro/funcaddrscoped", `package funcaddrscoped +import "unsafe" +//llgo:link Func llgo.funcAddr +func Func(any) unsafe.Pointer +var published any +func target() {} +func publish() { published = target } +func root() unsafe.Pointer { return Func(target) } +`, nil) + plan, _, call, err := analyzeCoroFuncAddrTest(t, ssaPkg, files) + if err != nil { + t.Fatal(err) + } + if !plan.RawFunctionAddressArgument(call, 0) { + t.Fatal("exact funcAddr consumer lost its raw-address fact") + } + targetPlan, ok := plan.FunctionPlan(ssaPkg.Func("target")) + if !ok || targetPlan.FuncRep != coro.Dispatch { + t.Fatalf("target with an ordinary interface publication = %+v, %v; want Dispatch", targetPlan, ok) + } +} + +func TestCoroFuncAddrRejectsNonExactSites(t *testing.T) { + tests := []struct { + name string + source string + wantErr string + }{ + { + name: "dynamic any", + source: `package funcaddrinvalid +import "unsafe" +//llgo:link Func llgo.funcAddr +func Func(any) unsafe.Pointer +func root(value any) unsafe.Pointer { return Func(value) } +`, + wantErr: "want *ssa.MakeInterface", + }, + { + name: "non-function payload", + source: `package funcaddrinvalid +import "unsafe" +//llgo:link Func llgo.funcAddr +func Func(any) unsafe.Pointer +func root() unsafe.Pointer { return Func(1) } +`, + wantErr: "requires MakeInterface{X:*ssa.Function}", + }, + { + name: "captured closure", + source: `package funcaddrinvalid +import "unsafe" +//llgo:link Func llgo.funcAddr +func Func(any) unsafe.Pointer +func root(value int) unsafe.Pointer { + fn := func() { _ = value } + return Func(fn) +} +`, + wantErr: "requires MakeInterface{X:*ssa.Function}", + }, + { + name: "shared interface consumer", + source: `package funcaddrinvalid +import "unsafe" +//llgo:link Func llgo.funcAddr +func Func(any) unsafe.Pointer +func consume(any) {} +func target() {} +func root() unsafe.Pointer { + value := any(target) + consume(value) + return Func(value) +} +`, + wantErr: "exact sole consumer", + }, + { + name: "wrong result", + source: `package funcaddrinvalid +//llgo:link Func llgo.funcAddr +func Func(any) uintptr +func target() {} +func root() uintptr { return Func(target) } +`, + wantErr: "exact func(any) unsafe.Pointer shape", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ssaPkg, files := buildCoroPlanTestPackage(t, "example.com/coro/funcaddrinvalid", test.source, nil) + _, _, _, err := analyzeCoroFuncAddrTest(t, ssaPkg, files) + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("funcAddr invalid-site error = %v; want %q", err, test.wantErr) + } + }) + } +} + +func analyzeCoroFuncAddrTest(t *testing.T, ssaPkg *ssa.Package, files []*ast.File) (*coro.SSAPlan, *cl.EmissionUniverse, ssa.CallInstruction, error) { + t.Helper() + prog := llssa.NewProgram(nil) + t.Cleanup(prog.Dispose) + emission, err := cl.PrepareEmissionUniverse(prog, nil, []cl.EmissionPackage{{ + SSA: ssaPkg, Files: files, Identity: ssaPkg.Pkg.Path(), + }}) + if err != nil { + return nil, nil, nil, err + } + ssaEmission, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, emission.Functions()) + if err != nil { + return nil, nil, nil, err + } + root := ssaPkg.Func("root") + var intrinsicCall ssa.CallInstruction + for _, call := range coroPlanTestCalls(root) { + if callee := call.Common().StaticCallee(); callee != nil && callee.Name() == "Func" { + intrinsicCall = call + break + } + } + if intrinsicCall == nil { + return nil, nil, nil, fmt.Errorf("root has no funcAddr call") + } + input := CoroPlanInput{ + Program: ssaPkg.Prog, + EmissionUniverse: ssaEmission, + resolveFunction: emission.Resolve, + functionBackground: emission.FunctionBackground, + intrinsicCallSemantics: emission.CoroIntrinsicCallSiteSemantics, + rawFunctionAddressCallArgument: emission.CoroRawFunctionAddressCallArgument, + } + functionIDs := emission.FunctionIDConfig() + functionIDs.CoroABI = coro.EntryResolutionABIV0 + functionIDs.SchedulerABI = coro.SchedulerNoneABIV0 + functionIDs.ArchiveReady = true + plan, err := input.Analyze(coro.Roots{{Function: root, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + MaxPlainInstructions: -1, + FunctionIDs: functionIDs, + }) + return plan, emission, intrinsicCall, err +} diff --git a/internal/build/coro_panic_legacy_test.go b/internal/build/coro_panic_legacy_test.go new file mode 100644 index 0000000000..97dd3c89e8 --- /dev/null +++ b/internal/build/coro_panic_legacy_test.go @@ -0,0 +1,59 @@ +//go:build !llgo + +package build + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/goplus/llgo/internal/coro" +) + +// This is an integration regression for the real runtime closure, not a probe. +// It keeps the legacy ABI fail-closed at the first user-code-capable terminal +// panic edge. If Rethrow's terminal-unhandled branch is later split behind a +// different panic ABI adapter, this expected chain must be deliberately +// replaced by the new adapter's certificate test. +func TestRealRuntimeLegacyPanicPlainCertificateStopsAtDynamicError(t *testing.T) { + sentinel := errors.New("legacy panic blocker verified") + conf := NewDefaultConf(ModeGen) + conf.ForceRebuild = true + conf.EnableCoroEntryResolution = true + conf.EnableCoroPhysicalABI = true + conf.EnableCoroChildAwait = true + conf.EnableCoroPlainDispatch = true + conf.EnableCoroProgramBootstrapABI = true + conf.EnableCoroProgramBootstrapRun = true + conf.CoroPlanBuilder = func(input CoroPlanInput) (*coro.SSAPlan, error) { + plan, err := input.Analyze(nil, coro.SSAConfig{MaxPlainInstructions: -1}) + if err != nil { + return nil, err + } + err = validateCoroUnwindOnlyLoweredCalls(plan, coro.PanicLegacyABIV0) + if err == nil { + return nil, fmt.Errorf("real runtime legacy panic closure unexpectedly received a plain certificate") + } + message := err.Error() + cursor := 0 + for _, part := range []string{ + "runtime.Panic[", + "runtime.Rethrow[", + "runtime.TracePanic[", + "runtime.printany[", + "dynamic invoke Error", + } { + index := strings.Index(message[cursor:], part) + if index < 0 { + return nil, fmt.Errorf("real runtime legacy panic blocker %q lacks ordered path component %q", message, part) + } + cursor += index + len(part) + } + return nil, sentinel + } + _, err := Do([]string{"../../cl/_testgo/print"}, conf) + if !errors.Is(err, sentinel) { + t.Fatalf("Do error = %v, want verified legacy panic blocker", err) + } +} diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index 2b9753e1db..d5a6056fc5 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -192,6 +192,50 @@ func root() { _ = CStr(1) } `, wantErr: "requires exactly one compile-time string constant argument", }, + { + name: "advance pointer by integer", + source: `package intrinsiccalls +//llgo:link Advance llgo.advance +func Advance(*int, int) *int +func root(value *int) { _ = Advance(value, 1) } +`, + }, + { + name: "advance wrong arity", + source: `package intrinsiccalls +//llgo:link Advance llgo.advance +func Advance(*int, int, int) *int +func root(value *int) { _ = Advance(value, 1, 2) } +`, + wantErr: "requires exactly two arguments", + }, + { + name: "advance non-pointer", + source: `package intrinsiccalls +//llgo:link Advance llgo.advance +func Advance(int, int) int +func root(value int) { _ = Advance(value, 1) } +`, + wantErr: "requires a pointer first argument", + }, + { + name: "advance non-integer offset", + source: `package intrinsiccalls +//llgo:link Advance llgo.advance +func Advance(*int, string) *int +func root(value *int) { _ = Advance(value, "1") } +`, + wantErr: "requires an integer offset argument", + }, + { + name: "advance mismatched result", + source: `package intrinsiccalls +//llgo:link Advance llgo.advance +func Advance(*int, int) *byte +func root(value *int) { _ = Advance(value, 1) } +`, + wantErr: "requires one result matching its pointer argument", + }, } for _, test := range tests { @@ -245,10 +289,10 @@ func root() { _ = CStr(1) } t.Fatalf("alias intrinsic site semantics = %v, %v, %v; want inline-no-suspend, true, nil", semantics, intrinsic, err) } if !plan.ElidesCall(call) { - t.Fatal("valid aliased cstr site was not retained as exact elided call") + t.Fatal("valid intrinsic site was not retained as exact elided call") } if _, ok := plan.CallPlan(call); ok { - t.Fatal("valid aliased cstr site unexpectedly has a managed CallPlan") + t.Fatal("valid intrinsic site unexpectedly has a managed CallPlan") } metadata := coro.PlanDigestMetadata{ CoroABI: coro.EntryResolutionABIV0, SchedulerABI: coro.SchedulerNoneABIV0, @@ -272,13 +316,105 @@ func root() { _ = CStr(1) } } } +func TestCoroParkIntrinsicSeedsCallerEffectAndStableDigest(t *testing.T) { + ssaPkg, files := buildCoroPlanTestPackage(t, "example.com/coropark", `package coropark +type WaitToken struct { word uint32 } +type WaitTicket uint32 +//llgo:link Park llgo.coroPark +func Park(*WaitToken, WaitTicket) +func root(token *WaitToken, ticket WaitTicket) uint32 { + before := uint32(ticket) + 1 + Park(token, ticket) + return before + uint32(ticket) +} +`, nil) + prog := llssa.NewProgram(nil) + defer prog.Dispose() + emission, err := cl.PrepareEmissionUniverse(prog, nil, []cl.EmissionPackage{{ + SSA: ssaPkg, Files: files, Identity: "example.com/coropark", + }}) + if err != nil { + t.Fatal(err) + } + ssaEmission, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, emission.Functions()) + if err != nil { + t.Fatal(err) + } + root := ssaPkg.Func("root") + calls := coroPlanTestCalls(root) + if len(calls) != 1 { + t.Fatalf("root calls = %d, want one exact park site", len(calls)) + } + parkCall := calls[0] + semantics, intrinsic, err := emission.CoroIntrinsicCallSiteSemantics(parkCall) + if err != nil || !intrinsic || semantics != cl.CoroIntrinsicCallInlineSuspend || !semantics.SuspendsCurrentFrame() { + t.Fatalf("park semantics = %v, %v, %v; want inline-suspend, true, nil", semantics, intrinsic, err) + } + input := CoroPlanInput{ + Program: ssaPkg.Prog, + EmissionUniverse: ssaEmission, + resolveFunction: emission.Resolve, + functionBackground: emission.FunctionBackground, + intrinsicCallSemantics: emission.CoroIntrinsicCallSiteSemantics, + } + functionIDs := emission.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerChildAwaitABIV0 + functionIDs.ArchiveReady = true + analyze := func() (*coro.SSAPlan, error) { + return input.Analyze(coro.Roots{{Function: root, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + MaxPlainInstructions: -1, + FunctionIDs: functionIDs, + }) + } + plan, err := analyze() + if err != nil { + t.Fatal(err) + } + rootPlan, ok := plan.FunctionPlan(root) + if !ok || rootPlan.Emission != coro.EmitCoroutine || rootPlan.Primary != coro.PrimaryCoroutine || + rootPlan.FuncRep != coro.DirectCoro || !rootPlan.DeclaredEffect.Contains(coro.MayPark) || + !rootPlan.LocalEffect.Contains(coro.MayPark) || !rootPlan.Effect.Contains(coro.MayPark) { + t.Fatalf("park root plan = %+v, present=%t; want one tainted coroutine primary", rootPlan, ok) + } + if !plan.ElidesCall(parkCall) { + t.Fatal("park declaration call is not retained as an exact elided site") + } + if _, ok := plan.CallPlan(parkCall); ok { + t.Fatal("park declaration unexpectedly retained a managed CallPlan") + } + metadata := coro.PlanDigestMetadata{ + CoroABI: coro.PhysicalABIV1, SchedulerABI: coro.SchedulerChildAwaitABIV0, + PanicABI: coro.PanicLegacyABIV0, FuncRepABI: coro.FuncRepABIV0, + TargetTriple: "x86_64-unknown-linux-gnu", PointerBits: 64, + Endianness: "little", DataLayout: "e-p:64:64", + } + digest, err := plan.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + again, err := analyze() + if err != nil { + t.Fatal(err) + } + againDigest, err := again.CoroPlanDigest(metadata) + if err != nil || againDigest != digest || !again.ElidesCall(parkCall) { + t.Fatalf("park plan digest = %q, %v (elided=%t); want stable %q", againDigest, err, again.ElidesCall(parkCall), digest) + } +} + func TestRequiredCoroProgramRuntimePlanPlainClosureAndConflicts(t *testing.T) { ssaPkg, files := buildCoroPlanTestPackage(t, llssa.PkgRuntime, `package runtime func __llgo_coro_program_begin_v1() { bootstrapHelper() } func __llgo_coro_program_run_v1() {} +func __llgo_coro_frame_allocator_bootstrap_v1() {} func __llgo_coro_frame_alloc_v1() {} func __llgo_coro_frame_publish_v1() {} func __llgo_coro_await_prepare_v1() {} +var preemptRequest uint32 +func __llgo_coro_preempt_poll_v1() bool { return atomicExchange(&preemptRequest, 0) == 1 } +func __llgo_coro_yield_prepare_v1() {} +func __llgo_coro_park_prepare_v1() {} func __llgo_coro_complete_prepare_v1() {} func __llgo_coro_frame_free_v1() {} func bootstrapHelper() { closureLoop(); externalABI(); inlineIntrinsic("bootstrap") } @@ -288,6 +424,8 @@ func unrelatedLoop() { for {} } func externalABI() //llgo:link inlineIntrinsic llgo.cstr func inlineIntrinsic(string) *byte +//llgo:link atomicExchange llgo.atomicXchg +func atomicExchange(*uint32, uint32) uint32 `, nil) prog := llssa.NewProgram(nil) defer prog.Dispose() @@ -302,7 +440,7 @@ func inlineIntrinsic(string) *byte t.Fatal(err) } ctx := &context{ - buildConf: &Config{EnableCoroProgramBootstrapRun: true}, + buildConf: &Config{EnableCoroChildAwait: true, EnableCoroProgramBootstrapRun: true}, coroEmission: emission, coroSSAEmission: ssaEmission, } @@ -323,11 +461,15 @@ func inlineIntrinsic(string) *byte } wantRoots := []string{ "init", + coroFrameAllocatorBootstrapSymbolV1, coroProgramBeginSymbolV1, coroProgramRunSymbolV1, "__llgo_coro_frame_alloc_v1", "__llgo_coro_frame_publish_v1", "__llgo_coro_await_prepare_v1", + "__llgo_coro_preempt_poll_v1", + "__llgo_coro_yield_prepare_v1", + "__llgo_coro_park_prepare_v1", "__llgo_coro_complete_prepare_v1", "__llgo_coro_frame_free_v1", } @@ -335,14 +477,22 @@ func inlineIntrinsic(string) *byte t.Fatalf("required runtime roots = %d, want %d", len(roots), len(wantRoots)) } for index, root := range roots { - if root.Function == nil || root.Function.Name() != wantRoots[index] || root.Demand != coro.SyncDemand { - t.Fatalf("required root %d = %+v, want %s/sync", index, root, wantRoots[index]) + wantDemand := coro.SyncDemand + if index == 0 { + wantDemand = coro.AsyncDemand + } + if root.Function == nil || root.Function.Name() != wantRoots[index] || root.Demand != wantDemand { + t.Fatalf("required root %d = %+v, want %s/%s", index, root, wantRoots[index], wantDemand) } } + if _, ok := requiredPlain[ssaPkg.Func("init")]; ok { + t.Fatal("managed runtime.init leaked into the native required-plain island") + } closureLoop := ssaPkg.Func("closureLoop") unrelatedLoop := ssaPkg.Func("unrelatedLoop") externalABI := ssaPkg.Func("externalABI") inlineIntrinsic := ssaPkg.Func("inlineIntrinsic") + atomicExchange := ssaPkg.Func("atomicExchange") for _, fn := range []*ssa.Function{ssaPkg.Func("bootstrapHelper"), closureLoop, externalABI} { if _, ok := requiredPlain[fn]; !ok { t.Fatalf("required plain closure omitted %s", fn.Name()) @@ -351,12 +501,17 @@ func inlineIntrinsic(string) *byte if _, ok := requiredPlain[unrelatedLoop]; ok { t.Fatal("required plain closure captured an unrelated function") } - if _, ok := requiredPlain[inlineIntrinsic]; ok { - t.Fatal("compiler-inline no-suspend intrinsic entered the runtime plain-function island") + for _, intrinsic := range []*ssa.Function{inlineIntrinsic, atomicExchange} { + if _, ok := requiredPlain[intrinsic]; ok { + t.Fatalf("compiler-inline no-suspend intrinsic %q entered the runtime plain-function island", intrinsic.Name()) + } } if semantics, intrinsic, err := emission.CoroIntrinsicSemantics(inlineIntrinsic); err != nil || !intrinsic || semantics != cl.CoroIntrinsicCallInlineNoSuspend { t.Fatalf("inline intrinsic semantics = %v, %v, %v; want inline-no-suspend, true, nil", semantics, intrinsic, err) } + if semantics, intrinsic, err := emission.CoroIntrinsicSemantics(atomicExchange); err != nil || !intrinsic || semantics != cl.CoroIntrinsicCallInlineNoSuspend { + t.Fatalf("atomic exchange semantics = %v, %v, %v; want inline-no-suspend, true, nil", semantics, intrinsic, err) + } input := CoroPlanInput{ Program: ssaPkg.Prog, @@ -371,7 +526,7 @@ func inlineIntrinsic(string) *byte } functionIDs := emission.FunctionIDConfig() functionIDs.CoroABI = coro.PhysicalABIV1 - functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapABIV1 + functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapABIV2 functionIDs.ArchiveReady = true analyze := func(classify func(*ssa.Function) (coro.SSAFunctionPolicy, error)) (*coro.SSAPlan, error) { return input.Analyze(coro.Roots{{Function: unrelatedLoop, Demand: coro.AsyncDemand}}, coro.SSAConfig{ @@ -388,6 +543,16 @@ func inlineIntrinsic(string) *byte if !ok || closurePlan.Exec.Contains(coro.NeedsPreempt) || closurePlan.Effect.MaySuspend() || closurePlan.Emission != coro.EmitPlain { t.Fatalf("required closure loop plan = %+v, want one trusted plain body", closurePlan) } + pollPlan, ok := plan.FunctionPlan(ssaPkg.Func("__llgo_coro_preempt_poll_v1")) + if !ok || pollPlan.Effect.MaySuspend() || pollPlan.Exec.Contains(coro.NeedsPreempt) || pollPlan.Emission != coro.EmitPlain { + t.Fatalf("preempt poll plan = %+v, want one trusted plain atomic poll", pollPlan) + } + parkHookPlan, ok := plan.FunctionPlan(ssaPkg.Func("__llgo_coro_park_prepare_v1")) + if !ok || parkHookPlan.Effect.MaySuspend() || parkHookPlan.Exec.Contains(coro.NeedsPreempt) || + parkHookPlan.Emission != coro.EmitPlain || parkHookPlan.Demand != coro.SyncDemand || + parkHookPlan.FuncRep != coro.DirectPlain { + t.Fatalf("park prepare hook plan = %+v, want one required sync direct-plain body", parkHookPlan) + } unrelatedPlan, ok := plan.FunctionPlan(unrelatedLoop) if !ok || !unrelatedPlan.Exec.Contains(coro.NeedsPreempt) || !unrelatedPlan.Effect.Contains(coro.YieldOnly) || unrelatedPlan.Emission != coro.EmitCoroutine { t.Fatalf("unrelated loop plan = %+v, want coroutine preemption", unrelatedPlan) @@ -411,7 +576,7 @@ func inlineIntrinsic(string) *byte } metadata := coro.PlanDigestMetadata{ - CoroABI: coro.PhysicalABIV1, SchedulerABI: coro.SchedulerProgramBootstrapABIV1, + CoroABI: coro.PhysicalABIV1, SchedulerABI: coro.SchedulerProgramBootstrapABIV2, PanicABI: coro.PanicLegacyABIV0, FuncRepABI: coro.FuncRepABIV0, TargetTriple: "x86_64-unknown-linux-gnu", PointerBits: 64, Endianness: "little", DataLayout: "e-p:64:64", @@ -432,6 +597,21 @@ func inlineIntrinsic(string) *byte t.Fatalf("required runtime plan digest changed: %s != %s", secondDigest, digest) } + irqPlan, err := analyze(func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == closureLoop { + return coro.SSAFunctionPolicy{Exec: coro.IRQUnsafe}, nil + } + return coro.SSAFunctionPolicy{}, nil + }) + if err != nil { + t.Fatalf("required plain ordinary-G IRQ-unsafe plan: %v", err) + } + irqClosure, ok := irqPlan.FunctionPlan(closureLoop) + if !ok || irqClosure.Emission != coro.EmitPlain || !irqClosure.Exec.Contains(coro.IRQUnsafe) || + irqClosure.Exec.Contains(coro.ThreadAffine|coro.BlockForeign|coro.OpaqueExec) { + t.Fatalf("required plain IRQ-unsafe closure plan = %+v, want exact ordinary-G plain implementation", irqClosure) + } + conflicts := []struct { name string target *ssa.Function @@ -459,13 +639,80 @@ func inlineIntrinsic(string) *byte } } +func TestRequiredCoroProgramRuntimePlanKeepsEntryInitWithoutRunnableBootstrap(t *testing.T) { + ssaPkg, files := buildCoroPlanTestPackage(t, llssa.PkgRuntime, `package runtime +func __llgo_coro_program_begin_v1() {} +func __llgo_coro_program_run_v1() {} +func __llgo_coro_frame_allocator_bootstrap_v1() {} +func __llgo_coro_frame_alloc_v1() {} +func __llgo_coro_frame_publish_v1() {} +func __llgo_coro_await_prepare_v1() {} +func __llgo_coro_preempt_poll_v1() bool { return false } +func __llgo_coro_yield_prepare_v1() {} +func __llgo_coro_park_prepare_v1() {} +func __llgo_coro_complete_prepare_v1() {} +func __llgo_coro_frame_free_v1() {} +`, nil) + prog := llssa.NewProgram(nil) + defer prog.Dispose() + emission, err := cl.PrepareEmissionUniverse(prog, nil, []cl.EmissionPackage{{ + SSA: ssaPkg, Files: files, Identity: llssa.PkgRuntime, + }}) + if err != nil { + t.Fatal(err) + } + ssaEmission, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, emission.Functions()) + if err != nil { + t.Fatal(err) + } + ctx := &context{ + buildConf: &Config{EnableCoroChildAwait: true}, + coroEmission: emission, + coroSSAEmission: ssaEmission, + } + roots, requiredPlain, directPlain, closedDynamic, err := requiredCoroProgramRuntimePlan(ctx) + if err != nil { + t.Fatal(err) + } + if len(roots) != 1 || roots[0].Function != ssaPkg.Func("init") || roots[0].Demand != coro.SyncDemand { + t.Fatalf("entry-only runtime roots = %+v, want exact runtime package init/sync", roots) + } + if _, ok := requiredPlain[ssaPkg.Func("init")]; !ok { + t.Fatal("entry-only runtime init is absent from required plain closure") + } + for _, name := range []string{ + coroFrameAllocatorBootstrapSymbolV1, + coroProgramBeginSymbolV1, + coroProgramRunSymbolV1, + "__llgo_coro_frame_alloc_v1", + "__llgo_coro_frame_publish_v1", + "__llgo_coro_await_prepare_v1", + "__llgo_coro_preempt_poll_v1", + "__llgo_coro_yield_prepare_v1", + "__llgo_coro_park_prepare_v1", + "__llgo_coro_complete_prepare_v1", + "__llgo_coro_frame_free_v1", + } { + if _, ok := requiredPlain[ssaPkg.Func(name)]; ok { + t.Fatalf("descriptor-only child-await plan trusted runnable hook %q", name) + } + } + if len(directPlain) != 0 || len(closedDynamic) != 0 { + t.Fatalf("entry-only runtime plan produced callback proofs: direct=%d dynamic=%d", len(directPlain), len(closedDynamic)) + } +} + func TestRequiredCoroProgramRuntimePlanRejectsInvalidIntrinsicSite(t *testing.T) { ssaPkg, files := buildCoroPlanTestPackage(t, llssa.PkgRuntime, `package runtime func __llgo_coro_program_begin_v1() { bootstrapHelper() } func __llgo_coro_program_run_v1() {} +func __llgo_coro_frame_allocator_bootstrap_v1() {} func __llgo_coro_frame_alloc_v1() {} func __llgo_coro_frame_publish_v1() {} func __llgo_coro_await_prepare_v1() {} +func __llgo_coro_preempt_poll_v1() bool { return false } +func __llgo_coro_yield_prepare_v1() {} +func __llgo_coro_park_prepare_v1() {} func __llgo_coro_complete_prepare_v1() {} func __llgo_coro_frame_free_v1() {} func intrinsicInput() string { return "not constant at the call site" } @@ -486,7 +733,7 @@ func inlineIntrinsic(string) *byte t.Fatal(err) } ctx := &context{ - buildConf: &Config{EnableCoroProgramBootstrapRun: true}, + buildConf: &Config{EnableCoroChildAwait: true, EnableCoroProgramBootstrapRun: true}, coroEmission: emission, coroSSAEmission: ssaEmission, } @@ -829,9 +1076,13 @@ func buildRequiredCoroRuntimeFixture(t *testing.T, body string) requiredCoroRunt source := `package runtime func __llgo_coro_program_begin_v1() { install() } func __llgo_coro_program_run_v1() {} +func __llgo_coro_frame_allocator_bootstrap_v1() {} func __llgo_coro_frame_alloc_v1() {} func __llgo_coro_frame_publish_v1() {} func __llgo_coro_await_prepare_v1() {} +func __llgo_coro_preempt_poll_v1() bool { return false } +func __llgo_coro_yield_prepare_v1() {} +func __llgo_coro_park_prepare_v1() {} func __llgo_coro_complete_prepare_v1() {} func __llgo_coro_frame_free_v1() {} ` + body @@ -851,7 +1102,7 @@ func __llgo_coro_frame_free_v1() {} } ctx := &context{ prog: prog, - buildConf: &Config{EnableCoroProgramBootstrapRun: true}, + buildConf: &Config{EnableCoroChildAwait: true, EnableCoroProgramBootstrapRun: true}, coroEmission: emission, coroSSAEmission: ssaEmission, coroTLSDestructorFixturePkg: llssa.PkgRuntime, @@ -862,7 +1113,7 @@ func __llgo_coro_frame_free_v1() {} } functionIDs := emission.FunctionIDConfig() functionIDs.CoroABI = coro.PhysicalABIV1 - functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapABIV1 + functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapABIV2 functionIDs.ArchiveReady = true return requiredCoroRuntimeFixture{ pkg: ssaPkg, @@ -1358,6 +1609,7 @@ func alias() {} {name: "extra", requested: []coro.SSALoweredCall{{LogicalName: "runtime.helper", Target: helper}, {LogicalName: "runtime.helper2", Target: helper2}, {LogicalName: "runtime.extra", Target: extra}}}, {name: "renamed", requested: []coro.SSALoweredCall{{LogicalName: "runtime.renamed", Target: helper}, {LogicalName: "runtime.helper2", Target: helper2}}}, {name: "retargeted", requested: []coro.SSALoweredCall{{LogicalName: "runtime.helper", Target: helper2}, {LogicalName: "runtime.helper2", Target: helper}}}, + {name: "unwind class", requested: []coro.SSALoweredCall{{LogicalName: "runtime.helper", Target: helper, UnwindOnly: true}, {LogicalName: "runtime.helper2", Target: helper2}}}, {name: "alias", requested: []coro.SSALoweredCall{{LogicalName: "runtime.helper", Target: alias}, {LogicalName: "runtime.helper2", Target: helper2}}}, {name: "duplicate", requested: []coro.SSALoweredCall{{LogicalName: "runtime.helper", Target: helper}, {LogicalName: "runtime.helper", Target: helper}}}, } @@ -1397,6 +1649,104 @@ func alias() {} } } +func TestValidateCoroUnwindOnlyLoweredCallsRequiresLegacyPlainTarget(t *testing.T) { + ssaPkg, _ := buildCoroPlanTestPackage(t, "example.com/unwindlowered", `package unwindlowered +var channel chan int +func owner() {} +func plain() {} +func suspending() { <-channel } +func external() +`, nil) + owner := ssaPkg.Func("owner") + plain := ssaPkg.Func("plain") + suspending := ssaPkg.Func("suspending") + external := ssaPkg.Func("external") + universe, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, []*ssa.Function{owner, plain, suspending, external}) + if err != nil { + t.Fatal(err) + } + build := func(target *ssa.Function) *coro.SSAPlan { + t.Helper() + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: owner, Demand: coro.SyncDemand}}, coro.SSAConfig{ + EmissionUniverse: universe, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == suspending { + // These flags describe a control-flow role; they are not a + // certificate that a physically suspending body is plain. + return coro.SSAFunctionPolicy{Exec: coro.NoReturn | coro.PanicOnly}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + ClassifyLoweredCalls: func(fn *ssa.Function) ([]coro.SSALoweredCall, error) { + if fn == owner { + return []coro.SSALoweredCall{{LogicalName: "runtime.Helper", Target: target, UnwindOnly: true}}, nil + } + return nil, nil + }, + MaxPlainInstructions: -1, + }) + if err != nil { + t.Fatal(err) + } + return plan + } + plainPlan := build(plain) + if err := validateCoroUnwindOnlyLoweredCalls(plainPlan, coro.PanicLegacyABIV0); err != nil { + t.Fatalf("bounded plain unwind helper rejected: %v", err) + } + forged := coroLegacyPanicPlainCertificate{owner: owner, logicalName: "runtime.Helper", target: suspending} + if err := forged.validate(plainPlan); err == nil || !strings.Contains(err.Error(), "not bound to an exact frozen unwind-only target") { + t.Fatalf("name-only retargeted certificate error = %v", err) + } + suspendingPlan := build(suspending) + if got, ok := suspendingPlan.FunctionPlan(owner); !ok || got.Effect != coro.NoSuspend || got.Emission != coro.EmitPlain { + t.Fatalf("unwind-only edge polluted owner before preflight: %+v, present=%v", got, ok) + } + err = validateCoroUnwindOnlyLoweredCalls(suspendingPlan, coro.PanicLegacyABIV0) + if err == nil || !strings.Contains(err.Error(), "exact "+coro.PanicLegacyABIV0+" plain certificate") || + !strings.Contains(err.Error(), "effect=may-park") || !strings.Contains(err.Error(), "panic-only") { + t.Fatalf("suspending unwind helper error = %v", err) + } + if err := validateCoroUnwindOnlyLoweredCalls(build(external), coro.PanicLegacyABIV0); err == nil || + !strings.Contains(err.Error(), "is not a defined Go body") { + t.Fatalf("external unwind helper error = %v", err) + } +} + +func TestValidateCoroUnwindOnlyLoweredCallsRejectsDynamicErrorMethod(t *testing.T) { + ssaPkg, _ := buildCoroPlanTestPackage(t, "example.com/unwinderror", `package unwinderror +func owner() {} +func failure(err error) { _ = err.Error() } +`, nil) + owner := ssaPkg.Func("owner") + failure := ssaPkg.Func("failure") + universe, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, []*ssa.Function{owner, failure}) + if err != nil { + t.Fatal(err) + } + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: owner, Demand: coro.SyncDemand}}, coro.SSAConfig{ + EmissionUniverse: universe, + ClassifyLoweredCalls: func(fn *ssa.Function) ([]coro.SSALoweredCall, error) { + if fn == owner { + return []coro.SSALoweredCall{{LogicalName: "runtime.Panic", Target: failure, UnwindOnly: true}}, nil + } + return nil, nil + }, + MaxPlainInstructions: -1, + }) + if err != nil { + t.Fatal(err) + } + err = validateCoroUnwindOnlyLoweredCalls(plan, coro.PanicLegacyABIV0) + if err == nil || !strings.Contains(err.Error(), "dynamic invoke Error") || + !strings.Contains(err.Error(), "not a bounded DirectPlain edge") { + t.Fatalf("dynamic error method unwind helper error = %v", err) + } + if got, ok := plan.FunctionPlan(failure); !ok || got.FuncRep != coro.DirectCoro || !got.Exec.Contains(coro.OpaqueExec) { + t.Fatalf("dynamic Error target was unexpectedly forced plain: %+v, present=%v", got, ok) + } +} + func TestActiveCoroABIVersions(t *testing.T) { tests := []struct { name string @@ -1409,7 +1759,7 @@ func TestActiveCoroABIVersions(t *testing.T) { {"physical leaf", &Config{EnableCoroPhysicalABI: true}, coro.PhysicalABIV0, coro.SchedulerNoneABIV0, coro.FuncRepABIV0}, {"plain dispatch", &Config{EnableCoroPlainDispatch: true}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0, coro.FuncRepABIV1}, {"child await", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true}, coro.PhysicalABIV1, coro.SchedulerChildAwaitABIV0, coro.FuncRepABIV0}, - {"program bootstrap runtime with plain dispatch", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroPlainDispatch: true, EnableCoroProgramBootstrapRun: true}, coro.PhysicalABIV1, coro.SchedulerProgramBootstrapABIV1, coro.FuncRepABIV1}, + {"program bootstrap runtime with plain dispatch", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroPlainDispatch: true, EnableCoroProgramBootstrapRun: true}, coro.PhysicalABIV1, coro.SchedulerProgramBootstrapABIV2, coro.FuncRepABIV1}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/internal/build/coro_registry.go b/internal/build/coro_registry.go index eaf0563e9e..2855858470 100644 --- a/internal/build/coro_registry.go +++ b/internal/build/coro_registry.go @@ -91,8 +91,18 @@ func coroProgramManifestHashV1(ctx *context, anchors []string, bootstrap ...*cor return [16]byte{}, fmt.Errorf("coroutine program manifest accepts at most one bootstrap table") } if len(bootstrap) == 1 && bootstrap[0] != nil { - write("llgo.coro.program-bootstrap.v1") - write(hex.EncodeToString(bootstrap[0].StepHash[:])) + program := bootstrap[0] + write(fmt.Sprintf("llgo.coro.program-bootstrap.v%d", program.abiVersion())) + write(hex.EncodeToString(program.StepHash[:])) + for _, step := range program.Steps { + write(fmt.Sprintf("%d", step.Kind)) + write(fmt.Sprintf("%d", step.Role)) + write(string(step.FunctionID)) + write(step.Target) + write(step.Owner) + write(step.CatalogTarget) + write(fmt.Sprintf("%d", step.Aux)) + } } sum := h.Sum(nil) var hash [16]byte diff --git a/internal/build/coro_runtime_abi_gate_test.go b/internal/build/coro_runtime_abi_gate_test.go new file mode 100644 index 0000000000..2fb88a1003 --- /dev/null +++ b/internal/build/coro_runtime_abi_gate_test.go @@ -0,0 +1,56 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "testing" + + "github.com/goplus/llgo/cl" + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/packages" +) + +func TestPrepareCoroEmissionUniverseEnablesCompleteRuntimeABI(t *testing.T) { + ssaPkg, files := buildCoroPlanTestPackage(t, llssa.PkgRuntime, `package runtime +func Present() {} +`, nil) + prog := llssa.NewProgram(nil) + t.Cleanup(prog.Dispose) + cl.ParsePkgSyntax(prog, ssaPkg.Pkg, files) + pkg := &aPackage{ + Package: &packages.Package{ + ID: llssa.PkgRuntime, + PkgPath: llssa.PkgRuntime, + Types: ssaPkg.Pkg, + Syntax: files, + }, + SSA: ssaPkg, + } + ctx := &context{ + prog: prog, + progSSA: ssaPkg.Prog, + buildConf: &Config{EnableCoroEntryResolution: true}, + } + if err := prepareCoroEmissionUniverse(ctx, []*aPackage{pkg}); err != nil { + t.Fatal(err) + } + if ctx.coroEmission == nil || !ctx.coroEmission.CompleteRuntimeABI() { + t.Fatal("active internal/build runtime input did not enable the complete runtime ABI contract") + } +} diff --git a/internal/build/coro_tls_destructor_test.go b/internal/build/coro_tls_destructor_test.go index df904bca84..3dbf388113 100644 --- a/internal/build/coro_tls_destructor_test.go +++ b/internal/build/coro_tls_destructor_test.go @@ -453,9 +453,13 @@ func buildCoroTLSRuntimePlanError(t *testing.T, body string) error { source += ` func __llgo_coro_program_begin_v1() { install() } func __llgo_coro_program_run_v1() {} +func __llgo_coro_frame_allocator_bootstrap_v1() {} func __llgo_coro_frame_alloc_v1() {} func __llgo_coro_frame_publish_v1() {} func __llgo_coro_await_prepare_v1() {} +func __llgo_coro_preempt_poll_v1() bool { return false } +func __llgo_coro_yield_prepare_v1() {} +func __llgo_coro_park_prepare_v1() {} func __llgo_coro_complete_prepare_v1() {} func __llgo_coro_frame_free_v1() {} ` + body @@ -475,7 +479,7 @@ func __llgo_coro_frame_free_v1() {} } ctx := &context{ prog: prog, - buildConf: &Config{EnableCoroProgramBootstrapRun: true}, + buildConf: &Config{EnableCoroChildAwait: true, EnableCoroProgramBootstrapRun: true}, coroEmission: emission, coroSSAEmission: ssaEmission, coroTLSDestructorFixturePkg: llssa.PkgRuntime, diff --git a/internal/build/fingerprint.go b/internal/build/fingerprint.go index 96a6688a74..9073d05cee 100644 --- a/internal/build/fingerprint.go +++ b/internal/build/fingerprint.go @@ -116,6 +116,7 @@ type commonSection struct { AbiMode string `yaml:"ABI_MODE,omitempty"` BuildTags []string `yaml:"BUILD_TAGS,omitempty"` Target string `yaml:"TARGET,omitempty"` + RuntimeGC string `yaml:"RUNTIME_GC,omitempty"` LLVMCPU string `yaml:"LLVM_CPU,omitempty"` LLVMFeatures string `yaml:"LLVM_FEATURES,omitempty"` TargetABI string `yaml:"TARGET_ABI,omitempty"` @@ -141,7 +142,7 @@ type commonSection struct { } func (s *commonSection) empty() bool { - return s.AbiMode == "" && len(s.BuildTags) == 0 && s.Target == "" && s.LLVMCPU == "" && + return s.AbiMode == "" && len(s.BuildTags) == 0 && s.Target == "" && s.RuntimeGC == "" && s.LLVMCPU == "" && s.LLVMFeatures == "" && s.TargetABI == "" && s.CoroPlanDigest == "" && s.CoroABI == "" && s.CoroSchedulerABI == "" && s.CoroPanicABI == "" && s.CoroFuncRepABI == "" && s.CoroTargetTriple == "" && s.CoroTargetCPU == "" && diff --git a/internal/build/gc_target_test.go b/internal/build/gc_target_test.go new file mode 100644 index 0000000000..7b8be117ef --- /dev/null +++ b/internal/build/gc_target_test.go @@ -0,0 +1,54 @@ +//go:build !llgo + +package build + +import ( + "slices" + "testing" + + "github.com/goplus/llgo/internal/crosscompile" +) + +func TestTargetGCBuildTags(t *testing.T) { + tests := []struct { + gc string + wantTag bool + wantErr bool + }{ + {gc: ""}, + {gc: "precise"}, + {gc: "conservative"}, + {gc: "leaking", wantTag: true}, + {gc: "none", wantTag: true}, + {gc: "invented", wantErr: true}, + } + for _, test := range tests { + t.Run(test.gc, func(t *testing.T) { + tags, err := targetGCBuildTags(test.gc) + if (err != nil) != test.wantErr { + t.Fatalf("targetGCBuildTags(%q) error = %v, wantErr %v", test.gc, err, test.wantErr) + } + if !test.wantErr && slices.Contains(tags, "nogc") != test.wantTag { + t.Fatalf("targetGCBuildTags(%q) = %v, want nogc=%v", test.gc, tags, test.wantTag) + } + }) + } +} + +func TestTargetGCProfileAffectsFingerprint(t *testing.T) { + fingerprint := func(gc string) string { + ctx := &context{ + buildConf: &Config{Goos: "linux", Goarch: "arm", Target: "wasip2"}, + crossCompile: crosscompile.Export{GC: gc}, + } + manifest := newManifestBuilder() + ctx.collectCommonInputs(manifest) + if got := manifest.common.RuntimeGC; got != gc { + t.Fatalf("manifest runtime GC = %q, want %q", got, gc) + } + return manifest.Fingerprint() + } + if leaking, precise := fingerprint("leaking"), fingerprint("precise"); leaking == precise { + t.Fatal("runtime GC capability did not affect package fingerprint") + } +} diff --git a/internal/build/main_module.go b/internal/build/main_module.go index 98fbc768e9..e5c30d51d1 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -88,9 +88,21 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g return mainAPkg } - runtimeStub := defineWeakNoArgStub(mainPkg, "runtime.init") - // TODO(lijie): workaround for syscall patch - defineWeakNoArgStub(mainPkg, "syscall.init") + managedBootstrapV2 := ctx.buildConf.EnableCoroProgramBootstrapRun && cfg.coroBootstrap != nil && + cfg.coroBootstrap.abiVersion() == coroProgramBootstrapVersionV2 + var runtimeStub llssa.Function + if !managedBootstrapV2 { + // Legacy entry modes retain the historical optional public-runtime hook. + // V2 resolves the exact public runtime SSA init through its managed table; + // defining a weak symbol here would satisfy the archive relocation with a + // no-op and could silently prevent extraction of the real strong body. + runtimeStub = defineWeakNoArgStub(mainPkg, "runtime.init") + // TODO(lijie): legacy workaround for syscall patch. It is deliberately + // absent from V2: a weak entry-module definition could also intercept a + // real syscall.init relocation reached through the managed package-init + // chain and violate the single-primary plan. + defineWeakNoArgStub(mainPkg, "syscall.init") + } var pyInit llssa.Function var pyFinalize llssa.Function @@ -100,7 +112,7 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g } var rtInit llssa.Function - if cfg.rtInit { + if cfg.rtInit && !managedBootstrapV2 { rtInit = declareNoArgFunc(mainPkg, rtPkgPath+".init") } @@ -113,31 +125,54 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g return filterAbiSymbol(cfg.abiInit, sym) }) } + if ctx.buildConf.EnableCoroProgramBootstrapRun && cfg.coroBootstrap != nil && + cfg.coroBootstrap.abiVersion() == coroProgramBootstrapVersionV2 { + // The v2 table always contains the compiler ABI-init stage. Profiles with + // no selected ABI symbols still define the exact target as a bounded no-op + // so the five-stage program never relies on an optional external symbol. + if abiInit == nil { + abiInit = mainPkg.FuncOf("init$abitypes") + if abiInit == nil { + abiInit = declareNoArgFunc(mainPkg, "init$abitypes") + } + if !abiInit.HasBody() { + body := abiInit.MakeBody(1) + body.Return() + } + } + } - mainInit := declareNoArgFunc(mainPkg, pkg.PkgPath+".init") - mainMain := declareNoArgFunc(mainPkg, pkg.PkgPath+".main") + var mainInit, mainMain llssa.Function + if !ctx.buildConf.EnableCoroProgramBootstrapRun { + mainInit = declareNoArgFunc(mainPkg, pkg.PkgPath+".init") + mainMain = declareNoArgFunc(mainPkg, pkg.PkgPath+".main") + } var coroBegin llssa.Function var coroRun llssa.Function + var coroAllocatorBootstrap llssa.Function if ctx.buildConf.EnableCoroProgramBootstrapRun { if coroEntry.manifest.IsNil() || coroEntry.factory == nil { panic("coroutine program bootstrap runtime enabled without a manifest and factory") } + coroAllocatorBootstrap = declareNoArgFunc(mainPkg, coroFrameAllocatorBootstrapSymbolV1) coroBegin = declareCoroProgramBeginV1(mainPkg) coroRun = declareCoroProgramRunV1(mainPkg) } entryFn := defineEntryFunction(ctx, mainPkg, argcVar, argvVar, argvValueType, entryFunctions{ - runtimeStub: runtimeStub, - mainInit: mainInit, - mainMain: mainMain, - pyInit: pyInit, - pyFinalize: pyFinalize, - rtInit: rtInit, - abiInit: abiInit, - coroManifest: coroEntry.manifest, - coroFactory: coroEntry.factory, - coroBegin: coroBegin, - coroRun: coroRun, + runtimeStub: runtimeStub, + mainInit: mainInit, + mainMain: mainMain, + pyInit: pyInit, + pyFinalize: pyFinalize, + rtInit: rtInit, + abiInit: abiInit, + coroManifest: coroEntry.manifest, + coroFactory: coroEntry.factory, + coroAllocatorBootstrap: coroAllocatorBootstrap, + coroBegin: coroBegin, + coroRun: coroRun, + coroBootstrapVersion: cfg.coroBootstrap.abiVersion(), }) if needStart(ctx) { @@ -178,8 +213,10 @@ func emitCoroControlWrappers(ctx *context, pkg llssa.Package) { } const ( - coroProgramManifestSymbolV1 = "__llgo_coro_program_manifest_v1" - coroProgramBootstrapSymbolV1 = "__llgo_coro_program_bootstrap_v1" + coroProgramManifestSymbolV1 = "__llgo_coro_program_manifest_v1" + coroProgramBootstrapSymbolV1 = "__llgo_coro_program_bootstrap_v1" + coroProgramBootstrapSymbolV2 = "__llgo_coro_program_bootstrap_v2" + coroFrameAllocatorBootstrapSymbolV1 = "__llgo_coro_frame_allocator_bootstrap_v1" ) type coroProgramEntryV1 struct { @@ -201,12 +238,14 @@ func emitCoroProgramManifest(ctx *context, pkg llssa.Package, cfg *genConfig) co prog.VoidPtr(), ) anchors := make([]llssa.Expr, len(cfg.coroRootAnchors)) + anchorByName := make(map[string]llssa.Expr, len(cfg.coroRootAnchors)) for i, name := range cfg.coroRootAnchors { anchor := pkg.NewVarEx(name, prog.Pointer(anchorType)) global := pkg.Module().NamedGlobal(name) global.SetLinkage(llvm.ExternalLinkage) global.SetVisibility(llvm.HiddenVisibility) anchors[i] = anchor.Expr + anchorByName[name] = anchor.Expr } var bootstrap llssa.Expr var factory llssa.Function @@ -215,34 +254,72 @@ func emitCoroProgramManifest(ctx *context, pkg llssa.Package, cfg *genConfig) co panic("coroutine program bootstrap ABI enabled without a validated startup table") } steps := make([]llssa.CoroProgramStep, len(cfg.coroBootstrap.Steps)) - targets := make([]llssa.Function, len(cfg.coroBootstrap.Steps)) - for i, step := range cfg.coroBootstrap.Steps { - target := declareNoArgFunc(pkg, step.Target) - targets[i] = target - steps[i] = llssa.CoroProgramStep{ - Kind: llssa.CoroProgramStepKind(step.Kind), - Flags: step.Role, - Target: target.Expr, - Aux: uint64(step.Aux), + version := cfg.coroBootstrap.abiVersion() + if version == coroProgramBootstrapVersionV2 { + targets := make([]coroProgramBootstrapFactoryTargetV2, len(cfg.coroBootstrap.Steps)) + for i, step := range cfg.coroBootstrap.Steps { + var tableTarget llssa.Expr + switch step.Kind { + case coroProgramStepDirectPlainV1: + plain := declareNoArgFunc(pkg, step.Target) + if step.FunctionID == coroProgramPublicRuntimeNoopIDV2 { + if step.Role != coroProgramStepRolePublicRuntimeInitV2 || step.Target != coroProgramPublicRuntimeNoopSymbolV2 { + panic("coroutine program bootstrap v2 public-runtime no-op has noncanonical identity") + } + if !plain.HasBody() { + body := plain.MakeBody(1) + body.Return() + } + } + targets[i].Plain = plain + tableTarget = plain.Expr + case coroProgramStepCoroRootV1: + anchor := anchorByName[step.CatalogTarget] + if anchor.IsNil() { + panic(fmt.Sprintf("coroutine program bootstrap v2 step %d has unlinked catalog anchor %q", i, step.CatalogTarget)) + } + targets[i].Anchor = anchor + tableTarget = anchor + default: + panic(fmt.Sprintf("coroutine program bootstrap v2 step %d has invalid kind %d", i, step.Kind)) + } + steps[i] = llssa.CoroProgramStep{ + Kind: llssa.CoroProgramStepKind(step.Kind), Flags: step.Role, + Target: tableTarget, Aux: step.Aux, + } } - } - if ctx.buildConf.EnableCoroProgramBootstrapRun { - if len(targets) != 2 { - panic("coroutine program bootstrap runtime requires exactly two static targets") + if ctx.buildConf.EnableCoroProgramBootstrapRun { + factory = emitCoroProgramBootstrapFactoryV2(pkg, cfg.coroBootstrap, targets, cfg.coroManifestHash) + } + } else { + targets := make([]llssa.Function, len(cfg.coroBootstrap.Steps)) + for i, step := range cfg.coroBootstrap.Steps { + target := declareNoArgFunc(pkg, step.Target) + targets[i] = target + steps[i] = llssa.CoroProgramStep{ + Kind: llssa.CoroProgramStepKind(step.Kind), Flags: step.Role, + Target: target.Expr, Aux: step.Aux, + } + } + if ctx.buildConf.EnableCoroProgramBootstrapRun { + if len(targets) != 2 { + panic("coroutine program bootstrap v1 runtime requires exactly two static targets") + } + factory = emitCoroProgramBootstrapFactoryV1( + pkg, cfg.coroBootstrap, [2]llssa.Function{targets[0], targets[1]}, cfg.coroManifestHash, + ) } - factory = emitCoroProgramBootstrapFactoryV1( - pkg, - cfg.coroBootstrap, - [2]llssa.Function{targets[0], targets[1]}, - cfg.coroManifestHash, - ) } var factoryExpr llssa.Expr if factory != nil { factoryExpr = factory.Expr } - bootstrap = pkg.NewCoroProgramBootstrap(coroProgramBootstrapSymbolV1, llssa.CoroProgramBootstrapOptions{ - Version: coroProgramBootstrapVersionV1, + bootstrapSymbol := coroProgramBootstrapSymbolV1 + if version == coroProgramBootstrapVersionV2 { + bootstrapSymbol = coroProgramBootstrapSymbolV2 + } + bootstrap = pkg.NewCoroProgramBootstrap(bootstrapSymbol, llssa.CoroProgramBootstrapOptions{ + Version: version, // The runtime validates one program ABI identity across the manifest // and startup table. StepHash is an input to this final manifest hash, // not a second externally visible ABI identity. @@ -327,17 +404,19 @@ func filterAbiSymbol(abiInit int, sym *llssa.AbiSymbol) bool { } type entryFunctions struct { - runtimeStub llssa.Function - mainInit llssa.Function - mainMain llssa.Function - pyInit llssa.Function - pyFinalize llssa.Function - rtInit llssa.Function - abiInit llssa.Function - coroManifest llssa.Expr - coroFactory llssa.Function - coroBegin llssa.Function - coroRun llssa.Function + runtimeStub llssa.Function + mainInit llssa.Function + mainMain llssa.Function + pyInit llssa.Function + pyFinalize llssa.Function + rtInit llssa.Function + abiInit llssa.Function + coroManifest llssa.Expr + coroFactory llssa.Function + coroAllocatorBootstrap llssa.Function + coroBegin llssa.Function + coroRun llssa.Function + coroBootstrapVersion uint32 } // defineEntryFunction creates the program's entry function. The name is @@ -363,22 +442,27 @@ func defineEntryFunction(ctx *context, pkg llssa.Package, argcVar, argvVar llssa b := fn.MakeBody(1) b.Store(argcVar.Expr, fn.Param(0)) b.Store(argvVar.Expr, fn.Param(1)) + if fns.coroAllocatorBootstrap != nil { + b.Call(fns.coroAllocatorBootstrap.Expr) + } if IsStdioNobuf() { emitStdioNobuf(b, pkg, ctx.buildConf.Goos) } if fns.pyInit != nil { b.Call(fns.pyInit.Expr) } - if fns.rtInit != nil { - b.Call(fns.rtInit.Expr) - } - if fns.abiInit != nil { - b.Call(fns.abiInit.Expr) + if fns.coroBootstrapVersion != coroProgramBootstrapVersionV2 { + if fns.rtInit != nil { + b.Call(fns.rtInit.Expr) + } + if fns.abiInit != nil { + b.Call(fns.abiInit.Expr) + } + b.Call(fns.runtimeStub.Expr) } - b.Call(fns.runtimeStub.Expr) if fns.coroFactory != nil { - if fns.coroManifest.IsNil() || fns.coroBegin == nil || fns.coroRun == nil { - panic("coroutine program entry requires manifest, begin, factory, and run") + if fns.coroManifest.IsNil() || fns.coroAllocatorBootstrap == nil || fns.coroBegin == nil || fns.coroRun == nil { + panic("coroutine program entry requires allocator bootstrap, manifest, begin, factory, and run") } null := prog.Nil(prog.VoidPtr()) manifest := b.Convert(prog.VoidPtr(), fns.coroManifest) diff --git a/internal/build/main_module_test.go b/internal/build/main_module_test.go index 689e55e623..e4fb8b6b08 100644 --- a/internal/build/main_module_test.go +++ b/internal/build/main_module_test.go @@ -352,6 +352,243 @@ func TestGenMainModuleCoroProgramBootstrapNativeAndWasm(t *testing.T) { } } +func TestGenMainModuleCoroProgramBootstrapV2MixedNativeAndWasm(t *testing.T) { + llvm.InitializeAllTargets() + t.Setenv(llgoStdioNobuf, "") + tests := []struct { + name string + target *llssa.Target + goos string + goarch string + uintptrIR string + entryIR string + entryName string + }{ + { + name: "native", + goos: "linux", + goarch: "amd64", + uintptrIR: "i64", + entryIR: "define i32 @main(", + entryName: "main", + }, + { + name: "wasm", + target: &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}, + goos: "wasip1", + goarch: "wasm", + uintptrIR: "i32", + entryIR: "define hidden i32 @__main_argc_argv(", + entryName: "__main_argc_argv", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + prog := llssa.NewProgram(test.target) + defer prog.Dispose() + ctx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: test.goos, + Goarch: test.goarch, + EnableCoroEntryResolution: true, + EnableCoroPhysicalABI: true, + EnableCoroChildAwait: true, + EnableCoroProgramBootstrapABI: true, + EnableCoroProgramBootstrapRun: true, + }, + } + const anchor = "__llgo_coro_root_package_v1.0123456789abcdef0123456789abcdef" + var programHash [16]byte + for i := range programHash { + programHash[i] = byte(i + 1) + } + bootstrap := &coroProgramBootstrapV1{ + Version: coroProgramBootstrapVersionV2, + Steps: []coroProgramBootstrapStepV1{ + { + Kind: coroProgramStepCoroRootV1, Role: coroProgramStepRoleRuntimeInitV2, + FunctionID: "runtime-init-id", Target: llssa.PkgRuntime + ".init$coro", + Owner: llssa.PkgRuntime, CatalogTarget: anchor, Aux: 0, + }, + { + Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleABIInitV2, + FunctionID: "abi-init-id", Target: "init$abitypes", + }, + { + Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRolePublicRuntimeInitV2, + FunctionID: "public-runtime-init-id", Target: "runtime.init", + }, + { + Kind: coroProgramStepCoroRootV1, Role: coroProgramStepRolePackageInitV2, + FunctionID: "package-init-id", Target: "example.com/foo.init$coro", + Owner: "example.com/foo", CatalogTarget: anchor, Aux: 1, + }, + { + Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleMainV2, + FunctionID: "main-id", Target: "example.com/foo.main", + }, + }, + } + entry := genMainModule(ctx, llssa.PkgRuntime, + &packages.Package{ID: "example.com/foo", PkgPath: "example.com/foo", ExportFile: "foo.a"}, + &genConfig{ + rtInit: true, + pyInit: true, + coroRootAnchors: []string{anchor}, + coroManifestHash: programHash, + coroBootstrap: bootstrap, + }) + ir := entry.LPkg.String() + if !strings.Contains(ir, test.entryIR) { + t.Fatalf("mixed v2 bootstrap entry module missing %q:\n%s", test.entryIR, ir) + } + stepsLine := irLineWithPrefix(ir, "@"+coroProgramBootstrapSymbolV2+".steps =") + bootstrapLine := irLineWithPrefix(ir, "@"+coroProgramBootstrapSymbolV2+" =") + manifestLine := irLineWithPrefix(ir, "@"+coroProgramManifestSymbolV1+" =") + if stepsLine == "" || bootstrapLine == "" || manifestLine == "" { + t.Fatalf("missing mixed v2 bootstrap/manifest globals:\n%s", ir) + } + for _, want := range []string{ + "i32 2, i32 1, ptr @" + anchor + ", " + test.uintptrIR + " 0", + "i32 1, i32 2, ptr @\"init$abitypes\", " + test.uintptrIR + " 0", + "i32 1, i32 4, ptr @runtime.init, " + test.uintptrIR + " 0", + "i32 2, i32 8, ptr @" + anchor + ", " + test.uintptrIR + " 1", + "i32 1, i32 16, ptr @\"example.com/foo.main\", " + test.uintptrIR + " 0", + } { + if !strings.Contains(stepsLine, want) { + t.Fatalf("mixed v2 startup table missing %q: %s", want, stepsLine) + } + } + if !strings.Contains(bootstrapLine, "i32 2, i32 0") || + !strings.Contains(bootstrapLine, test.uintptrIR+" 5, ptr @"+coroProgramBootstrapSymbolV2+".steps, ptr @"+coroProgramBootstrapFactorySymbolV2) { + t.Fatalf("mixed v2 bootstrap version/count/steps/factory are not canonical: %s", bootstrapLine) + } + if !strings.Contains(manifestLine, "ptr @"+coroProgramBootstrapSymbolV2) { + t.Fatalf("manifest does not reference the mixed v2 bootstrap: %s", manifestLine) + } + if got := entry.LPkg.CoroProgramBootstrap(); got != coroProgramBootstrapSymbolV2 { + t.Fatalf("program bootstrap symbol = %q, want %q", got, coroProgramBootstrapSymbolV2) + } + + mod := entry.LPkg.Module() + publicRuntimeInit := mod.NamedFunction("runtime.init") + if publicRuntimeInit.IsNil() || !publicRuntimeInit.IsDeclaration() { + t.Fatalf("managed public runtime init must remain an unresolved archive reference, not an entry-module weak body:\n%s", ir) + } + factory := mod.NamedFunction(coroProgramBootstrapFactorySymbolV2) + if factory.IsNil() || factory.IsDeclaration() { + t.Fatalf("compiler-owned mixed v2 bootstrap factory is missing:\n%s", ir) + } + factoryBody := factory.String() + if got := strings.Count(factoryBody, "call void @__llgo_coro_await_prepare_v1"); got != 2 { + t.Fatalf("mixed v2 main-module factory await calls = %d, want 2:\n%s", got, factoryBody) + } + assertInOrder(t, factoryBody, + "call ptr %", + "call void @__llgo_coro_await_prepare_v1", + "call void @\"init$abitypes\"()", + "call void @runtime.init()", + "call ptr %", + "call void @__llgo_coro_await_prepare_v1", + "call void @\"example.com/foo.main\"()", + "call void @"+coroProgramCompletePrepareHookV1, + ) + + entryBody := mod.NamedFunction(test.entryName).String() + for _, legacyCall := range []string{ + "call void @\"" + llssa.PkgRuntime + ".init\"()", + "call void @\"init$abitypes\"()", + "call void @runtime.init()", + "call void @\"example.com/foo.init\"()", + "call void @\"example.com/foo.main\"()", + } { + if strings.Contains(entryBody, legacyCall) { + t.Fatalf("mixed v2 platform entry retained legacy call %q:\n%s", legacyCall, entryBody) + } + } + assertInOrder(t, entryBody, + "call void @"+coroFrameAllocatorBootstrapSymbolV1+"()", + "call void @Py_Initialize()", + "call ptr @"+coroProgramBeginSymbolV1, + "call ptr @"+coroProgramBootstrapFactorySymbolV2, + "call void @"+coroProgramRunSymbolV1, + "call void @Py_Finalize()", + ) + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify mixed v2 main module before coroutine passes: %v\n%s", err, ir) + } + if err := lowerCoroControlWrappers(ctx, entry.LPkg); err != nil { + t.Fatalf("lower mixed v2 main module coroutine: %v\n%s", err, entry.LPkg.String()) + } + post := mod.String() + for _, suffix := range []string{".resume", ".destroy"} { + if mod.NamedFunction(coroProgramBootstrapFactorySymbolV2 + suffix).IsNil() { + t.Fatalf("main-module CoroSplit did not create mixed v2 factory%s:\n%s", suffix, post) + } + } + for _, intrinsic := range []string{"llvm.coro.id", "llvm.coro.begin", "llvm.coro.suspend", "llvm.coro.resume", "llvm.coro.done", "llvm.coro.destroy"} { + if regexp.MustCompile(`call [^\n]*@` + regexp.QuoteMeta(intrinsic) + `\b`).MatchString(post) { + t.Fatalf("lowered mixed v2 main module still references %s:\n%s", intrinsic, post) + } + } + object, err := prog.TargetMachine().EmitToMemoryBuffer(mod, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit mixed v2 main-module object: %v\n%s", err, post) + } + object.Dispose() + }) + } +} + +func TestGenMainModuleCoroProgramBootstrapV2DefinesOnlyOwnedPublicRuntimeNoop(t *testing.T) { + llvm.InitializeAllTargets() + t.Setenv(llgoStdioNobuf, "") + prog := llssa.NewProgram(nil) + defer prog.Dispose() + ctx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + EnableCoroEntryResolution: true, + EnableCoroPhysicalABI: true, + EnableCoroChildAwait: true, + EnableCoroProgramBootstrapABI: true, + EnableCoroProgramBootstrapRun: true, + }, + } + bootstrap := &coroProgramBootstrapV1{ + Version: coroProgramBootstrapVersionV2, + Steps: []coroProgramBootstrapStepV1{ + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleRuntimeInitV2, FunctionID: "internal-runtime", Target: llssa.PkgRuntime + ".init"}, + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleABIInitV2, FunctionID: "compiler-abi", Target: "init$abitypes"}, + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRolePublicRuntimeInitV2, FunctionID: coroProgramPublicRuntimeNoopIDV2, Target: coroProgramPublicRuntimeNoopSymbolV2}, + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRolePackageInitV2, FunctionID: "package-init", Target: "example.com/no-public-runtime.init"}, + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleMainV2, FunctionID: "main", Target: "example.com/no-public-runtime.main"}, + }, + } + entry := genMainModule(ctx, llssa.PkgRuntime, + &packages.Package{ID: "example.com/no-public-runtime", PkgPath: "example.com/no-public-runtime", ExportFile: "no-public-runtime.a"}, + &genConfig{coroBootstrap: bootstrap}, + ) + module := entry.LPkg.Module() + if function := module.NamedFunction(coroProgramPublicRuntimeNoopSymbolV2); function.IsNil() || function.IsDeclaration() { + t.Fatalf("compiler-owned public runtime no-op is not defined:\n%s", module.String()) + } + if function := module.NamedFunction("runtime.init"); !function.IsNil() { + t.Fatalf("absent public runtime acquired a guessed runtime.init symbol:\n%s", module.String()) + } + if function := module.NamedFunction("syscall.init"); !function.IsNil() { + t.Fatalf("managed V2 entry retained a weak syscall.init interception body:\n%s", module.String()) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify absent-public-runtime v2 module: %v\n%s", err, module.String()) + } +} + func TestGenMainModuleCoroProgramBootstrapRuntimeSwitch(t *testing.T) { llvm.InitializeAllTargets() t.Setenv(llgoStdioNobuf, "") @@ -403,7 +640,11 @@ func TestGenMainModuleCoroProgramBootstrapRuntimeSwitch(t *testing.T) { if strings.Contains(entryBody, "call void @\"example.com/foo.init\"()") || strings.Contains(entryBody, "call void @\"example.com/foo.main\"()") { t.Fatalf("platform entry retained legacy direct init/main calls:\n%s", entryBody) } + if got := strings.Count(entryBody, "call void @"+coroFrameAllocatorBootstrapSymbolV1+"()"); got != 1 { + t.Fatalf("platform entry allocator bootstrap calls = %d, want exactly one:\n%s", got, entryBody) + } assertInOrder(t, entryBody, + "call void @"+coroFrameAllocatorBootstrapSymbolV1+"()", "call void @Py_Initialize()", "call void @\""+llssa.PkgRuntime+".init\"()", "call void @runtime.init()", diff --git a/internal/build/target_config_test.go b/internal/build/target_config_test.go index 6080b6b20c..408b4a2566 100644 --- a/internal/build/target_config_test.go +++ b/internal/build/target_config_test.go @@ -100,6 +100,34 @@ func TestNewLLSSATargetUsesResolvedLLVMConfig(t *testing.T) { Features: "+bulk-memory,+mutable-globals,+nontrapping-fptoint,+sign-ext,+atomics", }, }, + { + name: "wasip2-freestanding-frontend", + conf: &Config{Goos: "linux", Goarch: "arm", Target: "wasip2"}, + export: crosscompile.Export{ + LLVMTarget: "wasm32-unknown-wasi", + CPU: "generic", + Features: "+bulk-memory,+mutable-globals,+nontrapping-fptoint,+sign-ext,-multivalue,-reference-types", + }, + want: llssa.TargetSpec{ + Triple: "wasm32-unknown-wasi", + CPU: "generic", + Features: "+bulk-memory,+mutable-globals,+nontrapping-fptoint,+sign-ext,-multivalue,-reference-types", + }, + }, + { + name: "wasm-unknown-freestanding-frontend", + conf: &Config{Goos: "linux", Goarch: "arm", Target: "wasm-unknown"}, + export: crosscompile.Export{ + LLVMTarget: "wasm32-unknown-unknown", + CPU: "generic", + Features: "+mutable-globals,+nontrapping-fptoint,+sign-ext,-multivalue,-reference-types", + }, + want: llssa.TargetSpec{ + Triple: "wasm32-unknown-unknown", + CPU: "generic", + Features: "+mutable-globals,+nontrapping-fptoint,+sign-ext,-multivalue,-reference-types", + }, + }, { name: "thumb", conf: &Config{Goos: "linux", Goarch: "arm", Target: "rp2040", OptLevel: optlevel.Oz}, @@ -249,6 +277,8 @@ func TestResolvedTargetCompatibilityAudit(t *testing.T) { {name: "rp2040", applied: true}, // thumb/arm are layout-compatible {name: "riscv32", applied: true}, // riscv32/arm are layout-compatible {name: "wasip1", applied: true}, // llgo's wasm32 frontend override is compatible + {name: "wasip2", applied: true}, // 32-bit arm frontend, wasm32 WASI Preview 2 backend + {name: "wasm-unknown", applied: true}, // 32-bit arm frontend, freestanding wasm32 backend {name: "nintendoswitch", applied: true}, // aarch64/arm64 are layout-compatible } for _, tt := range tests { diff --git a/internal/coro/func_flow.go b/internal/coro/func_flow.go index c99888ceb2..88bd81e615 100644 --- a/internal/coro/func_flow.go +++ b/internal/coro/func_flow.go @@ -110,10 +110,10 @@ func (p *SSAPlan) CallPlan(call ssa.CallInstruction) (SSACallPlan, bool) { } // ElidesCall reports whether trusted frontend policy proved that the exact SSA -// call emits no callable function edge. The source operation may be omitted or -// lowered inline as a no-suspend compiler intrinsic. Elided calls deliberately -// have no CallPlan and must not be treated as DirectPlain or another callable -// ABI edge. +// declaration call emits no callable edge. The source operation may be omitted, +// lowered inline, or replaced by separately frozen lowered calls. Elided calls +// deliberately have no CallPlan and must not be treated as DirectPlain or +// another callable ABI edge; replacement edges retain their own effects. func (p *SSAPlan) ElidesCall(call ssa.CallInstruction) bool { if p == nil || call == nil { return false @@ -122,6 +122,17 @@ func (p *SSAPlan) ElidesCall(call ssa.CallInstruction) bool { return ok } +// RawFunctionAddressArgument reports whether the exact call argument is +// lowered as a raw static function entry rather than as a Go interface or +// descriptor value. +func (p *SSAPlan) RawFunctionAddressArgument(call ssa.CallInstruction, argument int) bool { + if p == nil || call == nil || argument < 0 { + return false + } + _, ok := p.rawAddressArgs[ssaCallArgumentUse{call: call, argument: argument}] + return ok +} + func cloneSSAValuePlan(plan SSAValuePlan) SSAValuePlan { plan.Funcs = cloneFuncRepMap(plan.Funcs) return plan @@ -161,6 +172,9 @@ type ssaFuncFlow struct { canonicalizer *ssaFunctionCanonicalizer directPlainArgs map[ssaCallArgumentUse]struct{} directPlainOrder []ssaCallArgumentUse + rawAddressArgs map[ssaCallArgumentUse]struct{} + rawAddressOrder []ssaCallArgumentUse + rawAddressBoxes map[*ssa.MakeInterface]ssaCallArgumentUse closedValues map[ssa.Value]SSAClosedDynamicCallCertificate } @@ -177,12 +191,23 @@ func analyzeSSAFunctionFlow( dynamicResolution DynamicResolution, canonicalizer *ssaFunctionCanonicalizer, directPlainArgs []ssaCallArgumentUse, + rawAddressArgs []ssaCallArgumentUse, closedDynamicCalls map[ssa.CallInstruction]SSAClosedDynamicCallCertificate, ) (*ssaFuncFlow, error) { directPlainSet := make(map[ssaCallArgumentUse]struct{}, len(directPlainArgs)) for _, use := range directPlainArgs { directPlainSet[use] = struct{}{} } + rawAddressSet := make(map[ssaCallArgumentUse]struct{}, len(rawAddressArgs)) + rawAddressBoxes := make(map[*ssa.MakeInterface]ssaCallArgumentUse, len(rawAddressArgs)) + for _, use := range rawAddressArgs { + rawAddressSet[use] = struct{}{} + if use.call != nil && use.call.Common() != nil && use.argument >= 0 && use.argument < len(use.call.Common().Args) { + if boxed, ok := use.call.Common().Args[use.argument].(*ssa.MakeInterface); ok { + rawAddressBoxes[boxed] = use + } + } + } flow := &ssaFuncFlow{ allValues: make(map[ssa.Value]struct{}), index: make(map[ssa.Value]int), @@ -194,6 +219,9 @@ func analyzeSSAFunctionFlow( canonicalizer: canonicalizer, directPlainArgs: directPlainSet, directPlainOrder: append([]ssaCallArgumentUse(nil), directPlainArgs...), + rawAddressArgs: rawAddressSet, + rawAddressOrder: append([]ssaCallArgumentUse(nil), rawAddressArgs...), + rawAddressBoxes: rawAddressBoxes, closedValues: make(map[ssa.Value]SSAClosedDynamicCallCertificate, len(closedDynamicCalls)), } for call, certificate := range closedDynamicCalls { @@ -502,7 +530,9 @@ func (f *ssaFuncFlow) seedInstruction(instruction ssa.Instruction) { } } case *ssa.MakeInterface: - f.markBoundary(instruction.X) + if _, rawAddress := f.rawAddressBoxes[instruction]; !rawAddress { + f.markBoundary(instruction.X) + } case *ssa.MakeClosure: for _, binding := range instruction.Bindings { f.markBoundary(binding) @@ -536,6 +566,9 @@ func (f *ssaFuncFlow) seedInstruction(instruction ssa.Instruction) { if _, directPlain := f.directPlainArgs[ssaCallArgumentUse{call: instruction, argument: argument}]; directPlain { continue } + if _, rawAddress := f.rawAddressArgs[ssaCallArgumentUse{call: instruction, argument: argument}]; rawAddress { + continue + } f.markBoundary(value) } } @@ -560,6 +593,38 @@ func (f *ssaFuncFlow) validateDirectPlainCallArguments() error { return nil } +func (f *ssaFuncFlow) validateRawFunctionAddressCallArguments() error { + for _, use := range f.rawAddressOrder { + if use.call == nil || use.call.Common() == nil || use.argument < 0 || use.argument >= len(use.call.Common().Args) { + return fmt.Errorf("invalid raw function-address call argument index %d", use.argument) + } + boxed, ok := use.call.Common().Args[use.argument].(*ssa.MakeInterface) + if !ok { + return fmt.Errorf("raw function-address call argument %d in %q is not a MakeInterface", use.argument, use.call.Parent().Name()) + } + target, ok := boxed.X.(*ssa.Function) + if !ok { + return fmt.Errorf("raw function-address call argument %d in %q does not contain a static function", use.argument, use.call.Parent().Name()) + } + index, ok := f.index[target] + if !ok { + return fmt.Errorf("raw function-address target %q in %q has no function-value flow component", target.Name(), use.call.Parent().Name()) + } + root := f.root(index) + canonical, resolved, err := f.resolveTarget(target) + if err != nil { + return fmt.Errorf("resolve raw function-address target %q in %q: %w", target.Name(), use.call.Parent().Name(), err) + } + if !resolved || canonical == nil || !f.included[canonical] || f.unknown[root] || f.mayBeNil[root] || len(f.targets[root]) != 1 { + return fmt.Errorf("raw function-address target %q in %q is not a closed non-nil singleton in the emission universe", target.Name(), use.call.Parent().Name()) + } + if _, present := f.targets[root][canonical]; !present { + return fmt.Errorf("raw function-address target %q in %q disagrees with canonical function-value flow", target.Name(), use.call.Parent().Name()) + } + } + return nil +} + func (f *ssaFuncFlow) descriptorTargets(unknownTargets map[ssa.CallInstruction]UnknownTarget) map[*ssa.Function]bool { result := make(map[*ssa.Function]bool) seenRoots := make(map[int]bool) diff --git a/internal/coro/graph.go b/internal/coro/graph.go index bb681055c5..22a697e717 100644 --- a/internal/coro/graph.go +++ b/internal/coro/graph.go @@ -33,10 +33,15 @@ const ( CallSpawn // CallForeign stack-cuts the caller and contributes WaitForeign directly. CallForeign + // CallUnwind is an exact compiler-lowered call reachable only on a path + // that cannot return normally from the caller. It keeps the callee demanded + // for emission and panic-ABI verification, but its suspend effect and + // execution constraints do not describe the caller's normal-return body. + CallUnwind ) func (k CallKind) validate() error { - if k > CallForeign { + if k > CallUnwind { return fmt.Errorf("coro: invalid call kind %d", uint8(k)) } return nil @@ -209,6 +214,9 @@ func (g *Graph) AddUnknownCall(call UnknownCall) error { if err := call.Kind.validate(); err != nil { return err } + if call.Kind == CallUnwind { + return fmt.Errorf("coro: unwind-only call requires an exact target") + } if err := call.Target.validate(); err != nil { return err } @@ -326,6 +334,11 @@ func (g *Graph) Analyze() (*Plan, error) { case CallForeign: effectContribution = WaitForeign execContribution = execFlags[callee] & propagatedExecFlags + case CallUnwind: + // The exact target remains in the graph and is demanded below. + // Its behavior is confined to a path that cannot return normally, + // so it does not constrain the caller's normal-return body. + continue } nextEffect := effects[edge.Caller].Join(effectContribution) nextExec := execFlags[edge.Caller].Join(execContribution) @@ -389,6 +402,11 @@ func (g *Graph) Analyze() (*Plan, error) { contribution = AsyncDemand case CallForeign: contribution = SyncDemand + case CallUnwind: + // Legacy panic lowering is a synchronous boundary. A suspendable + // target is still emitted as a coroutine and must be rejected by + // the panic-ABI/lowering verifier until such an adapter exists. + contribution = SyncDemand case CallDirect, CallDefer: contribution = SyncDemand if effects[edge.Callee].MaySuspend() { diff --git a/internal/coro/plan_digest.go b/internal/coro/plan_digest.go index 7f652b49fd..86bcc754b8 100644 --- a/internal/coro/plan_digest.go +++ b/internal/coro/plan_digest.go @@ -31,7 +31,7 @@ import ( // PlanDigestSchema is the independent canonical schema used for archive cache // identity. It is deliberately separate from SummarySchema: summaries remain // diagnostic snapshots, while this document covers every lowering plan site. -const PlanDigestSchema = "llgo.coro.plan-digest.v5" +const PlanDigestSchema = "llgo.coro.plan-digest.v7" // Current experimental ABI identities. Keeping these in the analysis package // gives build, cache, and lowering code one version source of truth. @@ -45,11 +45,15 @@ const ( // its stack, but only the scheduler may subsequently resume or destroy either // frame. It deliberately does not claim spawn, park, preemption, or roots. SchedulerChildAwaitABIV0 = "llgo.coro.scheduler.child-await.v0" - // SchedulerProgramBootstrapABIV1 extends child-await with one - // compiler-owned stackless program root and the runtime's static single-P - // prepare/adopt/run driver. It still does not claim spawn, park, timers, or - // preemption. + // SchedulerProgramBootstrapABIV1 is the first compiler-owned stackless + // program root and static single-P prepare/adopt/run driver. It does not + // include preemption or heterogeneous startup steps. SchedulerProgramBootstrapABIV1 = "llgo.coro.scheduler.program-bootstrap.v1" + // SchedulerProgramBootstrapABIV2 adds conditional compiler safepoints, + // atomic preemption requests/requeue, and the heterogeneous startup-program + // contract. It still does not claim spawn, park, timers, or a production + // source of concurrent runnable Gs. + SchedulerProgramBootstrapABIV2 = "llgo.coro.scheduler.program-bootstrap.v2" PanicLegacyABIV0 = "llgo.coro.panic.legacy.v0" FuncRepABIV0 = "llgo.coro.func-rep.v0" // FuncRepABIV1 introduces an explicit descriptor/context representation for @@ -94,20 +98,21 @@ type planDigestRoot struct { } type planDigestFunction struct { - ID FunctionID `json:"id"` - IgnoredBody bool `json:"ignored_body"` - DeclaredEffect uint16 `json:"declared_effect"` - LocalEffect uint16 `json:"local_effect"` - Effect uint16 `json:"effect"` - DeclaredExec uint16 `json:"declared_exec"` - LocalExec uint16 `json:"local_exec"` - Exec uint16 `json:"exec"` - Demand uint8 `json:"demand"` - Emission uint8 `json:"emission"` - FuncRep uint8 `json:"func_rep"` - External uint8 `json:"external"` - Recursive bool `json:"recursive"` - Primary uint8 `json:"primary"` + ID FunctionID `json:"id"` + IgnoredBody bool `json:"ignored_body"` + ForeignNoBlockCertificate string `json:"foreign_noblock_certificate,omitempty"` + DeclaredEffect uint16 `json:"declared_effect"` + LocalEffect uint16 `json:"local_effect"` + Effect uint16 `json:"effect"` + DeclaredExec uint16 `json:"declared_exec"` + LocalExec uint16 `json:"local_exec"` + Exec uint16 `json:"exec"` + Demand uint8 `json:"demand"` + Emission uint8 `json:"emission"` + FuncRep uint8 `json:"func_rep"` + External uint8 `json:"external"` + Recursive bool `json:"recursive"` + Primary uint8 `json:"primary"` } type planDigestCall struct { @@ -126,6 +131,7 @@ type planDigestLoweredCall struct { Owner FunctionID `json:"owner"` LogicalName string `json:"logical_name"` Target FunctionID `json:"target"` + UnwindOnly bool `json:"unwind_only"` } type planDigestElidedCall struct { @@ -351,7 +357,12 @@ func (p *SSAPlan) canonicalDigestLoweredCalls() ([]planDigestLoweredCall, error) if !ok { return nil, fmt.Errorf("coro: lowered call %q in %q targets a function outside the plan", call.LogicalName, ownerID) } - ret = append(ret, planDigestLoweredCall{Owner: ownerID, LogicalName: call.LogicalName, Target: targetID}) + ret = append(ret, planDigestLoweredCall{ + Owner: ownerID, + LogicalName: call.LogicalName, + Target: targetID, + UnwindOnly: call.UnwindOnly, + }) } } sort.Slice(ret, func(i, j int) bool { @@ -506,6 +517,9 @@ func (p *SSAPlan) canonicalDigestFunctions() ([]planDigestFunction, error) { Recursive: plan.Recursive, Primary: uint8(plan.Primary), }) + if certificate, ok := p.ForeignNoBlockCertificate(function.Function); ok { + ret[len(ret)-1].ForeignNoBlockCertificate = certificate + } } return ret, nil } diff --git a/internal/coro/plan_digest_test.go b/internal/coro/plan_digest_test.go index 16b8c82bee..09b62a6173 100644 --- a/internal/coro/plan_digest_test.go +++ b/internal/coro/plan_digest_test.go @@ -667,6 +667,10 @@ func second() {} {LogicalName: "runtime.first", Target: second}, {LogicalName: "runtime.second", Target: first}, }) + unwindOnly := build([]SSALoweredCall{ + {LogicalName: "runtime.first", Target: first, UnwindOnly: true}, + {LogicalName: "runtime.second", Target: second}, + }) metadata := validPlanDigestMetadata() baselineDigest, err := baseline.CoroPlanDigest(metadata) if err != nil { @@ -686,6 +690,13 @@ func second() {} if baselineDigest == swappedDigest { t.Fatal("retargeting logical lowered-call identities did not change digest") } + unwindDigest, err := unwindOnly.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if baselineDigest == unwindDigest { + t.Fatal("changing a lowered call to unwind-only did not change digest") + } document, err := baseline.canonicalPlanDigest(metadata) if err != nil { t.Fatal(err) @@ -693,6 +704,13 @@ func second() {} if len(document.LoweredCalls) != 2 || document.LoweredCalls[0].LogicalName != "runtime.first" || document.LoweredCalls[1].LogicalName != "runtime.second" { t.Fatalf("canonical lowered calls = %+v", document.LoweredCalls) } + unwindDocument, err := unwindOnly.canonicalPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if len(unwindDocument.LoweredCalls) != 2 || !unwindDocument.LoweredCalls[0].UnwindOnly || unwindDocument.LoweredCalls[1].UnwindOnly { + t.Fatalf("canonical unwind-only lowered calls = %+v", unwindDocument.LoweredCalls) + } } func TestCoroPlanDigestDistinguishesExplicitAndPropagatedRoots(t *testing.T) { diff --git a/internal/coro/ssa_plan.go b/internal/coro/ssa_plan.go index 2be178954c..e1bd7cf39e 100644 --- a/internal/coro/ssa_plan.go +++ b/internal/coro/ssa_plan.go @@ -73,6 +73,13 @@ type Roots []Root type SSAFunctionPolicy struct { Effect Effect Exec ExecFlags + // ForeignNoBlockCertificate is a frozen frontend proof that one exact + // external declaration has a bounded, nonblocking physical ABI. The + // opaque certificate identity is retained in SSAPlan and its archive digest; + // it must never be synthesized from a display name. Certified declarations + // remain IRQUnsafe unless a separate proof exists; this certificate removes + // only BlockForeign/WaitForeign. + ForeignNoBlockCertificate string // IgnoreBody states that the frontend does not emit this SSA body's Go // instructions because the function is an external declaration in the // frozen physical ABI. AnalyzeSSA excludes that body from value flow, calls, @@ -121,12 +128,18 @@ type SSAClosedDynamicCallCertificate struct { // LogicalName is a frontend-owned stable identity used to resolve the exact // helper again during code generation; it is not a symbol-name heuristic. // -// The first lowering slice projects every record as an ordinary direct call. -// AnalyzeSSA may refine that edge to a foreign boundary from the target's -// frozen function policy, exactly as it does for an explicit static call. +// AnalyzeSSA projects an ordinary record as a direct call and may refine that +// edge to a foreign boundary from the target's frozen function policy, exactly +// as it does for an explicit static call. UnwindOnly records remain exact +// demand edges, but do not propagate target effects into the owner's +// normal-return plan. type SSALoweredCall struct { LogicalName string Target *ssa.Function + // UnwindOnly is true only when every physical use of LogicalName in this + // owner is in a CFG block that cannot reach a normal Return. It is a frozen + // frontend proof, not a target-name or runtime-policy heuristic. + UnwindOnly bool } // SSAConfig controls the SSA-to-Graph analysis bridge. It deliberately has no @@ -175,12 +188,15 @@ type SSAConfig struct { ClassifyUnknownCall func(caller *ssa.Function, call ssa.CallInstruction) (UnknownTarget, error) // ClassifyElidedCall identifies a direct static call for which the frontend - // emits no callable function edge: either the call is omitted entirely or a - // proven no-suspend compiler intrinsic is lowered inline in the caller. Such - // a site contributes no graph edge and has no CallPlan, but remains in the - // plan/digest. The callback is trusted frontend policy, not an effect - // summary: AnalyzeSSA rejects attempts to elide go, defer, or dynamic calls. - // Argument-producing SSA instructions remain analyzed independently. + // emits no callable edge to that exact SSA declaration: either the call is + // omitted entirely, a proven no-suspend compiler intrinsic is lowered inline + // in the caller, or the declaration is replaced by exact calls supplied + // through ClassifyLoweredCalls. Such a site has no CallPlan but remains in the + // plan/digest. Eliding the declaration does not elide separately classified + // lowered calls or their effects. The callback is trusted frontend policy, + // not an effect summary: AnalyzeSSA rejects attempts to elide go, defer, or + // dynamic calls. Argument-producing SSA instructions remain analyzed + // independently. ClassifyElidedCall func(caller *ssa.Function, call ssa.CallInstruction) (bool, error) // ClassifyDirectPlainCallArgument identifies one exact static-call argument @@ -194,6 +210,15 @@ type SSAConfig struct { // a named //llgo:type C callback parameter. ClassifyDirectPlainCallArgument func(caller *ssa.Function, call ssa.CallInstruction, argument int) (bool, error) + // ClassifyRawFunctionAddressCallArgument identifies an exact direct static + // call argument whose frontend lowering consumes a transient + // MakeInterface{X:*ssa.Function} structurally and emits only X's raw entry + // address. The interface value is never materialized, so this one use must + // not force X into Dispatch representation. AnalyzeSSA validates the exact + // SSA shape and sole-consumer relationship; all ordinary interface uses keep + // their canonical descriptor boundary. + ClassifyRawFunctionAddressCallArgument func(caller *ssa.Function, call ssa.CallInstruction, argument int) (bool, error) + // ClassifyClosedDynamicCall supplies a frozen whole-program proof for one // exact ordinary dynamic *ssa.Call whose callee value crosses descriptor // storage but has a closed nil-or-singleton target set. This is not a general @@ -249,17 +274,19 @@ type SSARootPlan struct { // SSAPlan is the compilation-scoped whole-program result. Its maps remain // private so consumers cannot reconstruct identities from display strings. type SSAPlan struct { - plan *Plan - roots []SSARootPlan - functions []SSAFunctionPlan - byFunction map[*ssa.Function]FunctionID - byID map[FunctionID]*ssa.Function - ignoredBodies map[*ssa.Function]struct{} - valuePlans map[ssa.Value]SSAValuePlan - callPlans map[ssa.CallInstruction]SSACallPlan - elidedCalls map[ssa.CallInstruction]struct{} - loweredCalls map[*ssa.Function][]SSALoweredCall - functionIDs FunctionIDConfig + plan *Plan + roots []SSARootPlan + functions []SSAFunctionPlan + byFunction map[*ssa.Function]FunctionID + byID map[FunctionID]*ssa.Function + ignoredBodies map[*ssa.Function]struct{} + valuePlans map[ssa.Value]SSAValuePlan + callPlans map[ssa.CallInstruction]SSACallPlan + elidedCalls map[ssa.CallInstruction]struct{} + rawAddressArgs map[ssaCallArgumentUse]struct{} + loweredCalls map[*ssa.Function][]SSALoweredCall + foreignNoBlock map[*ssa.Function]string + functionIDs FunctionIDConfig } type ssaFunctionResolution struct { @@ -400,6 +427,18 @@ func (p *SSAPlan) IgnoresBody(fn *ssa.Function) bool { return ok } +// ForeignNoBlockCertificate returns the opaque frozen frontend certificate +// attached to one exact external declaration. The certificate is part of the +// immutable SSA plan and CoroPlanDigest; callers must not infer it from a +// function name or external symbol spelling. +func (p *SSAPlan) ForeignNoBlockCertificate(fn *ssa.Function) (string, bool) { + if p == nil || fn == nil { + return "", false + } + certificate, ok := p.foreignNoBlock[fn] + return certificate, ok +} + // LoweredCalls returns the exact compiler-inserted calls frozen for owner in // LogicalName order. The returned slice is a defensive copy. func (p *SSAPlan) LoweredCalls(owner *ssa.Function) []SSALoweredCall { @@ -640,6 +679,15 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err bodyFunctions = append(bodyFunctions, fn) bodyFunctionSet[fn] = true } + if certificate := trusted.ForeignNoBlockCertificate; certificate != "" { + if !utf8.ValidString(certificate) { + return nil, fmt.Errorf("coro: classify SSA function %q: foreign noblock certificate is not a valid UTF-8 identity", fn.Name()) + } + if !trusted.IgnoreBody || !trusted.OverrideExternal || trusted.External != ExternalKnown || + trusted.Effect != NoSuspend || trusted.Exec != IRQUnsafe || trusted.NeedsDispatch { + return nil, fmt.Errorf("coro: classify SSA function %q: foreign noblock certificate requires an ignored external-known declaration with no suspend effect, exactly irq-unsafe execution, and no dispatch", fn.Name()) + } + } trustedPolicies[fn] = trusted } dynamicCandidates, err = filterSSADynamicCandidateSites(dynamicCandidates, bodyFunctionSet, canonicalizer) @@ -684,17 +732,24 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err if err != nil { return nil, err } + rawFunctionAddressCallArguments, err := classifySSARawFunctionAddressCallArguments(bodyFunctions, config) + if err != nil { + return nil, err + } closedDynamicCalls, err := classifySSAClosedDynamicCalls(bodyFunctions, includedSet, bodyFunctionSet, trustedPolicies, canonicalizer, config) if err != nil { return nil, err } - flow, err := analyzeSSAFunctionFlow(bodyFunctions, includedSet, ids, dynamicCandidates, config.DynamicResolution, canonicalizer, directPlainCallArguments, closedDynamicCalls) + flow, err := analyzeSSAFunctionFlow(bodyFunctions, includedSet, ids, dynamicCandidates, config.DynamicResolution, canonicalizer, directPlainCallArguments, rawFunctionAddressCallArguments, closedDynamicCalls) if err != nil { return nil, fmt.Errorf("coro: analyze SSA function-value flow: %w", err) } if err := flow.validateDirectPlainCallArguments(); err != nil { return nil, fmt.Errorf("coro: validate trusted direct-plain call arguments: %w", err) } + if err := flow.validateRawFunctionAddressCallArguments(); err != nil { + return nil, fmt.Errorf("coro: validate trusted raw function-address call arguments: %w", err) + } elidedCalls, err := classifySSAElidedCalls(bodyFunctions, config) if err != nil { return nil, err @@ -726,6 +781,7 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err // authoritative and is joined only after that suppression. policy.Exec = policy.Exec.Join(trusted.Exec) policy.NeedsDispatch = policy.NeedsDispatch || trusted.NeedsDispatch + policy.ForeignNoBlockCertificate = trusted.ForeignNoBlockCertificate if trusted.OverrideExternal { policy.External = trusted.External policy.OverrideExternal = true @@ -886,17 +942,27 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err } } result := &SSAPlan{ - plan: base, - roots: canonicalRoots, - functions: make([]SSAFunctionPlan, 0, len(included)), - byFunction: ids, - byID: byID, - ignoredBodies: ignoredBodies, - valuePlans: valuePlans, - callPlans: callPlans, - elidedCalls: elidedCallSet, - loweredCalls: loweredCalls, - functionIDs: config.FunctionIDs, + plan: base, + roots: canonicalRoots, + functions: make([]SSAFunctionPlan, 0, len(included)), + byFunction: ids, + byID: byID, + ignoredBodies: ignoredBodies, + valuePlans: valuePlans, + callPlans: callPlans, + elidedCalls: elidedCallSet, + rawAddressArgs: make(map[ssaCallArgumentUse]struct{}, len(rawFunctionAddressCallArguments)), + loweredCalls: loweredCalls, + foreignNoBlock: make(map[*ssa.Function]string), + functionIDs: config.FunctionIDs, + } + for fn, policy := range policies { + if policy.ForeignNoBlockCertificate != "" { + result.foreignNoBlock[fn] = policy.ForeignNoBlockCertificate + } + } + for _, use := range rawFunctionAddressCallArguments { + result.rawAddressArgs[use] = struct{}{} } for _, functionPlan := range base.Functions() { result.functions = append(result.functions, SSAFunctionPlan{ @@ -966,7 +1032,10 @@ func addSSAClassifiedLoweredCalls( result[owner] = calls } for _, call := range calls { - kind := staticCallKind(CallDirect, policies[call.Target]) + kind := CallUnwind + if !call.UnwindOnly { + kind = staticCallKind(CallDirect, policies[call.Target]) + } if err := graph.AddCall(CallEdge{Caller: ids[owner], Callee: ids[call.Target], Kind: kind}); err != nil { return nil, fmt.Errorf("coro: add lowered call %q from %q to %q: %w", call.LogicalName, owner.Name(), call.Target.Name(), err) } @@ -1096,6 +1165,53 @@ func classifySSADirectPlainCallArguments(functions []*ssa.Function, config SSACo return result, nil } +func classifySSARawFunctionAddressCallArguments(functions []*ssa.Function, config SSAConfig) ([]ssaCallArgumentUse, error) { + var result []ssaCallArgumentUse + if config.ClassifyRawFunctionAddressCallArgument == nil { + return nil, nil + } + for _, caller := range functions { + for _, block := range caller.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(ssa.CallInstruction) + if !ok || call.Common() == nil { + continue + } + for argument, value := range call.Common().Args { + rawAddress, err := config.ClassifyRawFunctionAddressCallArgument(caller, call, argument) + if err != nil { + return nil, fmt.Errorf("coro: classify trusted raw function-address call argument %d in %q: %w", argument, caller.Name(), err) + } + if !rawAddress { + continue + } + direct, directCall := call.(*ssa.Call) + if !directCall || call.Common().StaticCallee() == nil || call.Common().IsInvoke() { + return nil, fmt.Errorf("coro: trusted raw function-address argument %d in %q must belong to a direct static call", argument, caller.Name()) + } + if _, builtin := call.Common().Value.(*ssa.Builtin); builtin { + return nil, fmt.Errorf("coro: trusted raw function-address argument %d in %q cannot belong to a builtin call", argument, caller.Name()) + } + boxed, ok := value.(*ssa.MakeInterface) + if !ok { + return nil, fmt.Errorf("coro: trusted raw function-address argument %d in %q must be a MakeInterface", argument, caller.Name()) + } + target, ok := boxed.X.(*ssa.Function) + if !ok || len(target.FreeVars) != 0 { + return nil, fmt.Errorf("coro: trusted raw function-address argument %d in %q must contain a static function without captured state", argument, caller.Name()) + } + refs := boxed.Referrers() + if refs == nil || len(*refs) != 1 || (*refs)[0] != direct { + return nil, fmt.Errorf("coro: trusted raw function-address argument %d in %q must be the MakeInterface value's exact sole consumer", argument, caller.Name()) + } + result = append(result, ssaCallArgumentUse{call: call, argument: argument}) + } + } + } + } + return result, nil +} + func classifySSAClosedDynamicCalls( functions []*ssa.Function, included map[*ssa.Function]bool, diff --git a/internal/coro/ssa_plan_test.go b/internal/coro/ssa_plan_test.go index f89bc910c7..98123763d3 100644 --- a/internal/coro/ssa_plan_test.go +++ b/internal/coro/ssa_plan_test.go @@ -533,6 +533,64 @@ func outsideFrozenUniverse() {} } } +func TestAnalyzeSSAUnwindOnlyLoweredCallDoesNotPolluteNormalReturnPlan(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "lowered_unwind_only.go", `package coroid +func owner() {} +func helper() {} +`) + owner := packageFunction(t, pkg, "owner") + helper := packageFunction(t, pkg, "helper") + universe, err := NewSSAEmissionUniverse(prog, []*ssa.Function{owner, helper}) + if err != nil { + t.Fatal(err) + } + build := func(unwindOnly bool) *SSAPlan { + t.Helper() + plan, err := AnalyzeSSA(prog, Roots{{Function: owner, Demand: SyncDemand}}, SSAConfig{ + EmissionUniverse: universe, + ClassifyFunction: func(fn *ssa.Function) (SSAFunctionPolicy, error) { + if fn == helper { + return SSAFunctionPolicy{Effect: OpaqueSuspend, Exec: IRQUnsafe | OpaqueExec}, nil + } + return SSAFunctionPolicy{}, nil + }, + ClassifyLoweredCalls: func(fn *ssa.Function) ([]SSALoweredCall, error) { + if fn == owner { + return []SSALoweredCall{{LogicalName: "runtime.helper", Target: helper, UnwindOnly: unwindOnly}}, nil + } + return nil, nil + }, + MaxPlainInstructions: -1, + }) + if err != nil { + t.Fatal(err) + } + return plan + } + + unwind := build(true) + unwindOwner := functionPlanFor(t, unwind, owner) + if unwindOwner.Effect != NoSuspend || unwindOwner.Exec.Contains(IRQUnsafe|OpaqueExec) || unwindOwner.Emission != EmitPlain { + t.Fatalf("unwind-only owner plan = %+v, want an unpolluted normal-return plain body", unwindOwner) + } + unwindTarget := functionPlanFor(t, unwind, helper) + if unwindTarget.Demand != SyncDemand || unwindTarget.Emission != EmitCoroutine || !unwindTarget.Effect.IsOpaque() { + t.Fatalf("unwind-only target plan = %+v, want retained synchronous demand and coroutine emission", unwindTarget) + } + if got := unwind.LoweredCalls(owner); len(got) != 1 || !got[0].UnwindOnly || got[0].Target != helper { + t.Fatalf("unwind-only frozen calls = %+v", got) + } + + ordinary := build(false) + ordinaryOwner := functionPlanFor(t, ordinary, owner) + if !ordinaryOwner.Effect.IsOpaque() || !ordinaryOwner.Exec.Contains(IRQUnsafe|OpaqueExec) || ordinaryOwner.Emission != EmitCoroutine { + t.Fatalf("normal-return-reachable owner plan = %+v, want exact target effects propagated", ordinaryOwner) + } + if got := functionPlanFor(t, ordinary, helper); got.Demand != AsyncDemand { + t.Fatalf("normal-return-reachable target demand = %s, want async", got.Demand) + } +} + func TestAnalyzeSSAClassifiedLoweredCallsFailClosed(t *testing.T) { prog, pkg := buildCoroTestSSA(t, "lowered_calls_invalid.go", `package coroid func owner() {} diff --git a/runtime/internal/clite/pthread/pthread.go b/runtime/internal/clite/pthread/pthread.go index fab5c20c00..148340aeff 100644 --- a/runtime/internal/clite/pthread/pthread.go +++ b/runtime/internal/clite/pthread/pthread.go @@ -60,6 +60,10 @@ func Cancel(thread Thread) c.Int // This is the same value that is returned in *thread in the // pthread_create(3) call that created this thread. // +// pthread_self only reads the calling thread's fixed-size identity. It neither +// waits on an external resource nor invokes a callback; IRQUnsafe is retained. +// +//llgo:coro noblock //go:linkname Self C.pthread_self func Self() Thread diff --git a/runtime/internal/clite/pthread/sync/sync.go b/runtime/internal/clite/pthread/sync/sync.go index 688c88303a..fce83e4fd4 100644 --- a/runtime/internal/clite/pthread/sync/sync.go +++ b/runtime/internal/clite/pthread/sync/sync.go @@ -90,6 +90,10 @@ func (a *MutexAttr) SetType(typ MutexType) c.Int { return 0 } // ----------------------------------------------------------------------------- +// pthread_mutex_init initializes caller-owned fixed-size state; it does not +// acquire the mutex or invoke a callback. IRQUnsafe is retained. +// +//llgo:coro noblock //go:linkname c_pthread_mutex_init C.pthread_mutex_init func c_pthread_mutex_init(m *Mutex, attr *MutexAttr) c.Int @@ -99,6 +103,10 @@ func c_pthread_mutex_destroy(m *Mutex) c.Int //go:linkname c_pthread_mutex_lock C.pthread_mutex_lock func c_pthread_mutex_lock(m *Mutex) c.Int +// pthread_mutex_unlock releases rather than acquires the mutex; it does not +// wait for ownership or invoke a callback. IRQUnsafe is retained. +// +//llgo:coro noblock //go:linkname c_pthread_mutex_unlock C.pthread_mutex_unlock func c_pthread_mutex_unlock(m *Mutex) c.Int diff --git a/runtime/internal/clite/time/time.go b/runtime/internal/clite/time/time.go index eec83f6179..357df8ef3b 100644 --- a/runtime/internal/clite/time/time.go +++ b/runtime/internal/clite/time/time.go @@ -32,6 +32,10 @@ const ( ClockTSize = 8 ) +// time reads one fixed-size wall-clock value. It neither waits on an external +// resource nor invokes a caller-provided callback; IRQUnsafe is retained. +// +//llgo:coro noblock //go:linkname Time C.time func Time(timer *TimeT) TimeT diff --git a/runtime/internal/clite/tls/tls_gc.go b/runtime/internal/clite/tls/tls_gc.go index afd10494de..0293a6088b 100644 --- a/runtime/internal/clite/tls/tls_gc.go +++ b/runtime/internal/clite/tls/tls_gc.go @@ -65,10 +65,9 @@ func deregisterSlot[T any](s *slot[T]) { func (s *slot[T]) rootRange() (start, end c.Pointer) { begin := unsafe.Pointer(s) size := unsafe.Sizeof(*s) - beginAddr := uintptr(begin) - if beginAddr > ^uintptr(0)-size { - panic("tls: pointer arithmetic overflow in rootRange") - } - endPtr := unsafe.Pointer(beginAddr + size) + // A slot always points at one complete calloc allocation. unsafe.Add keeps + // this callback cleanup path allocation-free and non-panicking; a generic Go + // panic cannot cross the pthread TLS destructor's synchronous C ABI. + endPtr := unsafe.Add(begin, size) return c.Pointer(begin), c.Pointer(endPtr) } diff --git a/ssa/abitype.go b/ssa/abitype.go index 71f09014b8..48e3ffc6d2 100644 --- a/ssa/abitype.go +++ b/ssa/abitype.go @@ -62,6 +62,21 @@ var ( types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.Uintptr])), false) ) +// ABITypeRuntimeFunctions returns the logical runtime functions whose +// addresses abiType embeds while materializing the descriptor for t. These are +// references, not calls: consumers must demand the selected entries without +// inheriting their suspend effects. +func (p Program) ABITypeRuntimeFunctions(t types.Type) []string { + ret := make([]string, 0, 2) + if name := p.abi.EqualName(t); name != "" { + ret = append(ret, name) + } + if _, ok := types.Unalias(t).(*types.Map); ok { + ret = append(ret, "typehash") + } + return ret +} + func directIfaceType(t types.Type) bool { switch t := types.Unalias(t).(type) { case *types.Named: diff --git a/ssa/coro.go b/ssa/coro.go index 9cb8bcc78c..5d3a56aa88 100644 --- a/ssa/coro.go +++ b/ssa/coro.go @@ -120,8 +120,9 @@ type CoroProgramManifestOptions struct { } // CoroProgramStepKind identifies one statically ordered program startup step. -// The numeric values are part of the version-one runtime ABI; zero is reserved -// so a zero-initialized or missing step always fails validation. +// The numeric values are shared by the version-one and version-two runtime +// ABIs; zero is reserved so a zero-initialized or missing step always fails +// validation. type CoroProgramStepKind uint32 const ( @@ -140,11 +141,23 @@ const ( CoroProgramStepMain ) -// CoroProgramStep describes one entry in a version-one program startup table. -// Flags is exactly one CoroProgramStepInit or CoroProgramStepMain role. Target -// must be a same-module constant function for DirectPlain or a same-module -// constant global for CoroRoot. Aux is encoded as target uintptr and is the -// root descriptor index for CoroRoot. +// Version-two startup step role flags. The bits intentionally start at bit +// zero again: a bootstrap version selects the meaning of the complete table, +// and a step role is never interpreted without first validating that version. +// Exactly one role is required on every step in this order. +const ( + CoroProgramStepInternalRuntimeInitV2 uint32 = 1 << iota + CoroProgramStepCompilerABIInitV2 + CoroProgramStepPublicRuntimeInitV2 + CoroProgramStepMainPackageInitV2 + CoroProgramStepMainV2 +) + +// CoroProgramStep describes one entry in a versioned program startup table. +// Flags is the exact role required at the entry's canonical position for that +// bootstrap version. Target must be a same-module constant function for +// DirectPlain or a same-module constant global for CoroRoot. Aux is encoded as +// target uintptr and is the root descriptor index for CoroRoot. type CoroProgramStep struct { Kind CoroProgramStepKind Flags uint32 @@ -153,9 +166,10 @@ type CoroProgramStep struct { } // CoroProgramBootstrapOptions describes the entry module's immutable startup -// table. Flags is reserved and must be zero. ABIHash covers the ordered steps -// and their referenced catalog. Factory may be Nil in the data-only phase; a -// non-Nil factory must use the root factory ABI and belong to this module. +// table. Version must be one or two. Flags is reserved and must be zero. +// ABIHash covers the ordered steps and their referenced catalog. Factory may +// be Nil in the data-only phase; a non-Nil factory must use the root factory +// ABI and belong to this module. type CoroProgramBootstrapOptions struct { Version uint32 Flags uint32 @@ -537,16 +551,16 @@ func (p Package) CoroProgramManifest() string { // { version i32, flags i32, hashLo i64, hashHi i64, // stepCount uintptr, steps ptr, factory ptr } // -// The canonical Init/Main step list is materialized as an internal constant -// array named name + ".steps", whose element layout is: +// The canonical version-specific step list is materialized as an internal +// constant array named name + ".steps", whose element layout is: // // { kind i32, flags i32, target ptr, aux uintptr } // -// Exactly two steps in Init, Main order are required, so a successfully emitted -// descriptor always has count two and a non-null steps pointer. Factory is null -// when omitted. Both the table and each step use target uintptr width and -// alignment. Each entry module may define at most one program bootstrap -// descriptor. +// Version one requires exactly Init, Main. Version two requires exactly +// InternalRuntimeInit, CompilerABIInit, PublicRuntimeInit, MainPackageInit, +// Main. Factory is null when omitted. Both the table and each step use target +// uintptr width and alignment. Each entry module may define at most one program +// bootstrap descriptor. func (p Package) NewCoroProgramBootstrap( name string, opts CoroProgramBootstrapOptions, ) Expr { @@ -559,8 +573,29 @@ func (p Package) NewCoroProgramBootstrap( if opts.Flags != 0 { panic("ssa: coroutine program bootstrap flags must be zero") } - if len(opts.Steps) != 2 { - panic(fmt.Sprintf("ssa: coroutine program bootstrap requires exactly two steps, got %d", len(opts.Steps))) + var roles []uint32 + switch opts.Version { + case 1: + roles = []uint32{CoroProgramStepInit, CoroProgramStepMain} + case 2: + roles = []uint32{ + CoroProgramStepInternalRuntimeInitV2, + CoroProgramStepCompilerABIInitV2, + CoroProgramStepPublicRuntimeInitV2, + CoroProgramStepMainPackageInitV2, + CoroProgramStepMainV2, + } + default: + panic(fmt.Sprintf("ssa: coroutine program bootstrap has unsupported version %d", opts.Version)) + } + if len(opts.Steps) != len(roles) { + if opts.Version == 1 { + panic(fmt.Sprintf("ssa: coroutine program bootstrap requires exactly two steps, got %d", len(opts.Steps))) + } + panic(fmt.Sprintf( + "ssa: coroutine program bootstrap version %d requires exactly %d steps, got %d", + opts.Version, len(roles), len(opts.Steps), + )) } if !coroProgramFitsUintptr(p.Prog, uint64(len(opts.Steps))) { panic("ssa: coroutine program bootstrap step count overflows target uintptr") @@ -588,10 +623,7 @@ func (p Package) NewCoroProgramBootstrap( stepValues := make([]llvm.Value, len(opts.Steps)) constantDeclarations := make([]llvm.Value, 0, len(opts.Steps)) for i, step := range opts.Steps { - wantRole := CoroProgramStepInit - if i == 1 { - wantRole = CoroProgramStepMain - } + wantRole := roles[i] if step.Flags != wantRole { panic(fmt.Sprintf( "ssa: coroutine program bootstrap step %d flags %#x must be %#x", @@ -894,6 +926,60 @@ func (c *CoroBuilder) Suspend() BasicBlock { return c.emitSuspend(false) } +// SuspendCurrentBlock emits a non-final stack cut while preserving the +// builder's current logical BasicBlock. The physical resume block becomes the +// logical block's last LLVM block, so later branches and phi incoming edges +// continue to refer to the source block even when one or more coroutine cuts +// split its physical control flow. Frontends lowering a multi-block source CFG +// must use this form; Suspend remains the low-level form that exposes the new +// resume block as a distinct logical block. +func (c *CoroBuilder) SuspendCurrentBlock() BasicBlock { + c.requireActive("suspend current block") + b := c.b + logical := b.blk + if logical == nil { + panic("ssa: suspend current block requires an active logical block") + } + resume := c.emitSuspend(false) + logical.last = resume.last + b.blk = logical + return logical +} + +// SuspendCurrentBlockIf emits a non-final stack cut only on condition's true +// edge. before runs in that edge immediately before llvm.coro.suspend and must +// append straight-line state publication only. Both the false edge and the +// resumed true edge join a new physical continuation that becomes the current +// logical block's tail, preserving source-CFG phi predecessor identity. +func (c *CoroBuilder) SuspendCurrentBlockIf(condition Expr, before func(Builder)) BasicBlock { + c.requireActive("conditionally suspend current block") + b := c.b + logical := b.blk + if logical == nil { + panic("ssa: conditionally suspend current block requires an active logical block") + } + if condition.IsNil() || condition.kind != vkBool { + panic("ssa: conditional coroutine suspend requires a boolean condition") + } + suspendBlk := b.Func.MakeBlock() + continueBlk := b.Func.MakeBlock() + b.If(condition, suspendBlk, continueBlk) + + b.SetBlock(suspendBlk) + if before != nil { + callbackPoint := captureCoroFrameCallbackPoint(b) + before(b) + callbackPoint.ensureContinuation(b, "conditional-suspend") + } + c.emitSuspend(false) + b.Jump(continueBlk) + + b.SetBlock(continueBlk) + logical.last = continueBlk.last + b.blk = logical + return logical +} + // Finish emits the final suspend and completes the shared cleanup/return // blocks. No further instructions may be emitted through c afterwards. func (c *CoroBuilder) Finish() { diff --git a/ssa/coro_test.go b/ssa/coro_test.go index bc5d452aef..7f02aa7aff 100644 --- a/ssa/coro_test.go +++ b/ssa/coro_test.go @@ -85,6 +85,125 @@ func TestCoroBuilderPresplitShape(t *testing.T) { } } +func TestCoroBuilderSuspendCurrentBlockPreservesLogicalCFG(t *testing.T) { + Initialize(InitAll) + prog := NewProgram(nil) + defer prog.Dispose() + pkg := prog.NewPackage("corologicalblock", "coro/logical/block") + defer pkg.Module().Dispose() + + fn := pkg.NewFunc("coro_logical_block", coroHandleSignature(), InGo) + b := fn.MakeBody(1) + defer b.Dispose() + coro := b.BeginCoro(CoroOptions{Frame: CoroFrameOps{ + Alloc: func(Builder, Expr, Expr) Expr { return prog.Nil(prog.VoidPtr()) }, + Free: func(Builder, Expr, Expr, Expr) {}, + }}) + + logical := fn.MakeBlock() + join := fn.MakeBlock() + b.Jump(logical) + b.SetBlock(logical) + first := logical.first + originalLast := logical.last + + if got := coro.SuspendCurrentBlock(); got != logical { + t.Fatalf("first suspend returned block %p, want logical block %p", got, logical) + } + firstResume := logical.last + if firstResume.C == originalLast.C { + t.Fatal("first suspend did not advance the logical block's physical tail") + } + if logical.first.C != first.C || b.blk != logical { + t.Fatal("first suspend did not preserve the current logical block") + } + + if got := coro.SuspendCurrentBlock(); got != logical { + t.Fatalf("second suspend returned block %p, want logical block %p", got, logical) + } + secondResume := logical.last + if secondResume.C == firstResume.C { + t.Fatal("second suspend did not advance the logical block's physical tail") + } + if logical.first.C != first.C || b.blk != logical { + t.Fatal("second suspend did not preserve the current logical block") + } + savedLogical := b.blk + b.blk = nil + mustPanicContains(t, "active logical block", func() { coro.SuspendCurrentBlock() }) + b.blk = savedLogical + + b.Jump(join) + b.SetBlock(join) + phi := b.Phi(prog.Byte()) + phi.AddIncoming(b, []BasicBlock{logical}, func(int, BasicBlock) Expr { + return prog.IntVal(1, prog.Byte()) + }) + coro.Finish() + b.EndBuild() + + if got := phi.impl.IncomingBlock(0); got.C != secondResume.C { + t.Fatal("phi predecessor does not use the logical block's post-suspend physical tail") + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify coroutine with logical-block suspends: %v\n%s", err, pkg.Module().String()) + } +} + +func TestCoroBuilderConditionalSuspendPreservesLogicalCFG(t *testing.T) { + Initialize(InitAll) + prog := NewProgram(nil) + defer prog.Dispose() + pkg := prog.NewPackage("coroconditionalblock", "coro/conditional/block") + defer pkg.Module().Dispose() + + fn := pkg.NewFunc("coro_conditional_block", coroHandleSignature(), InGo) + b := fn.MakeBody(1) + defer b.Dispose() + coro := b.BeginCoro(CoroOptions{Frame: CoroFrameOps{ + Alloc: func(Builder, Expr, Expr) Expr { return prog.Nil(prog.VoidPtr()) }, + Free: func(Builder, Expr, Expr, Expr) {}, + }}) + logical := fn.MakeBlock() + join := fn.MakeBlock() + b.Jump(logical) + b.SetBlock(logical) + first := logical.first + mustPanicContains(t, "boolean condition", func() { + coro.SuspendCurrentBlockIf(prog.IntVal(1, prog.Byte()), nil) + }) + callbackCalls := 0 + if got := coro.SuspendCurrentBlockIf(prog.BoolVal(true), func(b Builder) { + callbackCalls++ + b.Call(pkg.NewFunc("publish_yield", functionSignature(nil, nil), InC).Expr) + }); got != logical { + t.Fatalf("conditional suspend returned block %p, want %p", got, logical) + } + if callbackCalls != 1 || logical.first.C != first.C || b.blk != logical { + t.Fatal("conditional suspend did not preserve its logical block or publication callback") + } + continuation := logical.last + b.Jump(join) + b.SetBlock(join) + phi := b.Phi(prog.Byte()) + phi.AddIncoming(b, []BasicBlock{logical}, func(int, BasicBlock) Expr { + return prog.IntVal(1, prog.Byte()) + }) + coro.Finish() + b.EndBuild() + + if got := phi.impl.IncomingBlock(0); got.C != continuation.C { + t.Fatal("conditional suspend phi predecessor does not use the joined physical continuation") + } + ir := pkg.Module().String() + if !strings.Contains(ir, "br i1 true") || !strings.Contains(ir, "call void @publish_yield") { + t.Fatalf("conditional suspend lacks poll branch/publication path:\n%s", ir) + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify conditional coroutine suspend: %v\n%s", err, ir) + } +} + func TestCoroBuilderCoroSplit(t *testing.T) { fixture := newCoroTestFixture(t, nil, 32) mod := fixture.pkg.Module() @@ -1072,7 +1191,7 @@ func TestCoroProgramBootstrapTargetLayout(t *testing.T) { bootstrap := pkg.NewCoroProgramBootstrap( bootstrapName, CoroProgramBootstrapOptions{ - Version: 13, + Version: 1, ABIHash: hash, Steps: steps, Factory: factory, @@ -1120,7 +1239,7 @@ func TestCoroProgramBootstrapTargetLayout(t *testing.T) { t.Fatalf("bootstrap initializer is not a seven-field constant struct: %v", initializer) } wantFixed := []uint64{ - 13, + 1, 0, 0x5051525354555657, 0x6061626364656667, @@ -1216,6 +1335,7 @@ func TestCoroProgramBootstrapRejectsMisuse(t *testing.T) { plain := pkg.NewFunc("valid_plain", functionSignature(nil, nil), InC) anchor := newCoroProgramPackageAnchor(pkg, "valid_root_anchor", false) valid := CoroProgramBootstrapOptions{ + Version: 1, Steps: []CoroProgramStep{ {Kind: CoroProgramStepDirectPlain, Flags: CoroProgramStepInit, Target: plain.Expr}, {Kind: CoroProgramStepCoroRoot, Flags: CoroProgramStepMain, Target: anchor}, @@ -1236,6 +1356,16 @@ func TestCoroProgramBootstrapRejectsMisuse(t *testing.T) { bad.Flags = 1 pkg.NewCoroProgramBootstrap("bootstrap_flags", bad) }) + mustPanicContains(t, "unsupported version 0", func() { + bad := valid + bad.Version = 0 + pkg.NewCoroProgramBootstrap("bootstrap_version_zero", bad) + }) + mustPanicContains(t, "unsupported version 3", func() { + bad := valid + bad.Version = 3 + pkg.NewCoroProgramBootstrap("bootstrap_version_unknown", bad) + }) for name, steps := range map[string][]CoroProgramStep{ "zero": nil, "one": valid.Steps[:1], @@ -1389,6 +1519,7 @@ func TestCoroProgramBootstrapRejectsMisuse(t *testing.T) { wasmAnchor := newCoroProgramPackageAnchor(wasmPkg, "wasm_root_anchor", false) mustPanicContains(t, "aux overflows target uintptr", func() { wasmPkg.NewCoroProgramBootstrap("wasm_aux_overflow", CoroProgramBootstrapOptions{ + Version: 1, Steps: []CoroProgramStep{ {Kind: CoroProgramStepDirectPlain, Flags: CoroProgramStepInit, Target: wasmPlain.Expr}, { @@ -1417,10 +1548,153 @@ func TestCoroProgramBootstrapRejectsMisuse(t *testing.T) { }) } +func TestCoroProgramBootstrapV2MixedStartupTable(t *testing.T) { + Initialize(InitAll) + prog := NewProgram(nil) + defer prog.Dispose() + pkg := prog.NewPackage("corobootstrapv2", "coro/bootstrap/v2") + defer pkg.Module().Dispose() + + plains := [3]Function{ + pkg.NewFunc("internal_runtime_init", functionSignature(nil, nil), InC), + pkg.NewFunc("public_runtime_init", functionSignature(nil, nil), InC), + pkg.NewFunc("main", functionSignature(nil, nil), InC), + } + anchors := [2]Expr{ + newCoroProgramPackageAnchor(pkg, "compiler_abi_init_anchor", false), + newCoroProgramPackageAnchor(pkg, "main_package_init_anchor", false), + } + factory := pkg.NewFunc("bootstrap_factory_v2", coroRootFactoryTestSignature(), InC) + roles := [5]uint32{ + CoroProgramStepInternalRuntimeInitV2, + CoroProgramStepCompilerABIInitV2, + CoroProgramStepPublicRuntimeInitV2, + CoroProgramStepMainPackageInitV2, + CoroProgramStepMainV2, + } + steps := []CoroProgramStep{ + {Kind: CoroProgramStepDirectPlain, Flags: roles[0], Target: plains[0].Expr}, + {Kind: CoroProgramStepCoroRoot, Flags: roles[1], Target: anchors[0], Aux: 2}, + {Kind: CoroProgramStepDirectPlain, Flags: roles[2], Target: plains[1].Expr}, + {Kind: CoroProgramStepCoroRoot, Flags: roles[3], Target: anchors[1], Aux: 7}, + {Kind: CoroProgramStepDirectPlain, Flags: roles[4], Target: plains[2].Expr}, + } + bootstrap := pkg.NewCoroProgramBootstrap("__llgo_coro_program_bootstrap_v2", CoroProgramBootstrapOptions{ + Version: 2, + ABIHash: [16]byte{ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + }, + Steps: steps, + Factory: factory.Expr, + }) + + initializer := bootstrap.impl.Initializer() + if got := initializer.Operand(0).ZExtValue(); got != 2 { + t.Fatalf("bootstrap version = %d, want 2", got) + } + if got := initializer.Operand(4).ZExtValue(); got != 5 { + t.Fatalf("bootstrap step count = %d, want 5", got) + } + stepsGlobal := pkg.Module().NamedGlobal("__llgo_coro_program_bootstrap_v2.steps") + if stepsGlobal.IsNil() || !stepsGlobal.IsGlobalConstant() { + t.Fatal("v2 bootstrap lacks its constant steps table") + } + array := stepsGlobal.Initializer() + if got := array.OperandsCount(); got != 5 { + t.Fatalf("v2 steps count = %d, want 5", got) + } + wantKinds := [5]CoroProgramStepKind{ + CoroProgramStepDirectPlain, + CoroProgramStepCoroRoot, + CoroProgramStepDirectPlain, + CoroProgramStepCoroRoot, + CoroProgramStepDirectPlain, + } + wantAux := [5]uint64{0, 2, 0, 7, 0} + for index := 0; index < 5; index++ { + step := array.Operand(index) + if got := step.Operand(0).ZExtValue(); got != uint64(wantKinds[index]) { + t.Errorf("step %d kind = %d, want %d", index, got, wantKinds[index]) + } + if got := step.Operand(1).ZExtValue(); got != uint64(roles[index]) { + t.Errorf("step %d role = %#x, want %#x", index, got, roles[index]) + } + if got := step.Operand(3).ZExtValue(); got != wantAux[index] { + t.Errorf("step %d aux = %d, want %d", index, got, wantAux[index]) + } + } + if !anchors[0].impl.IsGlobalConstant() || !anchors[1].impl.IsGlobalConstant() { + t.Fatal("v2 coro-root anchor declarations were not normalized to constants") + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify v2 mixed bootstrap: %v\n%s", err, pkg.String()) + } +} + +func TestCoroProgramBootstrapV2RejectsShapeAndRoles(t *testing.T) { + Initialize(InitAll) + prog := NewProgram(nil) + defer prog.Dispose() + pkg := prog.NewPackage("badcorobootstrapv2", "bad/coro/bootstrap/v2") + defer pkg.Module().Dispose() + plains := [5]Function{ + pkg.NewFunc("v2_step_0", functionSignature(nil, nil), InC), + pkg.NewFunc("v2_step_1", functionSignature(nil, nil), InC), + pkg.NewFunc("v2_step_2", functionSignature(nil, nil), InC), + pkg.NewFunc("v2_step_3", functionSignature(nil, nil), InC), + pkg.NewFunc("v2_step_4", functionSignature(nil, nil), InC), + } + roles := [5]uint32{ + CoroProgramStepInternalRuntimeInitV2, + CoroProgramStepCompilerABIInitV2, + CoroProgramStepPublicRuntimeInitV2, + CoroProgramStepMainPackageInitV2, + CoroProgramStepMainV2, + } + valid := CoroProgramBootstrapOptions{Version: 2, Steps: make([]CoroProgramStep, 5)} + for index := range valid.Steps { + valid.Steps[index] = CoroProgramStep{ + Kind: CoroProgramStepDirectPlain, Flags: roles[index], Target: plains[index].Expr, + } + } + for _, count := range []int{0, 1, 2, 4, 6} { + bad := valid + bad.Steps = append([]CoroProgramStep(nil), valid.Steps...) + if count <= len(bad.Steps) { + bad.Steps = bad.Steps[:count] + } else { + bad.Steps = append(bad.Steps, valid.Steps[0]) + } + mustPanicContains(t, "version 2 requires exactly 5 steps", func() { + pkg.NewCoroProgramBootstrap(fmt.Sprintf("v2_bad_count_%d", count), bad) + }) + } + for index := range roles { + for name, role := range map[string]uint32{ + "zero": 0, + "next": roles[(index+1)%len(roles)], + "multiple": roles[index] | roles[(index+1)%len(roles)], + "unknown": 1 << 12, + } { + bad := valid + bad.Steps = append([]CoroProgramStep(nil), valid.Steps...) + bad.Steps[index].Flags = role + mustPanicContains(t, fmt.Sprintf("step %d flags", index), func() { + pkg.NewCoroProgramBootstrap(fmt.Sprintf("v2_bad_role_%d_%s", index, name), bad) + }) + } + } +} + func TestCoroBuilderRejectsMisuse(t *testing.T) { fixture := newCoroTestFixture(t, nil, 0) mustPanicContains(t, "finished coroutine", func() { fixture.coro.Suspend() }) + mustPanicContains(t, "finished coroutine", func() { fixture.coro.SuspendCurrentBlock() }) + mustPanicContains(t, "finished coroutine", func() { fixture.coro.SuspendCurrentBlockIf(fixture.prog.BoolVal(true), nil) }) mustPanicContains(t, "finished coroutine", func() { fixture.coro.Finish() }) + mustPanicContains(t, "nil coroutine builder", func() { (*CoroBuilder)(nil).SuspendCurrentBlock() }) + mustPanicContains(t, "nil coroutine builder", func() { (*CoroBuilder)(nil).SuspendCurrentBlockIf(Nil, nil) }) if (*CoroBuilder)(nil).Handle() != Nil { t.Fatal("nil coroutine builder returned a non-nil handle") } From 95288ede6cf7d4c03f54a728cd67ec1b76415e9c Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 22:38:45 +0800 Subject: [PATCH 15/32] docs(coro): record executable park and wasm prototype --- .github/workflows/coroutine.yml | 66 +++++++++++++++++++++++++++++++-- doc/llvm-coro-runtime-design.md | 42 +++++++++++---------- 2 files changed, 85 insertions(+), 23 deletions(-) diff --git a/.github/workflows/coroutine.yml b/.github/workflows/coroutine.yml index 8af6cad57d..d95fb56317 100644 --- a/.github/workflows/coroutine.yml +++ b/.github/workflows/coroutine.yml @@ -18,6 +18,7 @@ jobs: matrix: include: - { llvm: 19, go: "1.24.2", tags: "llvm19" } + - { llvm: 20, go: "1.24.2", tags: "llvm20" } - { llvm: 21, go: "1.24.2", tags: "llvm21" } - { llvm: 22, go: "1.24.2", tags: "llvm22" } - { llvm: 19, go: "1.26.5", tags: "llvm19" } @@ -46,6 +47,7 @@ jobs: - name: Test target-neutral coroutine runtime core run: | cd runtime + go test -race -shuffle=on ./internal/coroalloc -count=1 go test -race -shuffle=on ./internal/coro -count=1 # The complete LLGo runtime package intentionally owns symbols that # collide with the host Go runtime. Use the real production adapter @@ -57,14 +59,64 @@ jobs: ./internal/runtime/coro_program.go \ ./internal/runtime/coro_sched.go \ ./internal/runtime/coro_program_test.go \ - -run '^TestCoroProgramV1' -count=1 + -run '^TestCoroProgram(V1|V2)' -count=1 GOOS=js GOARCH=wasm CGO_ENABLED=0 go test \ -tags=coro_runtime_adapter_test \ -exec="$(go env GOROOT)/lib/wasm/go_js_wasm_exec" \ ./internal/runtime/coro_program.go \ ./internal/runtime/coro_sched.go \ ./internal/runtime/coro_program_test.go \ - -run '^TestCoroProgramV1' -count=1 + -run '^TestCoroProgram(V1|V2)' -count=1 + + - name: Compile coroutine allocator target backends + if: matrix.llvm == 19 && matrix.go == '1.24.2' + run: | + go build -o /tmp/llgo-coro ./cmd/llgo + check_backend() { + local label="$1" + local goos="$2" + local goarch="$3" + local tags="$4" + local log="/tmp/coroalloc-${label}.log" + ( + cd runtime + LLGO_BUILD_CACHE=off GOOS="$goos" GOARCH="$goarch" \ + /tmp/llgo-coro build -v -tags="$tags" ./internal/coroalloc + ) >"$log" 2>&1 + grep -F 'backend_webassembly.go' "$log" + grep -F 'NewFunc malloc func' "$log" + grep -F 'NewFunc free func' "$log" + ! grep -E 'backend_gc\.go|GC_malloc_uncollectable|GC_free' "$log" + } + check_backend js-wasm js wasm tinygo.wasm + check_backend wasip1 wasip1 wasm tinygo.wasm + check_backend wasip2 linux arm tinygo.wasm,wasip2 + check_backend wasm-unknown linux arm tinygo.wasm,wasm_unknown + + - name: Link named freestanding WebAssembly targets + if: matrix.llvm == 19 && matrix.go == '1.24.2' + env: + LLGO_WASM_TARGET_SMOKE: "1" + run: | + go test -v ./internal/crosscompile -run '^TestFreestandingWasmTargetToolchainSmoke$' -count=1 + go build -o /tmp/llgo-wasm-target ./cmd/llgo + for target in wasip2 wasm-unknown; do + output="/tmp/llgo-${target}.wasm" + LLGO_BUILD_CACHE=off LDFLAGS='--export=main' \ + /tmp/llgo-wasm-target build -target="$target" -o "$output" \ + ./internal/crosscompile/testdata/wasm_allocator + test "$(od -An -t x1 -N4 "$output" | tr -d ' \n')" = '0061736d' + symbols="$(llvm-nm --defined-only --format=just-symbols "$output")" + for symbol in main malloc free sbrk abort; do + grep -Fx "$symbol" <<<"$symbols" + done + ! grep -E '^GC_' <<<"$symbols" + test -z "$(llvm-nm --undefined-only --format=just-symbols "$output")" + if command -v wasmtime >/dev/null; then + result="$(wasmtime run --invoke main "$output" 0 0)" + test "$result" = '0' + fi + done - name: Compile coroutine runtime adapter across targets if: matrix.llvm == 19 @@ -77,7 +129,10 @@ jobs: - name: Test coroutine build integration if: matrix.llvm == 19 - run: go test ./internal/build -run 'Test(CoroPlanBuilderRunsBeforeCodegenWithoutChangingIR|CoroPlanInputCanonicalizesPatchedRoot|CoroPlanInputElidesOnlyFrontendNoInitCalls|CoroPlanInputOwnsFrozenDemandReferences|RequiredCoroProgramRuntimePlanPlainClosureAndConflicts|ActiveCoroABIVersions|BuildCoroPlanErrors|CoroEntryResolutionUsesPlanMatchedPackageCache|CoroEntryResolutionBuildsPreparedRuntimePackages|CoroRuntimeLinkRequirements|CoroEmissionCoverageStopsBeforeAnyPackageCodegen|CoroUnsupportedEntryResolutionReturnsErrorBeforeCodegen|CoroEmissionUniverseAcceptsModeTestVariants|CoroProgramBootstrapRejectsInvalidRootsBeforePackageCodegen)$' -count=1 + # Keep the focused workflow exhaustive for the build-side coroutine + # contract. This includes park effect seeding, frozen foreign noblock + # certificates, IRQUnsafe handling, and the exact legacy PanicABI stop. + run: go test ./internal/build -run 'Coro|Coroutine' -timeout=10m -count=1 - name: Test coroutine compiler integration if: matrix.llvm == 19 @@ -98,7 +153,10 @@ jobs: go test -tags='${{ matrix.tags }}' ./cl -run '^Test(CompilationCoroABIIdentityValidation|CoroEntryResolutionCacheRegistrationWithDigest|CoroPhysicalABICacheRegistrationPreservesPhysicalMetadata)$' -count=1 - name: Test coroutine physical ABI and function dispatch lowering - run: go test -tags='${{ matrix.tags }}' -v ./cl -run '^Test(Coro(LeafPhysicalABI|PhysicalABI|ChildAwaitPhysicalABIV1|ExplicitAsyncRootFactoryV1|ExplicitRootFactoryV1|ExplicitPlain|RootPackageAnchorV1|PlainDispatch)|EmissionUniverse(ActiveABIMethodTablesUseFrozenWrapperSymbols|ABIMethodDemandReferencesAreExactRecursiveAndOwnerScoped))' -count=1 + # Run every compiler test whose name is part of the coroutine contract; + # in particular this covers pure SSA aggregates/PHI and caller-frame + # park lowering on native64 and wasm32 before and after CoroSplit. + run: go test -tags='${{ matrix.tags }}' -v ./cl -run '^Test(Coro|EmissionUniverse(ActiveABIMethodTablesUseFrozenWrapperSymbols|ABIMethodDemandReferencesAreExactRecursiveAndOwnerScoped))' -count=1 - name: Test coroutine TLS function dispatch proof run: go test -tags='${{ matrix.tags }}' -v ./internal/build -run '^TestCoroTLS' -count=1 diff --git a/doc/llvm-coro-runtime-design.md b/doc/llvm-coro-runtime-design.md index e89c4fad9c..ef3efa09cb 100644 --- a/doc/llvm-coro-runtime-design.md +++ b/doc/llvm-coro-runtime-design.md @@ -1,12 +1,12 @@ # LLGo 基于 LLVM Coroutine 的运行时与抢占调度器总体设计 -状态：提案评审稿（完整总体设计） +状态：实现中（可验证无栈原型；尚非完整 Go runtime） -更新：2026-07-15 +更新：2026-07-16 -目标分支：`codex/llvm-coro-runtime-design` +目标分支：`cpunion/llgo:coro/phase14-plain-dispatch` -基线：`xgo-dev/main@2c9d1897d` +集成基线：`cpunion/llgo:llvm-coro` 关联提案：[Issue #1546](https://github.com/xgo-dev/llgo/issues/1546) @@ -1778,21 +1778,25 @@ Pure sync library/archive不需要链接scheduler。Executable一旦选择 `-sch 验收：纯 sync chain 只有 `F`；纯 async chain 只有 `F$coro`；动态 escape 才出现 descriptor/adapter；所有 `go` root和可挂起call都以LLVM-coro frame表示。 -当前落地状态（2026-07，实验 ABI v0/v1）： - -- 已完成全程序 SSA 的 Effect、Demand、FuncRep、稳定 FunctionID、精确 emission universe 和单 primary symbol 选择。激活 lowering 使用 archive-ready FunctionID，并以独立 canonical schema 对全部 function/call/value plan、Coro/Scheduler/Panic/FuncRep ABI 及 effective LLVM target/data layout 生成 `CoroPlanDigest`；相同完整计划可安全复用 package build cache，缺失或不匹配的 manifest 继续 fail closed。 -- `cpunion/llvm` 已覆盖 LLVM 19、21、22 的 switched-resume builder/CoroSplit；LLGo 的 v0 路径能为严格受限的 top-level `YieldOnly` 单块 leaf 只生成 `F$coro(Task, ResultSlot, args...) -> CoroHandle`，并生成目标相关 result descriptor 与版本化 frame alloc/free hook。未启用 v1 时，v0 symbol、hook 与 `scheduler.none` 行为保持不变。 -- v1 已加入 closed static `CallDirect + DirectCoro` 的 ordinary child await。父 frame 先按 Go 的从左到右顺序求值参数，在自己的 frame 中保留 result slot，创建只运行到 initial suspend 的 child，写入 parent link，发布 `Call/Suspended/stateID`，调用 `__llgo_coro_await_prepare_v1` 后切断栈。父代码不调用 child 的 `resume`、`done` 或 `destroy`；调度器是后续所有 resume/done/destroy 以及 active-frame 转换的唯一 owner。 -- v1 只为真正选择 `EmitCoroutine + DirectCoro` 的显式 async-only root 生成 `(g, out, startup) -> handle` typed factory 和 linker-discoverable descriptor；显式 root 若为 `EmitPlain + DirectPlain`，即使总 demand 因同步 caller 传播为 `BothDemand`，仍只保留唯一 plain body且不生成 per-function factory。仅因调用传播成为 async 的函数同样不生成第二入口。startup/result 的 size/alignment 使用目标 data layout，native64 与 wasm32 都有 pre-/post-CoroSplit 覆盖。每个含 coroutine root 的 package 按 canonical FunctionID 排序 descriptor，并生成唯一 `__llgo_coro_root_package_v1.` package anchor；descriptor 和 anchor 都由 `llvm.used` 保留，package cache manifest 同步记录 anchor symbol。 -- Build driver 从实际参与链接的 package cache metadata 收集并排序 anchor，在 entry module 生成 `__llgo_coro_program_manifest_v1`。Manifest 对 anchor 的普通 relocation 会从静态 archive 抽取对应 member，不依赖 section 扫描、constructor、`whole-archive` 或 `force-load`；native `-dead_strip`/`--gc-sections` 链接测试覆盖 manifest、anchor、descriptor 和 factory 的存活。默认及旧 capability 下 manifest 的 `bootstrap` 继续为 null；`EnableCoroProgramBootstrapABI` 仍是只生成、验证 descriptor 的独立 executable gate，并严格依赖 entry resolution、physical ABI 与 child-await。该 gate 在任何 package codegen 前，从实际 selected main package 的 exact SSA 对象冻结有序两步 `[synthetic package init, main.main]`，要求两者都是显式含 `AsyncDemand`、`Defined + EmitPlain + DirectPlain + NoSuspend`、无 `NeedsPreempt` 的 `func()`；不得扫描全部 main、依赖 init 或 root catalog。Entry module 随后发出 `__llgo_coro_program_bootstrap_v1` 与目标宽度 step table，manifest/bootstrap 共享覆盖 plan、target、catalog 和有序 step identity 的最终 hash。当前 `c-archive` 会形成嵌套 package archive，且 host 链接不会自动抽取含 manifest 的 entry member，所以 v1 对该 build mode明确 fail closed；只有实现 member flatten 与显式 host/bootstrap extraction contract 后才能开放。 -- Phase13-B 新增更窄的 `EnableCoroProgramBootstrapRun` production gate；它要求 descriptor gate，并把其中的 null factory 替换为 compiler-owned LLVM-coro factory。Factory 使用统一 HeaderV1、frame alloc/publish/complete/free hooks 和 initial/final suspend 生命周期，在同一无栈 root frame 中按顺序静态调用已验证的 init/main target。平台 entry 也只发出静态 `program_begin → factory → program_run` 调用，在该 gate 下移除旧 direct init/main；runtime 不接收、查找或调用任意用户函数指针。Descriptor-only gate、旧 scheduler ABI 和旧 entry 行为保持不变，可独立验证和回滚。 -- Entry module 在 v1 激活时生成编译器持有的 `__llgo_coro_resume_v1`、`__llgo_coro_done_v1` 和 `__llgo_coro_destroy_v1` C ABI wrapper，并在 object selection 前完成 coroutine pass lowering。Runtime 只通过这三个边界控制 handle，不读取 LLVM handle 私有布局；resume/done/destroy 的唯一 owner 规则不因 build mode 改变。Factory、entry driver 和 wrapper 均有 LLVM 19/21/22 以及 native64/wasm32 的 pre-/post-CoroSplit object 覆盖。 -- Promise/header 在 `coro.begin` 后、initial suspend 前发布；结果写入 frame 外、由 parent/root runtime 持有的 slot。v1 runtime contract 通过 `__llgo_coro_frame_alloc_v1`、`__llgo_coro_frame_publish_v1`、`__llgo_coro_await_prepare_v1`、`__llgo_coro_complete_prepare_v1`、`__llgo_coro_frame_free_v1` 传递 task/handle/header/storage；这些 hook 必须 NoSuspend、NoCallback，且不得进入用户 Go。`frame_publish_v1` 负责登记 handle/storage 并使 header 的 allocation-base 记录与实际分配一致。 -- `runtime/internal/coro` 已有不依赖 pthread、libuv、BDWGC 或 host API 的 target-neutral frame registry 与 deterministic single-P 生命周期 core：G 持有无栈 frame chain，P 维护 ready queue，child final suspend 后严格先 destroy/free 再恢复 parent，root/child 均检查 exactly-once destroy。Pointer-size-neutral manifest/bootstrap/anchor/descriptor ABI mirror 使用零分配完整校验器：先验证版本、flags、共享 hash、count/pointer/overflow、全 catalog、严格 Init→Main role 和 target/index，再返回只含静态 action 的 opaque snapshot；descriptor-only 校验允许 factory 为 null，runnable 校验则要求 exact expected factory。`runtime/internal/runtime` 的 production glue 使用静态单次 G/P 状态完成 `Validate → InitG → AdoptRoot → Enqueue → run → TerminalG`，任何嵌套、残留 frame/queue、重复运行或 ABI 不匹配都永久 fail closed；该调度状态不调用任意函数指针，也不依赖 TLS、libuv 或平台线程。LLVM root frame 仍经现有 `AllocRoot` 分配：native GC/nogc 当前分别落到 BDWGC/C malloc，baremetal 可使用 tinygogc；WASM linear-memory 与 embedded/bare-metal static/slab backend 尚须按 10.4 和 Phase 7 落地后，才可声明整个启动链 allocator-independent。这些分层状态机已有普通、race 和 native/wasm/embedded/bare-metal 交叉编译覆盖；production adapter 另以 test-only compiler-wrapper symbols 在 native 与 js/wasm32 实际执行完整 `Validate → destroy`，但这不是 LLGo/LLVM 跨语言链接测试。完整 entry→runtime→factory→scheduler linked smoke 仍受真实 TLS Dispatch 阻塞，必须在 plain descriptor/字段流解除该 blocker 后补齐，才能把 production gate 宣称为端到端可运行。 -- Production planner 将 compiler-generated entry/coroutine IR 引用的八个 runtime ABI body 及其精确 static call closure 作为显式 sync roots；只有这一 scheduler-stack island 可清除 scanner 产生的本地 loop/budget `NeedsPreempt`，用户显式 effect/exec/dispatch/external 冲突仍拒绝。Frontend 确实不生成的 noinit/decl zero-argument init call 以 exact call identity 记录为 elided，并进入 plan digest；builder 不能自行省略普通调用。Emission universe 还以冻结 opcode 记录 compiler intrinsic 的物理调用语义；当前只有参数必须为编译期字符串字面量、直接降为 LLVM 常量指针的 `llgo.cstr` 可作为 exact elided inline/NoSuspend site；`llgo.syscall`、分配、cgo、atomic、asm 和未知 intrinsic 均继续保守拒绝。每个 canonical function 的实际 frontend background 也被冻结：`//llgo:type C` 声明即使 SSA 中残留 fallback stub，也必须标记 `IgnoreBody`，其 stub 的 call、escape、递归和局部类型均不能进入 Go body 分析；普通 C 声明默认仍是 unknown foreign，不能仅凭 `InC` 推断 nonblocking。显式 `ExternalKnown` effect/exec summary 可以保留；只有 compiler-owned scheduler bootstrap closure 中精确到达的 C leaf 才临时提升为 `ExternalKnown + NoSuspend`，这是当前受控启动 island 的显式摘要，不是对所有 C 函数的通用信任。Named C callback 也仅在该 closure 内按 exact `(static call, argument index)` 豁免 Go closure canonicalization，而且 callback target 必须是 frozen `InGo`、closed、non-nil、无捕获、全 static、NoSuspend 的单一 plain body；同一值若还有 store、interface、普通 Go argument、open 或 multi-target use，仍强制 Dispatch。该边界可处理同步 signal handler，不会把 TLS destructor 这类真实动态 Go callback 假装成静态 C 回调。 -- 当前 frontend v1 的 coroutine body 仍只允许线性单块 scalar lowering，故意拒绝 spawn consumer、循环与抢占、channel/select、defer/panic、closure/method/generic、aggregate/pointer result 和 Dispatch。Production bootstrap 已能运行满足严格 DirectPlain init/main 与 runtime closure 的受限 executable，但真实标准库 runtime 路径会在 TLS 中存储并动态调用 Go destructor；它正确规划为 Dispatch，当前尚无 descriptor consumer，因此在 module 创建前 fail closed。当前 single-P core 也尚未实现 `go` spawn、park/wake、抢占请求/poll、channel/select、timer/netpoll 或多 P。本阶段形成可测试的 production root 生命周期、registry 和控制边界，不表示普通 executable 已完成 Go 标准库启动，更不表示提案已经完成。 -- 当前 cache digest 只解决同一完整程序计划下的内部 package cache；未知未来 caller 可复用的预编译 archive/标准库仍需 producer summary、canonical boundary Dispatch 和 linker ABI 校验，不能把 cache digest 当作 producer ABI summary。 -- 下一依赖顺序为：实现 v1 plain Dispatch descriptor 的 producer/consumer，使无捕获、NoSuspend、单 plain body 的动态 Go callback 保持源码同步调用风格且不复制函数主体；再用跨包高阶 summary/字段流证明 TLS `parameter → field → load → call` 的 closed target，未知或混合目标继续 fail closed。随后补齐 `go` spawn、park/wake，扩展 CFG/递归 coroutine lowering，并插入和验证 loop/recursion/long-block 抢占 poll。CoroRoot init/main 必须等通用 CFG/synthetic-init child-await lowering完成后开放。不得把 catalog 当作启动列表，也不得用扩大线性 allowlist或 runtime function-pointer fallback 绕过这些生命周期协议。 +当前落地状态（2026-07-16，实验 physical ABI v0/v1；scheduler ABI `llgo.coro.scheduler.program-bootstrap.v2`）： + +- 全程序 SSA 的 Effect、Demand、FuncRep、稳定 FunctionID、精确 emission universe、单 primary symbol 选择和 `CoroPlanDigest` 已落地。明确 plain 或 coro 的函数仍只有一个主体；仅真正动态的 func/`any`/interface consumer 才进入 descriptor/dispatch。缺失、过期或目标布局不匹配的计划与 cache manifest 均 fail closed。 +- LLGo 已固定使用 `cpunion/llvm` PR #5 的 LLVM 19–22 绑定。该分支吸收上游 LLVM 22 的完整 switch API 变更，并保留 LLGo 所需的 switched-resume builder/CoroSplit API；19、20、21、22 CI 均通过。LLGo 不再覆盖 LLVM 19 以下版本。 +- closed static `CallDirect + DirectCoro` 已使用 caller-frame await：父 frame 按 Go 从左到右顺序求值参数、保存 typed result slot、创建 initial-suspended child，然后由 scheduler 独占 resume/done/destroy。值传输已覆盖 pointer、uintptr、function、string、slice、named struct、fixed array 和多返回值；不是仅支持 scalar。 +- exact pure-SSA physical audit 已覆盖 stack alloc、local/global typed load/store、`FieldAddr`、static `IndexAddr`/`Index`、完整 fixed-array slice、`Field`/`Extract`/`Phi`、empty-interface direct value、受限 conversion/binop/unop、`len`/`cap`。heap escape、需要 allocation 的 interface box、slice 动态越界检查、pointer-containing global store、closure/type assertion/dynamic call 和任何隐藏 runtime helper 仍明确拒绝。 +- `program-bootstrap.v2` 在 codegen 前冻结五阶段表：`[internal runtime.init, init$abitypes, public runtime.init, selected main-package init, main.main]`。managed Go 阶段根据唯一 primary 选择 `DirectPlain` 或 `CoroRoot`；public runtime init 若存在则必须使用其 exact managed body，不存在时才由 compiler 生成 no-op。Coro 表项只绑定 package anchor/descriptor index，不复制函数体，也不把 catalog 当启动列表。 +- planner 已把 internal runtime init、selected package init 和 `main.main` 注入 managed demand。普通同步 Go/标准库调用风格不变，调用者根据精确 effect 自动被染成 coro；scheduler-stack hook closure 则是单独审计的 NoSuspend island，不能通过强改 demand 或放宽 trusted closure 绕过。 +- frozen foreign `//llgo:coro noblock` certificate 当前只授予已审计的 `time`、`pthread_self`、`pthread_mutex_init` 和 `pthread_mutex_unlock`。证书只移除未知阻塞，`IRQUnsafe` 仍保留但允许在普通 G 上执行。真实 runtime init 仍被 `pthread_key_create`、`rand`/`srand`、`GC_malloc`、mutex lock、Memcpy/Memset 等未完成边界挡住。 +- legacy PanicABI 仍是完整启动链的正式 blocker。exact proof 可追踪 `runtime.Panic → Rethrow → TracePanic → printany`，并在动态 `error.Error` 调用处停止；这里必须落地 non-legacy task-local PanicABI/descriptor dispatch，不能把动态调用误标为 plain。 +- 多基本块 CFG、聚合值、PHI 和抢占 lowering 已完成。自然循环、循环入口及每 64 条有效指令的长直线块插入 poll；scheduler 的 P 级原子 request 只有在 slow path 才执行 publish/yield/`llvm.coro.suspend`，fast path 不切换。LLVM 19–22 上均有 native64/wasm32 pre-/post-CoroSplit 与 object 测试。 +- park/wake handshake 已落地 32-bit 原子 `WaitToken`、generation ticket、early/late completion、唯一 waiter claim、ABA 范围校验及 terminal gate。精确 intrinsic `llgo.coroPark(token, ticket)` 被 Effect 分析识别为 `MayPark`，并在调用者当前 LLVM frame 中生成 park prepare、stateID、`coro.suspend` 和恢复路径；没有隐藏在普通同步 helper 中。channel/timer/syscall 的 submit/retry producer 尚未接入。 +- wait/preempt core 要求目标提供可靠的 32-bit atomic load/store/CAS。WASM 可直接满足；带 A 扩展的 RISC-V 可满足；ESP32-C3 RV32IMC 当前会在链接时缺少 `__atomic_*_4`，直到平台用 IRQ critical section 提供单核适配。这里故意不使用非原子 fallback。 +- `wasip1`、`wasip2` 和 `wasm-unknown` 明确选择 leaking/nogc frame backend，不依赖 libuv 或 BDWGC。`wasip2` 与 `wasm-unknown` 已通过真实 `llgo build -target=...`、wasm magic/symbol closure、无 `GC_*`/undefined 检查，并由 wasmtime 运行返回 0。当前 `wasip2` 产物是 Preview 2 目标的 core module，尚不是 WIT component。 +- frame allocator 已有 conservative BDWGC、nogc/WASM malloc 和 tinygogc/baremetal 后端。跨 suspend 的 pointer 目前只在 conservative 或 non-collecting 配置下安全；精确 frame root map、write barrier、STW、weak timer/finalizer 与 cleanup 语义尚未实现，不能据此宣称完整 Go GC 兼容。 +- deterministic single-P runtime 已能管理多个 frame、ready queue、preempt request、park/wake 和 terminal idle/requested/disabled 状态，但 production program 目前仍只有静态 bootstrap G。尚无 `go` spawn/newG、真实 tick/alarm request source、channel/select/sync slow path、timer/netpoll、异步 syscall submit/retry、task-local panic/defer/recover/Goexit 或多 P。 +- 完整真实 `entry → allocator → v2 factory → runtime/package init → main → scheduler` linked smoke 仍受上述 runtime/Panic/foreign blockers 限制；现有 runtime adapter 测试和 freestanding wasm CLI fixture 分别证明 scheduler ABI 与目标链接，不能合并表述为完整 Go runtime 已经端到端运行。 +- 当前 cache digest 只解决同一完整程序计划下的内部 package cache；未知未来 caller 可复用的预编译 archive/标准库仍需 producer summary、canonical boundary Dispatch 和 linker ABI 校验。 +- 后续依赖顺序是：closed static `go f(args)`/newG 与真实 platform request source；随后接 channel/timer/syscall producer 并跑完整 linked smoke；并行实现 non-legacy PanicABI；再补 suspended-frame GC、defer/recover/Goexit、多 P 与各 target event backend。所有阶段保持无栈、单 primary 和未证明即 fail closed。 ### Phase 1：单 P deterministic scheduler From a63666d660c0d030f409b201a217eb758065f894 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 22:46:24 +0800 Subject: [PATCH 16/32] test(coro): accept LLVM 20 GEP no-wrap flags --- cl/coro_abi_test.go | 2 +- cl/coro_pure_ssa_test.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cl/coro_abi_test.go b/cl/coro_abi_test.go index 9641b380aa..fdc0b6ed7a 100644 --- a/cl/coro_abi_test.go +++ b/cl/coro_abi_test.go @@ -655,7 +655,7 @@ func assertCoroResultSlotFields(t *testing.T, name, body, uintptrIR string) { resultType := regexp.QuoteMeta("{ ptr, " + uintptrIR + " }") for index, storeType := range []string{"ptr", uintptrIR} { field := regexp.MustCompile( - `(?m)^\s*(%[-a-zA-Z$._0-9]+) = getelementptr inbounds ` + resultType + + `(?m)^\s*(%[-a-zA-Z$._0-9]+) = getelementptr inbounds(?: (?:nuw|nusw))* ` + resultType + `, ptr [^,]+, i32 0, i32 ` + strconv.Itoa(index) + `\s*$`, ).FindStringSubmatch(body) if len(field) != 2 || !regexp.MustCompile(`(?m)^\s*store `+storeType+` [^,]+, ptr `+regexp.QuoteMeta(field[1])+`(?:,|\s*$)`).MatchString(body) { diff --git a/cl/coro_pure_ssa_test.go b/cl/coro_pure_ssa_test.go index ddb1e09848..d5ed5205ac 100644 --- a/cl/coro_pure_ssa_test.go +++ b/cl/coro_pure_ssa_test.go @@ -114,7 +114,6 @@ func TestCoroPureSSAPhysicalABIV1NativeAndWasm(t *testing.T) { aggregateIR := requireCoroPhysicalFunction(t, module, "foo.Aggregate").String() for _, required := range []string{ "alloca %foo.Pair", - "getelementptr inbounds %foo.Pair", "foo.Child$coro", "call void @" + coroAwaitPrepareHookV1, "call i1 @" + coroPreemptPollHookV1, @@ -123,6 +122,9 @@ func TestCoroPureSSAPhysicalABIV1NativeAndWasm(t *testing.T) { t.Fatalf("Root pure SSA coroutine lacks %q:\n%s", required, rootIR) } } + if !regexp.MustCompile(`getelementptr inbounds(?: (?:nuw|nusw))* %foo\.Pair`).MatchString(rootIR) { + t.Fatalf("Root pure SSA coroutine lacks typed Pair field addressing:\n%s", rootIR) + } for _, forbidden := range []string{ "CheckIndexRange", "AssertNilDeref", "AllocU", "AllocZ", "NewSlice2", "NewSlice3Bounds", "NewItab", } { From 7bb9c10cf5c3e98e9d9bd95c96870ebe84a7af7a Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 22:49:30 +0800 Subject: [PATCH 17/32] ci(coro): accept target-selected nogc allocator --- .github/workflows/coroutine.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coroutine.yml b/.github/workflows/coroutine.yml index d95fb56317..a86b16925f 100644 --- a/.github/workflows/coroutine.yml +++ b/.github/workflows/coroutine.yml @@ -78,15 +78,24 @@ jobs: local goarch="$3" local tags="$4" local log="/tmp/coroalloc-${label}.log" - ( + if ! ( cd runtime LLGO_BUILD_CACHE=off GOOS="$goos" GOARCH="$goarch" \ /tmp/llgo-coro build -v -tags="$tags" ./internal/coroalloc - ) >"$log" 2>&1 - grep -F 'backend_webassembly.go' "$log" - grep -F 'NewFunc malloc func' "$log" - grep -F 'NewFunc free func' "$log" - ! grep -E 'backend_gc\.go|GC_malloc_uncollectable|GC_free' "$log" + ) >"$log" 2>&1; then + cat "$log" + return 1 + fi + # Named wasm targets add nogc at LLGo target resolution, while a + # direct Go source-selection build uses backend_webassembly. Both + # are malloc/free backends and neither may retain BDWGC. + if ! grep -E 'Location: .*backend_(nogc|webassembly)\.go' "$log" || + ! grep -F 'NewFunc malloc func' "$log" || + ! grep -F 'NewFunc free func' "$log" || + grep -E 'backend_gc\.go|GC_malloc_uncollectable|GC_free' "$log"; then + cat "$log" + return 1 + fi } check_backend js-wasm js wasm tinygo.wasm check_backend wasip1 wasip1 wasm tinygo.wasm From 1734c3aa3bf573c4876e94ec5d1c123aae001156 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 22:54:25 +0800 Subject: [PATCH 18/32] test(coro): keep debug parameters across frame lowering --- .github/workflows/coroutine.yml | 34 --------------------------------- cl/coro_abi_test.go | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 36 deletions(-) diff --git a/.github/workflows/coroutine.yml b/.github/workflows/coroutine.yml index a86b16925f..1b92bbdea4 100644 --- a/.github/workflows/coroutine.yml +++ b/.github/workflows/coroutine.yml @@ -68,40 +68,6 @@ jobs: ./internal/runtime/coro_program_test.go \ -run '^TestCoroProgram(V1|V2)' -count=1 - - name: Compile coroutine allocator target backends - if: matrix.llvm == 19 && matrix.go == '1.24.2' - run: | - go build -o /tmp/llgo-coro ./cmd/llgo - check_backend() { - local label="$1" - local goos="$2" - local goarch="$3" - local tags="$4" - local log="/tmp/coroalloc-${label}.log" - if ! ( - cd runtime - LLGO_BUILD_CACHE=off GOOS="$goos" GOARCH="$goarch" \ - /tmp/llgo-coro build -v -tags="$tags" ./internal/coroalloc - ) >"$log" 2>&1; then - cat "$log" - return 1 - fi - # Named wasm targets add nogc at LLGo target resolution, while a - # direct Go source-selection build uses backend_webassembly. Both - # are malloc/free backends and neither may retain BDWGC. - if ! grep -E 'Location: .*backend_(nogc|webassembly)\.go' "$log" || - ! grep -F 'NewFunc malloc func' "$log" || - ! grep -F 'NewFunc free func' "$log" || - grep -E 'backend_gc\.go|GC_malloc_uncollectable|GC_free' "$log"; then - cat "$log" - return 1 - fi - } - check_backend js-wasm js wasm tinygo.wasm - check_backend wasip1 wasip1 wasm tinygo.wasm - check_backend wasip2 linux arm tinygo.wasm,wasip2 - check_backend wasm-unknown linux arm tinygo.wasm,wasm_unknown - - name: Link named freestanding WebAssembly targets if: matrix.llvm == 19 && matrix.go == '1.24.2' env: diff --git a/cl/coro_abi_test.go b/cl/coro_abi_test.go index fdc0b6ed7a..32284537d8 100644 --- a/cl/coro_abi_test.go +++ b/cl/coro_abi_test.go @@ -169,8 +169,20 @@ func Leaf(value uint32) uint32 { if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { t.Fatalf("verify debug coroutine before CoroSplit: %v\n%s", err, module.String()) } - if !strings.Contains(module.String(), "!dbg") { - t.Fatalf("debug coroutine omitted function/parameter metadata:\n%s", module.String()) + ir := module.String() + if !strings.Contains(ir, "!dbg") { + t.Fatalf("debug coroutine omitted function/parameter metadata:\n%s", ir) + } + parameter := regexp.MustCompile(`(?m)^(!\d+) = !DILocalVariable\(name: "value", arg: 1,`).FindStringSubmatch(ir) + if len(parameter) != 2 { + t.Fatalf("debug coroutine omitted source parameter metadata:\n%s", ir) + } + location := regexp.MustCompile( + `(?m)(?:#dbg_(?:value|declare)|@llvm\.dbg\.(?:value|declare))\([^\n]*` + + regexp.QuoteMeta(parameter[1]) + `(?:,|\))`, + ) + if !location.MatchString(ir) { + t.Fatalf("debug coroutine parameter metadata has no location record:\n%s", ir) } options := llvm.NewPassBuilderOptions() defer options.Dispose() From 6d0723f9e2e2619fcfba94c81f0c72ab62b574eb Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 23:00:48 +0800 Subject: [PATCH 19/32] runtime(coro): add closed static G spawn transaction --- runtime/internal/coro/frame.go | 21 +- runtime/internal/coro/scheduler.go | 54 ++- runtime/internal/coro/scheduler_spawn_test.go | 420 ++++++++++++++++++ runtime/internal/coro/scheduler_wait_test.go | 2 +- runtime/internal/coro/spawn.go | 277 ++++++++++++ runtime/internal/coroalloc/allocator.go | 23 + runtime/internal/runtime/coro_program.go | 2 +- runtime/internal/runtime/coro_program_test.go | 55 ++- runtime/internal/runtime/coro_sched.go | 26 +- runtime/internal/runtime/coro_spawn.go | 104 +++++ 10 files changed, 961 insertions(+), 23 deletions(-) create mode 100644 runtime/internal/coro/scheduler_spawn_test.go create mode 100644 runtime/internal/coro/spawn.go create mode 100644 runtime/internal/runtime/coro_spawn.go diff --git a/runtime/internal/coro/frame.go b/runtime/internal/coro/frame.go index cee69e4e52..4008b4c774 100644 --- a/runtime/internal/coro/frame.go +++ b/runtime/internal/coro/frame.go @@ -36,6 +36,19 @@ type HeaderV1 struct { Flags uint32 } +// FrameDescriptorV1 is the runtime prefix emitted for every physical +// coroutine frame. SpawnCommit currently admits only zero-result goroutine +// roots, so it validates this descriptor instead of trusting a nil result +// slot alone. +type FrameDescriptorV1 struct { + Version uint32 + Flags uint32 + HashLo uint64 + HashHi uint64 + ResultSize uintptr + ResultAlign uintptr +} + // SuspendReason describes why a coroutine returned control to its scheduler. type SuspendReason uint16 @@ -240,7 +253,7 @@ func PublishFrame(g *G, handle unsafe.Pointer, header *HeaderV1, storage unsafe. // coroutine; only the runtime driver may perform handle operations requested // by the scheduler action protocol. func PrepareAwait(g *G, parentHandle, childHandle unsafe.Pointer) bool { - if !ValidG(g) || g.pending.kind != pendingNone { + if !ValidG(g) || g.pending.kind != pendingNone || g.spawnChild != nil { return false } parent := findFrame(g, parentHandle) @@ -260,7 +273,7 @@ func PrepareAwait(g *G, parentHandle, childHandle unsafe.Pointer) bool { // PrepareComplete records a final-suspended frame. Destruction remains owned // by the scheduler and occurs only after the resume operation returns. func PrepareComplete(g *G, handle unsafe.Pointer, header *HeaderV1) bool { - if !ValidG(g) || handle == nil || header == nil || g.pending.kind != pendingNone { + if !ValidG(g) || handle == nil || header == nil || g.pending.kind != pendingNone || g.spawnChild != nil { return false } frame := findFrame(g, handle) @@ -278,7 +291,7 @@ func PrepareComplete(g *G, handle unsafe.Pointer, header *HeaderV1) bool { // handle remain owned by g; Resumed commits the transition only after the // direct llvm.coro.resume wrapper has returned to the scheduler. func PrepareYield(g *G, handle unsafe.Pointer, header *HeaderV1) bool { - if !ValidG(g) || handle == nil || header == nil || g.pending.kind != pendingNone { + if !ValidG(g) || handle == nil || header == nil || g.pending.kind != pendingNone || g.spawnChild != nil { return false } frame := findFrame(g, handle) @@ -297,7 +310,7 @@ func PrepareYield(g *G, handle unsafe.Pointer, header *HeaderV1) bool { // coroutine hooks, the transition is committed only after llvm.coro.resume // returns to Resumed on the scheduler stack. func PreparePark(g *G, handle unsafe.Pointer, header *HeaderV1, token *WaitToken, ticket WaitTicket) bool { - if !ValidG(g) || handle == nil || header == nil || g.pending.kind != pendingNone || + if !ValidG(g) || handle == nil || header == nil || g.pending.kind != pendingNone || g.spawnChild != nil || g.waitToken != nil || g.waitTicket != 0 || g.waiting || g.nextWait != nil { return false } diff --git a/runtime/internal/coro/scheduler.go b/runtime/internal/coro/scheduler.go index aa35628db5..fd15f54267 100644 --- a/runtime/internal/coro/scheduler.go +++ b/runtime/internal/coro/scheduler.go @@ -50,6 +50,20 @@ type G struct { // runP is scheduler-thread-only. An asynchronous producer requests a // reschedule through P's atomic gate and never reads this pointer. runP *P + + // spawnChild is non-nil only while this running G owns a begin/commit spawn + // transaction. The child remains reachable through the current P's root G + // while its initial-suspended root frame is being created. + spawnChild *G + spawnParent *G + spawnP *P + + // taskStorage owns the separately allocated scheduler G for a spawned + // goroutine. Static bootstrap Gs leave these fields zero. Target allocators + // must provide scanned/root memory whenever a collector is enabled. + taskStorage unsafe.Pointer + taskSize uintptr + taskState taskStorageState } const ( @@ -137,7 +151,9 @@ func InitG(g *G) bool { if g == nil || g.magic != 0 || preemptLoad(preemptAddress(g)) != preemptDisabled || g.state != GNew || g.frames != nil || g.active != nil || g.root != nil || g.pending.kind != pendingNone || g.pending.from != nil || g.pending.target != nil || g.pending.wait != nil || g.pending.ticket != 0 || g.destroyTarget != nil || g.destroyRoot || g.nextReady != nil || g.queued || - g.waitToken != nil || g.waitTicket != 0 || g.nextWait != nil || g.waiting || g.runP != nil { + g.waitToken != nil || g.waitTicket != 0 || g.nextWait != nil || g.waiting || g.runP != nil || + g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil || + g.taskStorage != nil || g.taskSize != 0 || g.taskState != taskStorageStatic { return false } g.magic = gMagic @@ -153,6 +169,12 @@ func InitG(g *G) bool { // frame-chain transitions are non-atomic and remain confined to the scheduler // thread. InitG enables the gate only after initialization, and terminal root // destruction disables it so a late requester cannot resurrect residual state. +// +// A dynamically allocated G is not a stable asynchronous handle. Compiler +// safepoints and the scheduler may call RequestPreempt while they synchronously +// own that G; platform/event producers must retain the stable P instead and use +// RequestSchedule. This lifetime rule is what makes per-G task reclamation safe +// without a per-request heap reference or epoch protocol. func RequestPreempt(g *G) bool { if g == nil { return false @@ -181,7 +203,7 @@ func PollPreempt(g *G) bool { g.active.owner != g || g.active.handle == nil || g.active.header == nil || g.active.state != FrameActive || g.active.header.G != unsafe.Pointer(g) || g.active.header.SuspendReason != uint16(SuspendNone) || - g.active.header.Lifecycle != uint16(FrameActive) || g.pending.kind != pendingNone { + g.active.header.Lifecycle != uint16(FrameActive) || g.pending.kind != pendingNone || g.spawnChild != nil { return false } requested := preemptCompareAndSwap(preemptAddress(g), preemptRequested, preemptIdle) @@ -244,7 +266,8 @@ func AdoptRoot(g *G, handle unsafe.Pointer) bool { // Enqueue appends a runnable G to p exactly once. func Enqueue(p *P, g *G) bool { if p == nil || !ValidG(g) || g.state != GRunnable || g.queued || g.nextReady != nil || - g.waiting || g.nextWait != nil || g.waitToken != nil || g.waitTicket != 0 || g.runP != nil { + g.waiting || g.nextWait != nil || g.waitToken != nil || g.waitTicket != 0 || g.runP != nil || + g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil { return false } schedule := preemptLoad(&p.schedule) @@ -319,7 +342,8 @@ func validReadyQueue(p *P) bool { var tail *G for g := p.readyHead; g != nil; g = g.nextReady { if !ValidG(g) || g.state != GRunnable || !g.queued || g.waiting || g.nextWait != nil || - g.waitToken != nil || g.waitTicket != 0 || g.runP != nil { + g.waitToken != nil || g.waitTicket != 0 || g.runP != nil || + g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil { return false } tail = g @@ -345,7 +369,8 @@ func validWaitQueue(p *P) bool { var tail *G for g := p.waitHead; g != nil; g = g.nextWait { if !ValidG(g) || g.state != GWaiting || !g.waiting || g.waitToken == nil || g.waitTicket == 0 || - g.queued || g.nextReady != nil || g.runP != nil || !validClaimedWait(g.waitToken, g.waitTicket) { + g.queued || g.nextReady != nil || g.runP != nil || + g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil || !validClaimedWait(g.waitToken, g.waitTicket) { return false } tail = g @@ -506,7 +531,8 @@ func BeginRunG(p *P, g *G) (Action, bool) { if p == nil || p.current != nil || p.inResume || p.action.Kind != ActionInvalid || !ValidG(g) || g.state != GRunnable || g.active == nil || g.root == nil || g.destroyTarget != nil || g.destroyRoot || g.queued || g.nextReady != nil || - g.waitToken != nil || g.waitTicket != 0 || g.nextWait != nil || g.waiting || g.runP != nil { + g.waitToken != nil || g.waitTicket != 0 || g.nextWait != nil || g.waiting || g.runP != nil || + g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil { return Action{}, false } schedule := preemptLoad(&p.schedule) @@ -656,6 +682,19 @@ func Destroyed(p *P, g *G, action Action) (Action, bool) { return setAction(p, ActionCheckResume, g.active.handle) } +// AcknowledgeTerminalSchedule classifies and consumes the one non-corruption +// failure of Destroyed: an asynchronous RequestSchedule won the final +// idle-to-disabled race after the last frame had already been destroyed. The +// runtime adapter may retry the same ActionDestroy without invoking +// llvm.coro.destroy again. Any queue, action, or G-state mismatch fails closed. +func AcknowledgeTerminalSchedule(p *P, g *G, action Action) bool { + return expectedAction(p, g, action, ActionDestroy) && !p.inResume && + g.state == GDispatching && g.destroyTarget == nil && g.destroyRoot && + g.active == nil && g.frames == nil && p.readyHead == nil && p.readyTail == nil && + p.waitHead == nil && p.waitTail == nil && validReadyQueue(p) && validWaitQueue(p) && + preemptCompareAndSwap(&p.schedule, scheduleRequested, scheduleIdle) +} + // TerminalG reports whether a scheduler run completely consumed g and left p // idle. This is a deliberately strict terminal-state check for program // startup: a dead G state alone is insufficient if any frame, transition, @@ -667,5 +706,6 @@ func TerminalG(p *P, g *G) bool { ValidG(g) && preemptLoad(preemptAddress(g)) == preemptDisabled && g.state == GDead && g.root == nil && g.active == nil && g.frames == nil && g.pending.kind == pendingNone && g.pending.from == nil && g.pending.target == nil && g.pending.wait == nil && g.pending.ticket == 0 && g.destroyTarget == nil && !g.destroyRoot && g.nextReady == nil && !g.queued && - g.waitToken == nil && g.waitTicket == 0 && g.nextWait == nil && !g.waiting && g.runP == nil + g.waitToken == nil && g.waitTicket == 0 && g.nextWait == nil && !g.waiting && g.runP == nil && + g.spawnChild == nil && g.spawnParent == nil && g.spawnP == nil && validTerminalTaskStorage(g) } diff --git a/runtime/internal/coro/scheduler_spawn_test.go b/runtime/internal/coro/scheduler_spawn_test.go new file mode 100644 index 0000000000..b5c385f2d4 --- /dev/null +++ b/runtime/internal/coro/scheduler_spawn_test.go @@ -0,0 +1,420 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "runtime" + "testing" + "unsafe" +) + +func newSpawnTestFrame(t *testing.T, g *G, handle unsafe.Pointer, resultSize, resultAlign uintptr) (*testFrame, *FrameDescriptorV1) { + t.Helper() + const ( + size = uintptr(37) + align = uintptr(16) + ) + total, ok := FrameAllocationSize(size, align) + if !ok { + t.Fatal("compute spawn test frame allocation") + } + memory := make([]byte, total) + descriptor := &FrameDescriptorV1{ + Version: 1, + HashLo: 0x0102030405060708, + HashHi: 0x1112131415161718, + ResultSize: resultSize, + ResultAlign: resultAlign, + } + descriptorPointer := unsafe.Pointer(descriptor) + storage, ok := RegisterFrame(g, unsafe.Pointer(&memory[0]), total, size, align, descriptorPointer) + if !ok { + t.Fatal("register spawn test frame") + } + header := &HeaderV1{ + G: unsafe.Pointer(g), + Descriptor: descriptorPointer, + SuspendReason: uint16(SuspendNone), + Lifecycle: uint16(FrameInitialSuspended), + } + if !PublishFrame(g, handle, header, storage) { + t.Fatal("publish spawn test frame") + } + return &testFrame{ + handle: handle, + header: header, + storage: storage, + descriptor: descriptorPointer, + size: size, + align: align, + memory: memory, + }, descriptor +} + +func beginSpawnTestResume(t *testing.T, p *P, task *yieldingTestG) Action { + t.Helper() + action, ok := BeginRunG(p, task.g) + if !ok || action.Kind != ActionCheckResume { + t.Fatalf("begin spawn test G %s = (%+v, %t)", task.name, action, ok) + } + action, ok = Checked(p, task.g, action, false) + if !ok || action.Kind != ActionResume { + t.Fatalf("activate spawn test G %s = (%+v, %t)", task.name, action, ok) + } + task.frame.header.SuspendReason = uint16(SuspendNone) + task.frame.header.Lifecycle = uint16(FrameActive) + return action +} + +func beginSpawnTestChildResume(t *testing.T, p *P, g *G, frame *testFrame) Action { + t.Helper() + action, ok := BeginRunG(p, g) + if !ok || action.Kind != ActionCheckResume { + t.Fatalf("begin spawned child = (%+v, %t)", action, ok) + } + action, ok = Checked(p, g, action, false) + if !ok || action.Kind != ActionResume { + t.Fatalf("activate spawned child = (%+v, %t)", action, ok) + } + frame.header.SuspendReason = uint16(SuspendNone) + frame.header.Lifecycle = uint16(FrameActive) + return action +} + +func yieldSpawnTestG(t *testing.T, p *P, g *G, frame *testFrame, action Action) { + t.Helper() + if !PollPreempt(g) { + t.Fatal("spawn commit did not request parent preemption") + } + frame.header.SuspendReason = uint16(SuspendYield) + frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(g, frame.handle, frame.header) { + t.Fatal("prepare spawned-parent yield") + } + got, ok := Resumed(p, g, action) + if !ok || got.Kind != ActionYield { + t.Fatalf("commit spawned-parent yield = (%+v, %t)", got, ok) + } +} + +func completeSpawnTestG(t *testing.T, p *P, g *G, frame *testFrame, action Action) Action { + t.Helper() + frame.header.SuspendReason = uint16(SuspendFrameComplete) + frame.header.Lifecycle = uint16(FrameFinalSuspended) + if !PrepareComplete(g, frame.handle, frame.header) { + t.Fatal("prepare spawn test completion") + } + action, ok := Resumed(p, g, action) + if !ok || action.Kind != ActionCheckDestroy { + t.Fatalf("spawn test completion = (%+v, %t)", action, ok) + } + action, ok = Checked(p, g, action, true) + if !ok || action.Kind != ActionDestroy { + t.Fatalf("spawn test destroy check = (%+v, %t)", action, ok) + } + releaseTestFrame(t, g, frame) + action, ok = Destroyed(p, g, action) + if !ok || action.Kind != ActionComplete { + t.Fatalf("spawn test destroy commit = (%+v, %t)", action, ok) + } + return action +} + +func TestSpawnBeginRollbackIsExactlyOnce(t *testing.T) { + p := new(P) + parent := newYieldingTestG(t, "rollback-parent") + if !Enqueue(p, parent.g) { + t.Fatal("enqueue rollback parent") + } + if got, ok := NextRunnable(p); !ok || got != parent.g { + t.Fatal("dequeue rollback parent") + } + action := beginSpawnTestResume(t, p, parent) + + child := new(G) + if !CanBeginSpawn(parent.g) || !BeginSpawn(parent.g, child, unsafe.Pointer(child), TaskStorageSize()) { + t.Fatal("begin rollback spawn") + } + if CanBeginSpawn(parent.g) { + t.Fatal("nested spawn begin was not blocked") + } + other := new(G) + if BeginSpawn(parent.g, other, unsafe.Pointer(other), TaskStorageSize()) || ValidG(other) { + t.Fatal("rejected nested spawn initialized another child") + } + if CommitSpawn(parent.g, child, nil) || child.root != nil || child.state != GNew || child.queued || p.readyHead != nil { + t.Fatal("invalid commit partially adopted or queued child") + } + raw, size, ok := RollbackSpawn(parent.g, child) + if !ok || raw != unsafe.Pointer(child) || size != TaskStorageSize() || parent.g.spawnChild != nil || + child.spawnParent != nil || child.spawnP != nil || child.state != GDead || child.taskState != taskStorageReleased { + t.Fatalf("rollback = (%p, %d, %t), child state=%d task=%d", raw, size, ok, child.state, child.taskState) + } + if _, _, ok := RollbackSpawn(parent.g, child); ok { + t.Fatal("spawn transaction rolled back twice") + } + + completeSpawnTestG(t, p, parent.g, parent.frame, action) + if !TerminalG(p, parent.g) { + t.Fatal("rollback test parent did not become terminal") + } + runtime.KeepAlive(parent.frame.memory) + runtime.KeepAlive(child) +} + +func TestSpawnCommitZeroResultAtomicAndTaskReclaim(t *testing.T) { + p := new(P) + parent := newYieldingTestG(t, "commit-parent") + if !Enqueue(p, parent.g) { + t.Fatal("enqueue commit parent") + } + if got, ok := NextRunnable(p); !ok || got != parent.g { + t.Fatal("dequeue commit parent") + } + parentAction := beginSpawnTestResume(t, p, parent) + + child := new(G) + if !BeginSpawn(parent.g, child, unsafe.Pointer(child), TaskStorageSize()) { + t.Fatal("begin committed spawn") + } + handle := unsafe.Pointer(new(byte)) + frame, descriptor := newSpawnTestFrame(t, child, handle, 8, 8) + if CommitSpawn(parent.g, child, handle) { + t.Fatal("non-zero-result goroutine root accepted") + } + if child.root != nil || child.active != nil || child.state != GNew || child.queued || + p.readyHead != nil || parent.g.spawnChild != child || preemptLoad(preemptAddress(parent.g)) != preemptIdle { + t.Fatal("rejected result layout partially committed spawn") + } + descriptor.ResultSize = 0 + descriptor.ResultAlign = 1 + if !CommitSpawn(parent.g, child, handle) { + t.Fatal("commit zero-result goroutine root") + } + if parent.g.spawnChild != nil || child.root == nil || child.active != child.root || child.state != GRunnable || + !child.queued || p.readyHead != child || p.readyTail != child || preemptLoad(preemptAddress(parent.g)) != preemptRequested { + t.Fatal("committed spawn state is incomplete") + } + if CommitSpawn(parent.g, child, handle) || p.readyHead != child || p.readyTail != child || child.nextReady != nil { + t.Fatal("duplicate spawn commit changed the ready queue") + } + + yieldSpawnTestG(t, p, parent.g, parent.frame, parentAction) + if got, ok := NextRunnable(p); !ok || got != child { + t.Fatalf("spawned child was not first after parent yield: (%p, %t)", got, ok) + } + childAction := beginSpawnTestChildResume(t, p, child, frame) + completeSpawnTestG(t, p, child, frame, childAction) + if !ReclaimableG(child) || TerminalG(p, child) { + t.Fatal("per-G reclaimability was confused with P-wide terminal state") + } + owned, ok := TaskStorageOwned(child) + if !ok || !owned { + t.Fatal("completed child did not retain one owned task allocation") + } + raw, size, ok := ReleaseTaskStorage(child) + if !ok || raw != unsafe.Pointer(child) || size != TaskStorageSize() { + t.Fatalf("release child task = (%p, %d, %t)", raw, size, ok) + } + if _, _, ok := ReleaseTaskStorage(child); ok { + t.Fatal("child task allocation released twice") + } + + if got, ok := NextRunnable(p); !ok || got != parent.g { + t.Fatalf("parent was not runnable after child completion: (%p, %t)", got, ok) + } + parentAction = beginSpawnTestResume(t, p, parent) + completeSpawnTestG(t, p, parent.g, parent.frame, parentAction) + if !TerminalG(p, parent.g) || !TerminalG(p, child) { + t.Fatal("spawn/parent completion retained scheduler state") + } + runtime.KeepAlive(parent.frame.memory) + runtime.KeepAlive(frame.memory) + runtime.KeepAlive(descriptor) + runtime.KeepAlive(child) +} + +func TestSpawnReadyQueuePreservesFIFOAndParentFairness(t *testing.T) { + p := new(P) + parent := newYieldingTestG(t, "fair-parent") + competitor := newYieldingTestG(t, "fair-competitor") + if !Enqueue(p, parent.g) || !Enqueue(p, competitor.g) { + t.Fatal("enqueue fairness tasks") + } + if got, ok := NextRunnable(p); !ok || got != parent.g { + t.Fatal("dequeue fairness parent") + } + parentAction := beginSpawnTestResume(t, p, parent) + child := new(G) + if !BeginSpawn(parent.g, child, unsafe.Pointer(child), TaskStorageSize()) { + t.Fatal("begin fairness child") + } + handle := unsafe.Pointer(new(byte)) + frame, descriptor := newSpawnTestFrame(t, child, handle, 0, 1) + if !CommitSpawn(parent.g, child, handle) { + t.Fatal("commit fairness child") + } + yieldSpawnTestG(t, p, parent.g, parent.frame, parentAction) + + wants := []*G{competitor.g, child, parent.g} + for index, want := range wants { + if got := dequeue(p); got != want { + t.Fatalf("ready[%d] = %p, want %p", index, got, want) + } + } + if p.readyHead != nil || p.readyTail != nil { + t.Fatal("fairness queue retained an unexpected task") + } + runtime.KeepAlive(parent.frame.memory) + runtime.KeepAlive(competitor.frame.memory) + runtime.KeepAlive(frame.memory) + runtime.KeepAlive(descriptor) + runtime.KeepAlive(child) +} + +func TestSpawnParkCompletionRacesPeerTerminalUsingStableP(t *testing.T) { + p := new(P) + parent := newYieldingTestG(t, "park-parent") + if !Enqueue(p, parent.g) { + t.Fatal("enqueue park parent") + } + if got, ok := NextRunnable(p); !ok || got != parent.g { + t.Fatal("dequeue park parent") + } + parentAction := beginSpawnTestResume(t, p, parent) + child := new(G) + if !BeginSpawn(parent.g, child, unsafe.Pointer(child), TaskStorageSize()) { + t.Fatal("begin parked child") + } + handle := unsafe.Pointer(new(byte)) + frame, descriptor := newSpawnTestFrame(t, child, handle, 0, 1) + if !CommitSpawn(parent.g, child, handle) { + t.Fatal("commit parked child") + } + yieldSpawnTestG(t, p, parent.g, parent.frame, parentAction) + if got, ok := NextRunnable(p); !ok || got != child { + t.Fatal("dequeue parked child") + } + childAction := beginSpawnTestChildResume(t, p, child, frame) + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm child park token") + } + frame.header.SuspendReason = uint16(SuspendPark) + frame.header.Lifecycle = uint16(FrameSuspended) + if !PreparePark(child, handle, frame.header, token, ticket) { + t.Fatal("prepare child park") + } + if action, ok := Resumed(p, child, childAction); !ok || action.Kind != ActionPark || !HasWaiting(p) { + t.Fatal("commit child park") + } + + if got, ok := NextRunnable(p); !ok || got != parent.g { + t.Fatal("dequeue parent beside parked child") + } + parentAction = beginSpawnTestResume(t, p, parent) + start := make(chan struct{}) + producerDone := make(chan bool, 1) + go func() { + <-start + producerDone <- CompleteWait(token, ticket) && RequestSchedule(p) + }() + close(start) + completeSpawnTestG(t, p, parent.g, parent.frame, parentAction) + if !<-producerDone { + t.Fatal("P-only completion producer was rejected") + } + if !DeadG(parent.g) || TerminalG(p, parent.g) { + t.Fatal("dead peer was mistaken for a P-wide terminal program") + } + if count, ok := PollReady(p); !ok || count != 1 || HasWaiting(p) { + t.Fatalf("promote spawned parked child = (%d, %t), waiting=%t", count, ok, HasWaiting(p)) + } + if got, ok := NextRunnable(p); !ok || got != child { + t.Fatal("dequeue completed parked child") + } + childAction = beginSpawnTestChildResume(t, p, child, frame) + completeSpawnTestG(t, p, child, frame, childAction) + if !ReclaimableG(child) { + t.Fatal("completed parked child is not reclaimable") + } + if _, _, ok := ReleaseTaskStorage(child); !ok { + t.Fatal("release completed parked child task") + } + if !TerminalG(p, parent.g) || !TerminalG(p, child) { + t.Fatal("park/terminal race retained scheduler state") + } + runtime.KeepAlive(parent.frame.memory) + runtime.KeepAlive(frame.memory) + runtime.KeepAlive(descriptor) + runtime.KeepAlive(child) +} + +func TestMainReturnWithParkedSpawnFailsClosed(t *testing.T) { + p := new(P) + main := newYieldingTestG(t, "main") + if !Enqueue(p, main.g) { + t.Fatal("enqueue main") + } + if got, ok := NextRunnable(p); !ok || got != main.g { + t.Fatal("dequeue main") + } + mainAction := beginSpawnTestResume(t, p, main) + child := new(G) + if !BeginSpawn(main.g, child, unsafe.Pointer(child), TaskStorageSize()) { + t.Fatal("begin main child") + } + handle := unsafe.Pointer(new(byte)) + frame, descriptor := newSpawnTestFrame(t, child, handle, 0, 1) + if !CommitSpawn(main.g, child, handle) { + t.Fatal("commit main child") + } + yieldSpawnTestG(t, p, main.g, main.frame, mainAction) + if got, ok := NextRunnable(p); !ok || got != child { + t.Fatal("dequeue main child") + } + childAction := beginSpawnTestChildResume(t, p, child, frame) + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm main child wait") + } + frame.header.SuspendReason = uint16(SuspendPark) + frame.header.Lifecycle = uint16(FrameSuspended) + if !PreparePark(child, handle, frame.header, token, ticket) { + t.Fatal("prepare main child park") + } + if action, ok := Resumed(p, child, childAction); !ok || action.Kind != ActionPark { + t.Fatal("park main child") + } + if got, ok := NextRunnable(p); !ok || got != main.g { + t.Fatal("dequeue main after child park") + } + mainAction = beginSpawnTestResume(t, p, main) + completeSpawnTestG(t, p, main.g, main.frame, mainAction) + if !DeadG(main.g) || TerminalG(p, main.g) || !HasWaiting(p) { + t.Fatal("main return drained or accepted a suspended background G") + } + if got, ok := NextRunnable(p); !ok || got != nil || !HasWaiting(p) { + t.Fatalf("parked background G became runnable after main return: (%p, %t)", got, ok) + } + runtime.KeepAlive(main.frame.memory) + runtime.KeepAlive(frame.memory) + runtime.KeepAlive(descriptor) + runtime.KeepAlive(child) +} diff --git a/runtime/internal/coro/scheduler_wait_test.go b/runtime/internal/coro/scheduler_wait_test.go index 2586162588..2fcc2c21d2 100644 --- a/runtime/internal/coro/scheduler_wait_test.go +++ b/runtime/internal/coro/scheduler_wait_test.go @@ -513,7 +513,7 @@ func TestTerminalDisableLinearizesWithLateScheduleRequest(t *testing.T) { task.g.state != GDispatching || p.current != task.g || p.action != action { t.Fatalf("iteration %d: request won race but terminal partially committed: request=%t gate=%d state=%d", iteration, requestOK, preemptLoad(&p.schedule), task.g.state) } - if !preemptCompareAndSwap(&p.schedule, scheduleRequested, scheduleIdle) { + if !AcknowledgeTerminalSchedule(p, task.g, action) { t.Fatalf("iteration %d: acknowledge winning late request", iteration) } terminalAction, terminalOK = Destroyed(p, task.g, action) diff --git a/runtime/internal/coro/spawn.go b/runtime/internal/coro/spawn.go new file mode 100644 index 0000000000..41862e3047 --- /dev/null +++ b/runtime/internal/coro/spawn.go @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import "unsafe" + +type taskStorageState uint8 + +const ( + // Static bootstrap Gs have no separately owned storage. The zero value is + // deliberately static so InitG can continue to initialize global G objects. + taskStorageStatic taskStorageState = iota + taskStorageOwned + taskStorageReleased +) + +// TaskStorageSize is the exact scanned/root allocation required for one +// independently scheduled G. The G begins at the allocation base so the C ABI +// can pass the returned address directly to a coroutine root factory. +func TaskStorageSize() uintptr { + return unsafe.Sizeof(G{}) +} + +func validLiveTaskStorage(g *G) bool { + if g == nil { + return false + } + switch g.taskState { + case taskStorageStatic: + return g.taskStorage == nil && g.taskSize == 0 + case taskStorageOwned: + return g.taskStorage == unsafe.Pointer(g) && g.taskSize == TaskStorageSize() + default: + return false + } +} + +func validTerminalTaskStorage(g *G) bool { + if g == nil { + return false + } + switch g.taskState { + case taskStorageStatic: + return g.taskStorage == nil && g.taskSize == 0 + case taskStorageOwned: + return g.taskStorage == unsafe.Pointer(g) && g.taskSize == TaskStorageSize() + case taskStorageReleased: + return g.taskStorage == nil && g.taskSize == 0 + default: + return false + } +} + +// runningSpawnContext validates the exact scheduler-stack episode in which a +// closed-static go statement may create a child. No scheduler-owned field may +// be touched from a factory running outside this parent/P resume pair. +func runningSpawnContext(parent *G) (*P, bool) { + if !ValidG(parent) || parent.state != GRunning || parent.root == nil || parent.active == nil || + parent.active.owner != parent || parent.active.handle == nil || parent.active.header == nil || + parent.active.state != FrameActive || parent.active.header.G != unsafe.Pointer(parent) || + parent.active.header.SuspendReason != uint16(SuspendNone) || + parent.active.header.Lifecycle != uint16(FrameActive) || + parent.pending.kind != pendingNone || parent.pending.from != nil || parent.pending.target != nil || + parent.pending.wait != nil || parent.pending.ticket != 0 || + parent.destroyTarget != nil || parent.destroyRoot || parent.queued || parent.nextReady != nil || + parent.waitToken != nil || parent.waitTicket != 0 || parent.nextWait != nil || parent.waiting || + parent.spawnParent != nil || parent.spawnP != nil || !validLiveTaskStorage(parent) { + return nil, false + } + p := parent.runP + if p == nil || p.current != parent || !p.inResume || + !expectedAction(p, parent, p.action, ActionResume) || + !validReadyQueue(p) || !validWaitQueue(p) { + return nil, false + } + schedule := preemptLoad(&p.schedule) + if schedule != scheduleIdle && schedule != scheduleRequested { + return nil, false + } + return p, true +} + +// CanBeginSpawn is a read-only preflight used by the runtime adapter before it +// allocates child task storage. BeginSpawn repeats every check before +// publishing ownership; the preflight is only an allocation fast-fail. +func CanBeginSpawn(parent *G) bool { + _, ok := runningSpawnContext(parent) + return ok && parent.spawnChild == nil +} + +// BeginSpawn publishes one parent-owned creation transaction around an empty, +// separately allocated G. The parent link is the GC root between this call and +// CommitSpawn while the compiler directly creates the child's initial- +// suspended LLVM coroutine frame. +func BeginSpawn(parent, child *G, storage unsafe.Pointer, size uintptr) bool { + p, ok := runningSpawnContext(parent) + if !ok || parent.spawnChild != nil || child == nil || child == parent || + storage != unsafe.Pointer(child) || size != TaskStorageSize() || + uintptr(storage)%unsafe.Alignof(G{}) != 0 { + return false + } + if !InitG(child) { + return false + } + child.taskStorage = storage + child.taskSize = size + child.taskState = taskStorageOwned + child.spawnParent = parent + child.spawnP = p + parent.spawnChild = child + return true +} + +func validZeroResultSpawnRoot(child *G, handle unsafe.Pointer) (*Frame, bool) { + root := findFrame(child, handle) + if root == nil || child.frames != root || root.next != nil || root.owner != child || + root.parent != nil || root.handle != handle || root.header == nil || + root.header.G != unsafe.Pointer(child) || root.header.Parent != nil || + root.header.Descriptor != root.descriptor || root.header.ResultSlot != nil || + root.header.SuspendReason != uint16(SuspendNone) || + root.header.Lifecycle != uint16(FrameInitialSuspended) || + root.state != FrameInitialSuspended || root.descriptor == nil || + !checkedProgramObjectV1(root.descriptor, unsafe.Sizeof(FrameDescriptorV1{}), unsafe.Alignof(FrameDescriptorV1{})) { + return nil, false + } + descriptor := (*FrameDescriptorV1)(root.descriptor) + if descriptor.Version != 1 || descriptor.Flags != 0 || + descriptor.ResultSize != 0 || descriptor.ResultAlign != 1 { + return nil, false + } + return root, true +} + +// CommitSpawn atomically adopts the independently created root and appends its +// G to the current P's ready queue. Every potentially failing check happens +// before RequestPreempt and the scheduler-owned stores, so failure never +// exposes a half-adopted or half-enqueued child. The request forces the parent +// through its next compiler safepoint; yielding then places the parent behind +// the newly ready child. +func CommitSpawn(parent, child *G, handle unsafe.Pointer) bool { + p, ok := runningSpawnContext(parent) + if !ok || handle == nil || parent.spawnChild != child || child == nil || + !ValidG(child) || child.state != GNew || child.root != nil || child.active != nil || + child.pending.kind != pendingNone || child.pending.from != nil || child.pending.target != nil || + child.pending.wait != nil || child.pending.ticket != 0 || + child.destroyTarget != nil || child.destroyRoot || child.nextReady != nil || child.queued || + child.waitToken != nil || child.waitTicket != 0 || child.nextWait != nil || child.waiting || child.runP != nil || + child.spawnChild != nil || child.spawnParent != parent || child.spawnP != p || + child.taskState != taskStorageOwned || child.taskStorage != unsafe.Pointer(child) || + child.taskSize != TaskStorageSize() || preemptLoad(preemptAddress(child)) != preemptIdle { + return false + } + root, ok := validZeroResultSpawnRoot(child, handle) + if !ok || (p.readyHead == nil) != (p.readyTail == nil) || + (p.readyTail != nil && p.readyTail.nextReady != nil) { + return false + } + // This cannot fail after the complete parent/child/P validation above. It is + // intentionally issued before queue publication so no post-publication + // operation can force CommitSpawn to report failure. + if !RequestPreempt(parent) { + return false + } + + child.root = root + child.active = root + child.state = GRunnable + child.spawnParent = nil + child.spawnP = nil + child.queued = true + if p.readyTail == nil { + p.readyHead = child + } else { + p.readyTail.nextReady = child + } + p.readyTail = child + // Clear the temporary root only after P's queue reaches the child. + parent.spawnChild = nil + return true +} + +// RollbackSpawn releases a begin transaction only before any coroutine frame +// has been allocated. Once a factory has published a handle, rejection is +// fail-stop: only the scheduler may destroy that handle, so the exported ABI +// aborts instead of trying to free it on the parent executor stack. +func RollbackSpawn(parent, child *G) (unsafe.Pointer, uintptr, bool) { + p, ok := runningSpawnContext(parent) + if !ok || parent.spawnChild != child || child == nil || !ValidG(child) || + child.spawnParent != parent || child.spawnP != p || child.spawnChild != nil || + child.state != GNew || child.root != nil || child.active != nil || child.frames != nil || + child.pending.kind != pendingNone || child.destroyTarget != nil || child.destroyRoot || + child.nextReady != nil || child.queued || child.waitToken != nil || child.waitTicket != 0 || + child.nextWait != nil || child.waiting || child.runP != nil || + child.taskState != taskStorageOwned || child.taskStorage != unsafe.Pointer(child) || + child.taskSize != TaskStorageSize() || preemptLoad(preemptAddress(child)) != preemptIdle { + return nil, 0, false + } + raw, size := child.taskStorage, child.taskSize + parent.spawnChild = nil + child.spawnParent = nil + child.spawnP = nil + child.taskStorage = nil + child.taskSize = 0 + child.taskState = taskStorageReleased + preemptStore(preemptAddress(child), preemptDisabled) + child.state = GDead + return raw, size, true +} + +// ReclaimableG is the per-G terminal predicate. Unlike TerminalG it does not +// require the whole P to be empty/disabled, so a completed child can retire +// while its parent and peers remain runnable or parked. It deliberately rejects +// taskStorageReleased: exactly one caller may observe a task as reclaimable and +// transfer its allocation. +func ReclaimableG(g *G) bool { + return ValidG(g) && preemptLoad(preemptAddress(g)) == preemptDisabled && g.state == GDead && + g.root == nil && g.active == nil && g.frames == nil && + g.pending.kind == pendingNone && g.pending.from == nil && g.pending.target == nil && + g.pending.wait == nil && g.pending.ticket == 0 && + g.destroyTarget == nil && !g.destroyRoot && g.nextReady == nil && !g.queued && + g.waitToken == nil && g.waitTicket == 0 && g.nextWait == nil && !g.waiting && g.runP == nil && + g.spawnChild == nil && g.spawnParent == nil && g.spawnP == nil && validLiveTaskStorage(g) +} + +// TaskStorageOwned reports the only two legal storage states at ActionComplete. +// A released value is rejected so the runtime cannot silently free one task +// allocation twice. +func TaskStorageOwned(g *G) (owned bool, ok bool) { + if !ReclaimableG(g) { + return false, false + } + switch g.taskState { + case taskStorageStatic: + return false, g.taskStorage == nil && g.taskSize == 0 + case taskStorageOwned: + return true, g.taskStorage == unsafe.Pointer(g) && g.taskSize == TaskStorageSize() + default: + return false, false + } +} + +// ReleaseTaskStorage transfers one terminal spawned G allocation back to the +// runtime adapter. It marks the transfer before returning; the caller must not +// dereference g after clearing/freeing raw. External completion producers own +// only stable P/WaitToken objects and must never retain a child G pointer. +func ReleaseTaskStorage(g *G) (raw unsafe.Pointer, size uintptr, ok bool) { + owned, valid := TaskStorageOwned(g) + if !valid || !owned { + return nil, 0, false + } + raw, size = g.taskStorage, g.taskSize + g.taskStorage = nil + g.taskSize = 0 + g.taskState = taskStorageReleased + return raw, size, true +} + +// DeadG is a narrow program-driver query. It does not imply that a command +// main may safely return: TerminalG must still prove that no ready or parked G +// survives. +func DeadG(g *G) bool { + return ValidG(g) && g.state == GDead +} diff --git a/runtime/internal/coroalloc/allocator.go b/runtime/internal/coroalloc/allocator.go index 5f2a903f2f..3599f73f97 100644 --- a/runtime/internal/coroalloc/allocator.go +++ b/runtime/internal/coroalloc/allocator.go @@ -103,3 +103,26 @@ func FreeFrame(ptr unsafe.Pointer) bool { backendFreeFrame(ptr) return true } + +// AllocTask allocates pointer-containing scheduler task storage. It uses the +// same statically selected scanned/root backend as coroutine frames: BDWGC's +// uncollectable allocation is conservatively scanned, tinygogc sees the task +// through the scheduler's static P/parent links, and nogc/WASM profiles have +// no tracing collector that an ordinary malloc range could hide pointers from. +func AllocTask(size uintptr) unsafe.Pointer { + if !Ready() || size == 0 { + return nil + } + return backendAllocFrame(size) +} + +// FreeTask performs the physical half of the scheduler's exactly-once task +// retirement protocol. The caller must first unlink and logically release the +// G through coro.ReleaseTaskStorage. +func FreeTask(ptr unsafe.Pointer) bool { + if !Ready() || ptr == nil { + return false + } + backendFreeFrame(ptr) + return true +} diff --git a/runtime/internal/runtime/coro_program.go b/runtime/internal/runtime/coro_program.go index b17d2e7cbc..db6db94859 100644 --- a/runtime/internal/runtime/coro_program.go +++ b/runtime/internal/runtime/coro_program.go @@ -96,7 +96,7 @@ func coroProgramRunV1(gPointer, handle unsafe.Pointer) bool { return false } coroProgramLifecycleV1State = coroProgramRunningV1 - if !coroRun(&coroProgramPV1State) || !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { + if !coroRun(&coroProgramPV1State, &coroProgramGV1State) || !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { coroProgramLifecycleV1State = coroProgramFailedV1 return false } diff --git a/runtime/internal/runtime/coro_program_test.go b/runtime/internal/runtime/coro_program_test.go index bf9a3a8c53..18a63bbcda 100644 --- a/runtime/internal/runtime/coro_program_test.go +++ b/runtime/internal/runtime/coro_program_test.go @@ -188,13 +188,14 @@ func newCoroProgramTestFrameV1(t *testing.T, g *coro.G) *coroProgramTestFrameV1 } type coroProgramTestDriverV1 struct { - t *testing.T - frame *coroProgramTestFrameV1 - doneCalls int - resumeCalls int - destroyCalls int - completeReady bool - released bool + t *testing.T + frame *coroProgramTestFrameV1 + doneCalls int + resumeCalls int + destroyCalls int + completeReady bool + released bool + requestScheduleOnDestroy bool } var activeCoroProgramDriver *coroProgramTestDriverV1 @@ -215,6 +216,15 @@ func coroRuntimeAbort(message string) { panic(message) } +// The named-source adapter test exercises only the static bootstrap G and does +// not link the target allocator backend. Keep the ActionComplete ownership +// check real while avoiding a reference to the production physical free hook. +// Spawn/task-storage tests live in runtime/internal/coro. +func coroReleaseCompletedTask(g *coroG) bool { + owned, ok := coro.TaskStorageOwned(g) + return ok && !owned +} + func (driver *coroProgramTestDriverV1) requireHandle(handle unsafe.Pointer) { if driver == nil { panic("coroutine test wrapper called without an active driver") @@ -263,6 +273,9 @@ func (driver *coroProgramTestDriverV1) destroy(handle unsafe.Pointer) { driver.t.Fatalf("release simulated coroutine frame = (%p, %d, %t), want (%p, %d, true)", raw, total, ok, frame.raw, frame.total) } driver.released = true + if driver.requestScheduleOnDestroy && !coro.RequestSchedule(&coroProgramPV1State) { + driver.t.Fatal("request terminal schedule retry") + } } func resetCoroProgramTestStateV1(t *testing.T) { @@ -344,6 +357,34 @@ func TestCoroProgramV2BeginRunAndDestroy(t *testing.T) { runtime.KeepAlive(manifest) } +func TestCoroProgramTerminalScheduleRetryDoesNotRedestroy(t *testing.T) { + resetCoroProgramTestStateV1(t) + manifest := newCoroProgramTestManifestV1() + factory := unsafe.Pointer(&manifest.factoryMarker) + + gPointer, ok := coroProgramBeginV1(unsafe.Pointer(&manifest.manifest), factory) + if !ok { + t.Fatal("begin terminal-retry coroutine program") + } + frame := newCoroProgramTestFrameV1(t, &coroProgramGV1State) + driver := &coroProgramTestDriverV1{ + t: t, + frame: frame, + requestScheduleOnDestroy: true, + } + activeCoroProgramDriver = driver + if !coroProgramRunV1(gPointer, frame.handle) { + t.Fatal("terminal schedule request was treated as corruption") + } + if driver.destroyCalls != 1 || !driver.released || + coroProgramLifecycleV1State != coroProgramCompleteV1 || + !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { + t.Fatalf("terminal retry = destroys:%d released:%t lifecycle:%d", driver.destroyCalls, driver.released, coroProgramLifecycleV1State) + } + runtime.KeepAlive(frame.memory) + runtime.KeepAlive(manifest) +} + func TestCoroProgramV1BeginFailsClosedOnFactoryIdentity(t *testing.T) { resetCoroProgramTestStateV1(t) manifest := newCoroProgramTestManifestV1() diff --git a/runtime/internal/runtime/coro_sched.go b/runtime/internal/runtime/coro_sched.go index 11573919dc..057cc198ca 100644 --- a/runtime/internal/runtime/coro_sched.go +++ b/runtime/internal/runtime/coro_sched.go @@ -59,7 +59,7 @@ func coroRunG(p *coroP, g *coroG) bool { return coroRunActions(p, g, action) } -func coroRun(p *coroP) bool { +func coroRun(p *coroP, main *coroG) bool { for { g, ok := coro.NextRunnable(p) if !ok { @@ -73,6 +73,12 @@ func coroRun(p *coroP) bool { if !coroRunG(p, g) { return false } + if g == main && coro.DeadG(main) { + // Command main must not drain background goroutines after returning. + // Until the runtime can cancel every ready/suspended child safely, only + // a fully terminal P is a supported main-return state. + return coro.TerminalG(p, main) + } } } @@ -83,7 +89,9 @@ func coroRunActions(p *coroP, g *coroG, action coro.Action) bool { for { var ok bool switch action.Kind { - case coro.ActionComplete, coro.ActionYield, coro.ActionPark: + case coro.ActionComplete: + return coroReleaseCompletedTask(g) + case coro.ActionYield, coro.ActionPark: return true case coro.ActionCheckResume, coro.ActionCheckDestroy: action, ok = coro.Checked(p, g, action, coroHandleDone(action.Handle)) @@ -92,7 +100,19 @@ func coroRunActions(p *coroP, g *coroG, action coro.Action) bool { action, ok = coro.Resumed(p, g, action) case coro.ActionDestroy: coroHandleDestroy(action.Handle) - action, ok = coro.Destroyed(p, g, action) + for { + next, committed := coro.Destroyed(p, g, action) + if committed { + action, ok = next, true + break + } + if !coro.AcknowledgeTerminalSchedule(p, g, action) { + ok = false + break + } + // Retry only the scheduler commit. The LLVM handle was already + // destroyed exactly once before entering this loop. + } default: return false } diff --git a/runtime/internal/runtime/coro_spawn.go b/runtime/internal/runtime/coro_spawn.go new file mode 100644 index 0000000000..caafb247d7 --- /dev/null +++ b/runtime/internal/runtime/coro_spawn.go @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/internal/coro" + "github.com/goplus/llgo/runtime/internal/coroalloc" +) + +// Safe cancellation of every ready/suspended background G at command-main +// return is not implemented yet. Keep the production C ABI present for +// compiler/link validation, but fail closed before allocating a child. Core and +// adapter tests call the unexported begin/commit functions to exercise the +// complete scheduler transaction without claiming production Go semantics. +const coroSpawnProductionEnabledV1 = false + +func coroSpawnBeginV1(parentPointer unsafe.Pointer) (unsafe.Pointer, bool) { + parent := (*coroG)(parentPointer) + if !coro.CanBeginSpawn(parent) || !coroalloc.Ready() { + return nil, false + } + size := coro.TaskStorageSize() + raw := coroalloc.AllocTask(size) + if raw == nil { + return nil, false + } + coro.Zero(raw, size) + child := (*coroG)(raw) + if !coro.BeginSpawn(parent, child, raw, size) { + coro.Zero(raw, size) + if !coroalloc.FreeTask(raw) { + return nil, false + } + return nil, false + } + return raw, true +} + +func coroSpawnCommitV1(parentPointer, childPointer, handle unsafe.Pointer) bool { + return coro.CommitSpawn((*coroG)(parentPointer), (*coroG)(childPointer), handle) +} + +// coroReleaseCompletedTask performs the physical half of spawned-G +// retirement. A platform producer may retain only P/WaitToken state, never a +// child G pointer, so disabling the G gate and unlinking it from P is the +// quiescence boundary for this allocation. +func coroReleaseCompletedTask(g *coroG) bool { + owned, ok := coro.TaskStorageOwned(g) + if !ok { + return false + } + if !owned { + return true + } + raw, size, ok := coro.ReleaseTaskStorage(g) + if !ok { + return false + } + coro.Zero(raw, size) + return coroalloc.FreeTask(raw) +} + +//export __llgo_coro_spawn_begin_v1 +func __llgo_coro_spawn_begin_v1(parent unsafe.Pointer) unsafe.Pointer { + if !coroSpawnProductionEnabledV1 { + coroRuntimeAbort("coroutine goroutine spawn is not production-enabled") + return nil + } + child, ok := coroSpawnBeginV1(parent) + if !ok { + coroRuntimeAbort("invalid coroutine goroutine spawn begin") + return nil + } + return child +} + +//export __llgo_coro_spawn_commit_v1 +func __llgo_coro_spawn_commit_v1(parent, child, handle unsafe.Pointer) { + if !coroSpawnProductionEnabledV1 { + coroRuntimeAbort("coroutine goroutine spawn is not production-enabled") + return + } + if !coroSpawnCommitV1(parent, child, handle) { + // A published LLVM handle is never destroyed here. Scheduler ownership is + // exclusive; malformed commit is therefore a terminal ABI violation. + coroRuntimeAbort("invalid coroutine goroutine spawn commit") + } +} From 9335f0b827e89f09845478444422d098fe2ecbb9 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 23:27:33 +0800 Subject: [PATCH 20/32] runtime(coro): cancel ready Gs on command main return --- runtime/internal/coro/frame_test.go | 1 + runtime/internal/coro/scheduler.go | 11 +- .../internal/coro/scheduler_shutdown_test.go | 415 ++++++++++++++++++ runtime/internal/coro/scheduler_wait_test.go | 2 +- runtime/internal/coro/shutdown.go | 241 ++++++++++ runtime/internal/runtime/coro_program.go | 49 ++- runtime/internal/runtime/coro_program_test.go | 82 +++- runtime/internal/runtime/coro_sched.go | 51 ++- runtime/internal/runtime/coro_spawn.go | 10 +- 9 files changed, 847 insertions(+), 15 deletions(-) create mode 100644 runtime/internal/coro/scheduler_shutdown_test.go create mode 100644 runtime/internal/coro/shutdown.go diff --git a/runtime/internal/coro/frame_test.go b/runtime/internal/coro/frame_test.go index eefcdb2870..c3770e1fc1 100644 --- a/runtime/internal/coro/frame_test.go +++ b/runtime/internal/coro/frame_test.go @@ -282,6 +282,7 @@ func TestTerminalGRejectsResidualSchedulerState(t *testing.T) { {"wait tail", func(p *P) { p.waitTail = dummyG }}, {"schedule idle", func(p *P) { preemptStore(&p.schedule, scheduleIdle) }}, {"schedule requested", func(p *P) { preemptStore(&p.schedule, scheduleRequested) }}, + {"schedule stopping", func(p *P) { preemptStore(&p.schedule, scheduleStopping) }}, {"in resume", func(p *P) { p.inResume = true }}, {"action kind", func(p *P) { p.action.Kind = ActionResume }}, {"action handle", func(p *P) { p.action.Handle = dummyActionHandle }}, diff --git a/runtime/internal/coro/scheduler.go b/runtime/internal/coro/scheduler.go index fd15f54267..6b8829411d 100644 --- a/runtime/internal/coro/scheduler.go +++ b/runtime/internal/coro/scheduler.go @@ -26,6 +26,7 @@ const ( GRunnable GRunning GDispatching + GCanceling GWaiting GDead ) @@ -78,6 +79,7 @@ const ( const ( scheduleIdle uint32 = iota scheduleRequested + scheduleStopping scheduleDisabled ) @@ -122,6 +124,13 @@ const ( // ticket has been linked into P's wait set. A platform event source may now // complete the ticket; only PollReady/NextRunnable can enqueue its G again. ActionPark + // ActionCancelDestroy asks the runtime shutdown adapter to call + // llvm.coro.destroy directly on one suspended frame. It never performs a + // coro.done check and never resumes an ancestor frame. + ActionCancelDestroy + // ActionCancelComplete transfers one fully destroyed spawned G to the task + // storage reclaimer. + ActionCancelComplete ) // Action is one deterministic scheduler operation or control event. Handle is @@ -133,7 +142,7 @@ type Action struct { } func setAction(p *P, kind ActionKind, handle unsafe.Pointer) (Action, bool) { - if p == nil || kind == ActionInvalid || kind == ActionComplete || kind == ActionYield || kind == ActionPark || handle == nil { + if p == nil || kind == ActionInvalid || kind == ActionComplete || kind == ActionYield || kind == ActionPark || kind == ActionCancelComplete || handle == nil { return Action{}, false } action := Action{Kind: kind, Handle: handle} diff --git a/runtime/internal/coro/scheduler_shutdown_test.go b/runtime/internal/coro/scheduler_shutdown_test.go new file mode 100644 index 0000000000..c39bd505d6 --- /dev/null +++ b/runtime/internal/coro/scheduler_shutdown_test.go @@ -0,0 +1,415 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "runtime" + "testing" + "unsafe" +) + +type commandShutdownChild struct { + g *G + handle unsafe.Pointer + frame *testFrame + descriptor *FrameDescriptorV1 +} + +type commandShutdownFixture struct { + p *P + main *yieldingTestG + mainAction Action + children []*commandShutdownChild +} + +func newCommandShutdownFixture(t *testing.T) *commandShutdownFixture { + t.Helper() + p := new(P) + main := newYieldingTestG(t, "command-main") + if !Enqueue(p, main.g) { + t.Fatal("enqueue command main") + } + if got, ok := NextRunnable(p); !ok || got != main.g { + t.Fatal("dequeue command main") + } + return &commandShutdownFixture{p: p, main: main, mainAction: beginSpawnTestResume(t, p, main)} +} + +func (fixture *commandShutdownFixture) spawn(t *testing.T) *commandShutdownChild { + t.Helper() + child := &commandShutdownChild{g: new(G), handle: unsafe.Pointer(new(byte))} + if !BeginSpawn(fixture.main.g, child.g, unsafe.Pointer(child.g), TaskStorageSize()) { + t.Fatal("begin command child") + } + child.frame, child.descriptor = newSpawnTestFrame(t, child.g, child.handle, 0, 1) + if !CommitSpawn(fixture.main.g, child.g, child.handle) { + t.Fatal("commit command child") + } + fixture.children = append(fixture.children, child) + return child +} + +func (fixture *commandShutdownFixture) completeMain(t *testing.T) { + t.Helper() + completeSpawnTestG(t, fixture.p, fixture.main.g, fixture.main.frame, fixture.mainAction) + if !ReclaimableG(fixture.main.g) { + t.Fatal("completed command main is not reclaimable") + } +} + +func cancelOneCommandChild(t *testing.T, p *P, want *commandShutdownChild) { + t.Helper() + g, action, ok := NextCommandCancel(p) + if !ok || g != want.g || action.Kind != ActionCancelDestroy || action.Handle != want.handle { + t.Fatalf("next command cancel = (g=%p action=%+v ok=%t), want %p/%p", g, action, ok, want.g, want.handle) + } + if _, ok := CancelDestroyed(p, g, action); ok || p.current != g || p.action != action || g.destroyTarget == nil { + t.Fatal("cancel commit ran before the destroy/free callback") + } + releaseTestFrame(t, g, want.frame) + action, ok = CancelDestroyed(p, g, action) + if !ok || action.Kind != ActionCancelComplete || action.Handle != nil || !ReclaimableG(g) { + t.Fatalf("complete command cancel = (%+v, %t), reclaimable=%t", action, ok, ReclaimableG(g)) + } + if _, ok := CancelDestroyed(p, g, Action{Kind: ActionCancelDestroy, Handle: want.handle}); ok { + t.Fatal("completed command cancellation committed twice") + } + if _, _, ok := ReleaseTaskStorage(g); !ok { + t.Fatal("release canceled command task") + } + if _, _, ok := ReleaseTaskStorage(g); ok { + t.Fatal("release canceled command task twice") + } +} + +func keepCommandShutdownFixtureAlive(fixture *commandShutdownFixture) { + runtime.KeepAlive(fixture.main.frame.memory) + for _, child := range fixture.children { + runtime.KeepAlive(child.frame.memory) + runtime.KeepAlive(child.descriptor) + runtime.KeepAlive(child.g) + } +} + +func TestCommandShutdownCancelsInitialSuspendedChild(t *testing.T) { + fixture := newCommandShutdownFixture(t) + child := fixture.spawn(t) + fixture.completeMain(t) + if !BeginCommandShutdown(fixture.p, fixture.main.g) || preemptLoad(&fixture.p.schedule) != scheduleStopping { + t.Fatal("begin command shutdown") + } + if RequestSchedule(fixture.p) { + t.Fatal("late schedule request entered stopping P") + } + cancelOneCommandChild(t, fixture.p, child) + if g, action, ok := NextCommandCancel(fixture.p); !ok || g != nil || action.Kind != ActionInvalid { + t.Fatalf("empty command cancel queue = (%p, %+v, %t)", g, action, ok) + } + if !FinishCommandShutdown(fixture.p, fixture.main.g) || + !TerminalG(fixture.p, fixture.main.g) || !TerminalG(fixture.p, child.g) { + t.Fatal("finish initial-suspended command shutdown") + } + if FinishCommandShutdown(fixture.p, fixture.main.g) { + t.Fatal("command shutdown finished twice") + } + keepCommandShutdownFixtureAlive(fixture) +} + +func TestCommandShutdownDestroysNestedInitialFrameBeforeRoot(t *testing.T) { + fixture := newCommandShutdownFixture(t) + child := fixture.spawn(t) + nestedHandle := unsafe.Pointer(new(byte)) + nested := newTestFrame(t, child.g, nestedHandle, child.handle) + rootFrame := FrameFromStorage(child.frame.storage) + nestedFrame := FrameFromStorage(nested.storage) + rootFrame.state = FrameSuspended + rootFrame.header.SuspendReason = uint16(SuspendCall) + rootFrame.header.Lifecycle = uint16(FrameSuspended) + nestedFrame.parent = rootFrame + child.g.active = nestedFrame + + fixture.completeMain(t) + if !BeginCommandShutdown(fixture.p, fixture.main.g) { + t.Fatal("begin nested-initial shutdown") + } + g, action, ok := NextCommandCancel(fixture.p) + if !ok || g != child.g || action.Kind != ActionCancelDestroy || action.Handle != nestedHandle { + t.Fatalf("nested initial first action = (g=%p action=%+v ok=%t)", g, action, ok) + } + releaseTestFrame(t, child.g, nested) + action, ok = CancelDestroyed(fixture.p, child.g, action) + if !ok || action.Kind != ActionCancelDestroy || action.Handle != child.handle { + t.Fatalf("nested initial root action = (%+v, %t)", action, ok) + } + releaseTestFrame(t, child.g, child.frame) + action, ok = CancelDestroyed(fixture.p, child.g, action) + if !ok || action.Kind != ActionCancelComplete { + t.Fatalf("nested initial completion = (%+v, %t)", action, ok) + } + if _, _, ok := ReleaseTaskStorage(child.g); !ok || !FinishCommandShutdown(fixture.p, fixture.main.g) { + t.Fatal("release/finish nested-initial shutdown") + } + runtime.KeepAlive(nested.memory) + keepCommandShutdownFixtureAlive(fixture) +} + +func TestCommandShutdownCancelsYieldedChild(t *testing.T) { + fixture := newCommandShutdownFixture(t) + child := fixture.spawn(t) + yieldSpawnTestG(t, fixture.p, fixture.main.g, fixture.main.frame, fixture.mainAction) + if got, ok := NextRunnable(fixture.p); !ok || got != child.g { + t.Fatal("dequeue child before yield") + } + childAction := beginSpawnTestChildResume(t, fixture.p, child.g, child.frame) + if !PollPreempt(child.g) { + t.Fatal("ready main did not preempt child") + } + child.frame.header.SuspendReason = uint16(SuspendYield) + child.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(child.g, child.handle, child.frame.header) { + t.Fatal("prepare child yield") + } + if action, ok := Resumed(fixture.p, child.g, childAction); !ok || action.Kind != ActionYield { + t.Fatal("commit child yield") + } + if got, ok := NextRunnable(fixture.p); !ok || got != fixture.main.g { + t.Fatal("dequeue main after child yield") + } + fixture.mainAction = beginSpawnTestResume(t, fixture.p, fixture.main) + fixture.completeMain(t) + if !BeginCommandShutdown(fixture.p, fixture.main.g) { + t.Fatal("begin yielded-child shutdown") + } + cancelOneCommandChild(t, fixture.p, child) + if !FinishCommandShutdown(fixture.p, fixture.main.g) { + t.Fatal("finish yielded-child shutdown") + } + keepCommandShutdownFixtureAlive(fixture) +} + +func TestCommandShutdownDestroysStructuredChainDeepestToRoot(t *testing.T) { + fixture := newCommandShutdownFixture(t) + child := fixture.spawn(t) + yieldSpawnTestG(t, fixture.p, fixture.main.g, fixture.main.frame, fixture.mainAction) + if got, ok := NextRunnable(fixture.p); !ok || got != child.g { + t.Fatal("dequeue structured child") + } + action := beginSpawnTestChildResume(t, fixture.p, child.g, child.frame) + + midHandle := unsafe.Pointer(new(byte)) + mid := newTestFrame(t, child.g, midHandle, child.handle) + child.frame.header.SuspendReason = uint16(SuspendCall) + child.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareAwait(child.g, child.handle, midHandle) { + t.Fatal("prepare root-to-mid await") + } + action, ok := Resumed(fixture.p, child.g, action) + if !ok || action.Kind != ActionCheckResume || action.Handle != midHandle { + t.Fatal("dispatch mid frame") + } + action, ok = Checked(fixture.p, child.g, action, false) + if !ok || action.Kind != ActionResume { + t.Fatal("activate mid frame") + } + mid.header.SuspendReason = uint16(SuspendNone) + mid.header.Lifecycle = uint16(FrameActive) + + leafHandle := unsafe.Pointer(new(byte)) + leaf := newTestFrame(t, child.g, leafHandle, midHandle) + mid.header.SuspendReason = uint16(SuspendCall) + mid.header.Lifecycle = uint16(FrameSuspended) + if !PrepareAwait(child.g, midHandle, leafHandle) { + t.Fatal("prepare mid-to-leaf await") + } + action, ok = Resumed(fixture.p, child.g, action) + if !ok || action.Kind != ActionCheckResume || action.Handle != leafHandle { + t.Fatal("dispatch leaf frame") + } + action, ok = Checked(fixture.p, child.g, action, false) + if !ok || action.Kind != ActionResume { + t.Fatal("activate leaf frame") + } + leaf.header.SuspendReason = uint16(SuspendNone) + leaf.header.Lifecycle = uint16(FrameActive) + if !PollPreempt(child.g) { + t.Fatal("structured leaf missed parent competitor preemption") + } + leaf.header.SuspendReason = uint16(SuspendYield) + leaf.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(child.g, leafHandle, leaf.header) { + t.Fatal("prepare structured leaf yield") + } + if action, ok = Resumed(fixture.p, child.g, action); !ok || action.Kind != ActionYield { + t.Fatal("commit structured leaf yield") + } + + if got, ok := NextRunnable(fixture.p); !ok || got != fixture.main.g { + t.Fatal("dequeue main beside structured child") + } + fixture.mainAction = beginSpawnTestResume(t, fixture.p, fixture.main) + fixture.completeMain(t) + if !BeginCommandShutdown(fixture.p, fixture.main.g) { + t.Fatal("begin structured shutdown") + } + g, action, ok := NextCommandCancel(fixture.p) + if !ok || g != child.g { + t.Fatal("select structured child for cancellation") + } + wants := []struct { + handle unsafe.Pointer + frame *testFrame + }{{leafHandle, leaf}, {midHandle, mid}, {child.handle, child.frame}} + for index, want := range wants { + if action.Kind != ActionCancelDestroy || action.Handle != want.handle { + t.Fatalf("destroy[%d] = %+v, want handle %p", index, action, want.handle) + } + releaseTestFrame(t, child.g, want.frame) + action, ok = CancelDestroyed(fixture.p, child.g, action) + if !ok { + t.Fatalf("commit destroy[%d]", index) + } + } + if action.Kind != ActionCancelComplete || !ReclaimableG(child.g) { + t.Fatal("structured child did not reach cancel-complete") + } + if _, _, ok := ReleaseTaskStorage(child.g); !ok || !FinishCommandShutdown(fixture.p, fixture.main.g) { + t.Fatal("release/finish structured shutdown") + } + runtime.KeepAlive(mid.memory) + runtime.KeepAlive(leaf.memory) + keepCommandShutdownFixtureAlive(fixture) +} + +func TestCommandShutdownCancelsMultipleChildrenFIFO(t *testing.T) { + fixture := newCommandShutdownFixture(t) + a := fixture.spawn(t) + b := fixture.spawn(t) + c := fixture.spawn(t) + fixture.completeMain(t) + if !BeginCommandShutdown(fixture.p, fixture.main.g) { + t.Fatal("begin multi-child shutdown") + } + for _, child := range []*commandShutdownChild{a, b, c} { + cancelOneCommandChild(t, fixture.p, child) + } + if !FinishCommandShutdown(fixture.p, fixture.main.g) { + t.Fatal("finish multi-child shutdown") + } + keepCommandShutdownFixtureAlive(fixture) +} + +func TestCommandShutdownRejectsWaitWithoutPartialMutation(t *testing.T) { + fixture := newCommandShutdownFixture(t) + child := fixture.spawn(t) + yieldSpawnTestG(t, fixture.p, fixture.main.g, fixture.main.frame, fixture.mainAction) + if got, ok := NextRunnable(fixture.p); !ok || got != child.g { + t.Fatal("dequeue child before park") + } + action := beginSpawnTestChildResume(t, fixture.p, child.g, child.frame) + token := new(WaitToken) + ticket, ok := ArmWait(token) + if !ok { + t.Fatal("arm shutdown-rejection wait") + } + child.frame.header.SuspendReason = uint16(SuspendPark) + child.frame.header.Lifecycle = uint16(FrameSuspended) + if !PreparePark(child.g, child.handle, child.frame.header, token, ticket) { + t.Fatal("prepare shutdown-rejection park") + } + if action, ok = Resumed(fixture.p, child.g, action); !ok || action.Kind != ActionPark { + t.Fatal("commit shutdown-rejection park") + } + if got, ok := NextRunnable(fixture.p); !ok || got != fixture.main.g { + t.Fatal("dequeue main beside parked child") + } + fixture.mainAction = beginSpawnTestResume(t, fixture.p, fixture.main) + fixture.completeMain(t) + beforeSchedule := preemptLoad(&fixture.p.schedule) + beforeWord := preemptLoad(&token.word) + beforeHead := fixture.p.waitHead + if BeginCommandShutdown(fixture.p, fixture.main.g) { + t.Fatal("shutdown accepted parked child") + } + if preemptLoad(&fixture.p.schedule) != beforeSchedule || preemptLoad(&token.word) != beforeWord || + fixture.p.waitHead != beforeHead || fixture.p.waitTail != child.g || child.g.state != GWaiting || + child.g.destroyTarget != nil || child.g.frames == nil { + t.Fatal("rejected wait shutdown partially mutated scheduler state") + } + keepCommandShutdownFixtureAlive(fixture) +} + +func TestCommandShutdownAcceptsIdleOrRequestedGateAndRejectsBusyP(t *testing.T) { + for _, requested := range []bool{false, true} { + t.Run(map[bool]string{false: "idle", true: "requested"}[requested], func(t *testing.T) { + fixture := newCommandShutdownFixture(t) + fixture.spawn(t) + fixture.completeMain(t) + if requested && !RequestSchedule(fixture.p) { + t.Fatal("request schedule before shutdown") + } + if !BeginCommandShutdown(fixture.p, fixture.main.g) || preemptLoad(&fixture.p.schedule) != scheduleStopping { + t.Fatal("begin shutdown from supported gate") + } + if RequestSchedule(fixture.p) { + t.Fatal("stopping gate accepted schedule request") + } + keepCommandShutdownFixtureAlive(fixture) + }) + } + + tests := []struct { + name string + mutate func(*P) + }{ + {"current", func(p *P) { p.current = new(G) }}, + {"in-resume", func(p *P) { p.inResume = true }}, + {"action", func(p *P) { p.action = Action{Kind: ActionResume, Handle: unsafe.Pointer(new(byte))} }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixture := newCommandShutdownFixture(t) + child := fixture.spawn(t) + fixture.completeMain(t) + test.mutate(fixture.p) + beforeHead, beforeTail := fixture.p.readyHead, fixture.p.readyTail + if BeginCommandShutdown(fixture.p, fixture.main.g) || preemptLoad(&fixture.p.schedule) == scheduleStopping || + fixture.p.readyHead != beforeHead || fixture.p.readyTail != beforeTail || child.g.destroyTarget != nil { + t.Fatal("busy-P shutdown did not fail before mutation") + } + keepCommandShutdownFixtureAlive(fixture) + }) + } +} + +func TestCommandShutdownLinearizesWithScheduleRequester(t *testing.T) { + for iteration := 0; iteration < 250; iteration++ { + fixture := newCommandShutdownFixture(t) + fixture.spawn(t) + fixture.completeMain(t) + start := make(chan struct{}) + result := make(chan bool, 1) + go func() { + <-start + result <- RequestSchedule(fixture.p) + }() + close(start) + if !BeginCommandShutdown(fixture.p, fixture.main.g) { + t.Fatalf("iteration %d: begin racing shutdown", iteration) + } + _ = <-result // true linearized before stopping; false linearized after it. + if preemptLoad(&fixture.p.schedule) != scheduleStopping || RequestSchedule(fixture.p) { + t.Fatalf("iteration %d: stopping gate reopened", iteration) + } + keepCommandShutdownFixtureAlive(fixture) + } +} diff --git a/runtime/internal/coro/scheduler_wait_test.go b/runtime/internal/coro/scheduler_wait_test.go index 2fcc2c21d2..25357f7626 100644 --- a/runtime/internal/coro/scheduler_wait_test.go +++ b/runtime/internal/coro/scheduler_wait_test.go @@ -471,7 +471,7 @@ func TestRequestScheduleConcurrentCoalescing(t *testing.T) { if count, ok := PollReady(p); !ok || count != 0 || preemptLoad(&p.schedule) != scheduleIdle { t.Fatalf("idle schedule acknowledgement = (%d, %t), gate=%d", count, ok, preemptLoad(&p.schedule)) } - preemptStore(&p.schedule, scheduleRequested+1) + preemptStore(&p.schedule, scheduleDisabled+1) if RequestSchedule(p) { t.Fatal("corrupt schedule gate accepted") } diff --git a/runtime/internal/coro/shutdown.go b/runtime/internal/coro/shutdown.go new file mode 100644 index 0000000000..220bc41851 --- /dev/null +++ b/runtime/internal/coro/shutdown.go @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import "unsafe" + +// CommandMainReturnPoint validates the scheduler episode in which the +// compiler's normal-main continuation may publish the main-return marker. It +// never mutates queues or frames while llvm.coro.resume is active. +func CommandMainReturnPoint(p *P, main *G) bool { + current, ok := runningSpawnContext(main) + return ok && current == p && main.spawnChild == nil +} + +func validCancelFrame(frame *Frame, g *G) bool { + return frame != nil && frame.owner == g && frame.handle != nil && frame.header != nil && + frame.storage != nil && frame.rawBase != nil && frame.descriptor != nil && + frame.header.G == unsafe.Pointer(g) && frame.header.Descriptor == frame.descriptor && + frame.header.AllocationBase == frame.rawBase +} + +// validCancelableReadyG proves that a ready G contains exactly one structured +// suspended frame chain and no orphan allocation. The active leaf may be a root +// that has never resumed, or a frame suspended only for scheduler yield. Every +// ancestor must be suspended awaiting its direct child. Parked/opaque states +// are rejected before command shutdown changes P.schedule. +func validCancelableReadyG(g *G) bool { + if !ValidG(g) || g.state != GRunnable || !g.queued || g.waiting || g.waitToken != nil || + g.waitTicket != 0 || g.nextWait != nil || g.runP != nil || g.root == nil || g.active == nil || + g.pending.kind != pendingNone || g.pending.from != nil || g.pending.target != nil || + g.pending.wait != nil || g.pending.ticket != 0 || g.destroyTarget != nil || g.destroyRoot || + g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil || + g.taskState != taskStorageOwned || g.taskStorage != unsafe.Pointer(g) || g.taskSize != TaskStorageSize() { + return false + } + gate := preemptLoad(preemptAddress(g)) + if gate != preemptIdle && gate != preemptRequested { + return false + } + + // Validate the allocation list independently, including cycle freedom. + for slow, fast := g.frames, g.frames; fast != nil && fast.next != nil; { + slow = slow.next + fast = fast.next.next + if slow == fast { + return false + } + } + frameCount := 0 + for frame := g.frames; frame != nil; frame = frame.next { + if !validCancelFrame(frame, g) { + return false + } + frameCount++ + } + if frameCount == 0 { + return false + } + + chainCount := 0 + for frame := g.active; frame != nil; frame = frame.parent { + if !validCancelFrame(frame, g) { + return false + } + chainCount++ + if chainCount > frameCount { + return false + } + if frame == g.active { + switch frame.state { + case FrameInitialSuspended: + if frame.header.SuspendReason != uint16(SuspendNone) || + frame.header.Lifecycle != uint16(FrameInitialSuspended) { + return false + } + case FrameSuspended: + if frame.header.SuspendReason != uint16(SuspendYield) || + frame.header.Lifecycle != uint16(FrameSuspended) { + return false + } + default: + return false + } + } else if frame.state != FrameSuspended || frame.header.SuspendReason != uint16(SuspendCall) || + frame.header.Lifecycle != uint16(FrameSuspended) { + return false + } + if frame.parent == nil { + if frame != g.root || frame.header.Parent != nil { + return false + } + } else if frame.header.Parent != frame.parent.handle { + return false + } + } + if chainCount != frameCount { + return false + } + // Count equality plus unique parent traversal is not sufficient if the + // allocation list repeats a chain member through corruption. Prove exact + // membership without allocating a map. + for listed := g.frames; listed != nil; listed = listed.next { + matches := 0 + for frame := g.active; frame != nil; frame = frame.parent { + if listed == frame { + matches++ + } + } + if matches != 1 { + return false + } + } + return true +} + +// BeginCommandShutdown atomically seals a command P against new scheduling +// requests after main has returned normally. Version one supports only ready +// YieldOnly/AwaitStructured children. Any wait/current/action state is rejected +// before the schedule gate changes, because raw WaitToken producers cannot yet +// be unregistered and quiesced safely. +func BeginCommandShutdown(p *P, main *G) bool { + if p == nil || !ReclaimableG(main) || main.taskState != taskStorageStatic || + p.current != nil || p.inResume || p.action.Kind != ActionInvalid || p.action.Handle != nil || + !validReadyQueue(p) || !validWaitQueue(p) || p.waitHead != nil || p.waitTail != nil { + return false + } + for g := p.readyHead; g != nil; g = g.nextReady { + if !validCancelableReadyG(g) { + return false + } + } + for { + schedule := preemptLoad(&p.schedule) + if schedule != scheduleIdle && schedule != scheduleRequested { + return false + } + if preemptCompareAndSwap(&p.schedule, schedule, scheduleStopping) { + return true + } + } +} + +func prepareCancelFrame(p *P, g *G, frame *Frame) (Action, bool) { + if p == nil || g == nil || frame == nil || p.current != g || g.state != GCanceling || + g.destroyTarget != nil || !validCancelFrame(frame, g) || + (frame.state != FrameInitialSuspended && frame.state != FrameSuspended) { + return Action{}, false + } + handle := frame.handle + g.active = frame.parent + g.destroyRoot = frame == g.root + frame.state = FrameDestroyPending + frame.header.Lifecycle = uint16(FrameDestroyPending) + g.destroyTarget = frame + return setAction(p, ActionCancelDestroy, handle) +} + +// NextCommandCancel removes one ready child in FIFO order and requests direct +// destruction of its deepest suspended frame. An empty ready queue returns +// (nil, ActionInvalid, true). +func NextCommandCancel(p *P) (*G, Action, bool) { + if p == nil || preemptLoad(&p.schedule) != scheduleStopping || p.current != nil || + p.inResume || p.action.Kind != ActionInvalid || p.action.Handle != nil || + !validReadyQueue(p) || !validWaitQueue(p) || p.waitHead != nil || p.waitTail != nil { + return nil, Action{}, false + } + g := p.readyHead + if g == nil { + return nil, Action{}, true + } + if !validCancelableReadyG(g) { + return nil, Action{}, false + } + if dequeue(p) != g { + return nil, Action{}, false + } + p.current = g + g.runP = p + g.state = GCanceling + action, ok := prepareCancelFrame(p, g, g.active) + if !ok { + return nil, Action{}, false + } + return g, action, true +} + +// CancelDestroyed commits the return from one direct llvm.coro.destroy. The +// compiler free hook must already have unlinked the destroyed frame. Ancestors +// are destroyed deepest-to-root without coro.done and without resume. +func CancelDestroyed(p *P, g *G, action Action) (Action, bool) { + if !expectedAction(p, g, action, ActionCancelDestroy) || p.inResume || + preemptLoad(&p.schedule) != scheduleStopping || g.state != GCanceling || g.destroyTarget != nil { + return Action{}, false + } + wasRoot := g.destroyRoot + if g.active != nil { + if wasRoot { + return Action{}, false + } + g.destroyRoot = false + return prepareCancelFrame(p, g, g.active) + } + if !wasRoot || g.frames != nil { + return Action{}, false + } + g.destroyRoot = false + g.root = nil + preemptStore(preemptAddress(g), preemptDisabled) + g.state = GDead + g.runP = nil + p.current = nil + p.action = Action{} + return Action{Kind: ActionCancelComplete}, true +} + +// FinishCommandShutdown disables the sealed P only after every ready child has +// been destroyed/reclaimed. No wait producer can survive a successful v1 +// shutdown because BeginCommandShutdown rejected a non-empty wait set. +func FinishCommandShutdown(p *P, main *G) bool { + if p == nil || !ReclaimableG(main) || main.taskState != taskStorageStatic || + p.current != nil || p.inResume || p.action.Kind != ActionInvalid || p.action.Handle != nil || + !validReadyQueue(p) || !validWaitQueue(p) || p.readyHead != nil || p.readyTail != nil || + p.waitHead != nil || p.waitTail != nil { + return false + } + return preemptCompareAndSwap(&p.schedule, scheduleStopping, scheduleDisabled) +} diff --git a/runtime/internal/runtime/coro_program.go b/runtime/internal/runtime/coro_program.go index db6db94859..38e0b61828 100644 --- a/runtime/internal/runtime/coro_program.go +++ b/runtime/internal/runtime/coro_program.go @@ -29,6 +29,8 @@ const ( coroProgramUnusedV1 coroProgramLifecycleV1 = iota coroProgramBegunV1 coroProgramRunningV1 + coroProgramMainReturnRequestedV1 + coroProgramStoppingV1 coroProgramCompleteV1 coroProgramFailedV1 ) @@ -96,7 +98,34 @@ func coroProgramRunV1(gPointer, handle unsafe.Pointer) bool { return false } coroProgramLifecycleV1State = coroProgramRunningV1 - if !coroRun(&coroProgramPV1State, &coroProgramGV1State) || !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { + if !coroRun(&coroProgramPV1State, &coroProgramGV1State) { + coroProgramLifecycleV1State = coroProgramFailedV1 + return false + } + switch coroProgramLifecycleV1State { + case coroProgramRunningV1: + // Backward-compatible no-spawn startup tables do not yet contain the + // explicit main-return hook. They remain valid only when the whole P is + // already terminal; a surviving child fails closed. + if !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { + coroProgramLifecycleV1State = coroProgramFailedV1 + return false + } + case coroProgramMainReturnRequestedV1: + if !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { + if !coro.BeginCommandShutdown(&coroProgramPV1State, &coroProgramGV1State) { + coroProgramLifecycleV1State = coroProgramFailedV1 + return false + } + coroProgramLifecycleV1State = coroProgramStoppingV1 + if !coroCancelReady(&coroProgramPV1State) || + !coro.FinishCommandShutdown(&coroProgramPV1State, &coroProgramGV1State) || + !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { + coroProgramLifecycleV1State = coroProgramFailedV1 + return false + } + } + default: coroProgramLifecycleV1State = coroProgramFailedV1 return false } @@ -104,6 +133,17 @@ func coroProgramRunV1(gPointer, handle unsafe.Pointer) bool { return true } +func coroProgramMainReturnV1(gPointer unsafe.Pointer) bool { + if coroProgramLifecycleV1State != coroProgramRunningV1 || + gPointer != unsafe.Pointer(&coroProgramGV1State) || + !coro.CommandMainReturnPoint(&coroProgramPV1State, &coroProgramGV1State) { + coroProgramLifecycleV1State = coroProgramFailedV1 + return false + } + coroProgramLifecycleV1State = coroProgramMainReturnRequestedV1 + return true +} + //export __llgo_coro_program_begin_v1 func __llgo_coro_program_begin_v1(manifest, expectedFactory unsafe.Pointer) unsafe.Pointer { g, ok := coroProgramBeginV1(manifest, expectedFactory) @@ -120,3 +160,10 @@ func __llgo_coro_program_run_v1(g, handle unsafe.Pointer) { coroRuntimeAbort("invalid coroutine program execution") } } + +//export __llgo_coro_program_main_return_v1 +func __llgo_coro_program_main_return_v1(g unsafe.Pointer) { + if !coroProgramMainReturnV1(g) { + coroRuntimeAbort("invalid coroutine command main return") + } +} diff --git a/runtime/internal/runtime/coro_program_test.go b/runtime/internal/runtime/coro_program_test.go index 18a63bbcda..6a3efa4be8 100644 --- a/runtime/internal/runtime/coro_program_test.go +++ b/runtime/internal/runtime/coro_program_test.go @@ -158,7 +158,7 @@ func newCoroProgramTestFrameV1(t *testing.T, g *coro.G) *coroProgramTestFrameV1 wordSize := unsafe.Sizeof(uintptr(0)) memory := make([]uintptr, (total+wordSize-1)/wordSize) raw := unsafe.Pointer(&memory[0]) - descriptor := unsafe.Pointer(new(byte)) + descriptor := unsafe.Pointer(&coro.FrameDescriptorV1{Version: 1, ResultAlign: 1}) storage, ok := coro.RegisterFrame(g, raw, total, size, align, descriptor) if !ok { t.Fatal("register coroutine program test frame") @@ -196,6 +196,11 @@ type coroProgramTestDriverV1 struct { completeReady bool released bool requestScheduleOnDestroy bool + spawnOnMainReturn bool + child *coro.G + childFrame *coroProgramTestFrameV1 + cancelDestroyCalls int + taskReleaseCalls int } var activeCoroProgramDriver *coroProgramTestDriverV1 @@ -222,7 +227,19 @@ func coroRuntimeAbort(message string) { // Spawn/task-storage tests live in runtime/internal/coro. func coroReleaseCompletedTask(g *coroG) bool { owned, ok := coro.TaskStorageOwned(g) - return ok && !owned + if !ok { + return false + } + if !owned { + return true + } + raw, size, ok := coro.ReleaseTaskStorage(g) + if !ok || raw != unsafe.Pointer(g) || size != coro.TaskStorageSize() || + activeCoroProgramDriver == nil || activeCoroProgramDriver.child != g { + return false + } + activeCoroProgramDriver.taskReleaseCalls++ + return activeCoroProgramDriver.taskReleaseCalls == 1 } func (driver *coroProgramTestDriverV1) requireHandle(handle unsafe.Pointer) { @@ -251,6 +268,26 @@ func (driver *coroProgramTestDriverV1) resume(handle unsafe.Pointer) { driver.t.Fatalf("coroutine resume calls = %d, want 1", driver.resumeCalls) } frame := driver.frame + frame.header.SuspendReason = uint16(coro.SuspendNone) + frame.header.Lifecycle = uint16(coro.FrameActive) + if driver.spawnOnMainReturn { + driver.child = new(coro.G) + if !coro.BeginSpawn(frame.g, driver.child, unsafe.Pointer(driver.child), coro.TaskStorageSize()) { + driver.t.Fatal("begin named-adapter command child") + } + driver.childFrame = newCoroProgramTestFrameV1(driver.t, driver.child) + if !coro.CommitSpawn(frame.g, driver.child, driver.childFrame.handle) { + driver.t.Fatal("commit named-adapter command child") + } + if !coroProgramMainReturnV1(unsafe.Pointer(frame.g)) { + driver.t.Fatal("publish named-adapter normal main return") + } + if coroProgramLifecycleV1State != coroProgramMainReturnRequestedV1 || + !coro.CommandMainReturnPoint(&coroProgramPV1State, frame.g) || + driver.cancelDestroyCalls != 0 || driver.taskReleaseCalls != 0 { + driver.t.Fatal("main-return hook mutated scheduler ownership inside resume") + } + } frame.header.SuspendReason = uint16(coro.SuspendFrameComplete) frame.header.Lifecycle = uint16(coro.FrameFinalSuspended) if !coro.PrepareComplete(frame.g, handle, frame.header) { @@ -260,6 +297,18 @@ func (driver *coroProgramTestDriverV1) resume(handle unsafe.Pointer) { } func (driver *coroProgramTestDriverV1) destroy(handle unsafe.Pointer) { + if driver.childFrame != nil && handle == driver.childFrame.handle { + driver.cancelDestroyCalls++ + if driver.cancelDestroyCalls != 1 { + driver.t.Fatalf("child coroutine destroy calls = %d, want 1", driver.cancelDestroyCalls) + } + frame := driver.childFrame + raw, total, ok := coro.ReleaseFrame(frame.g, frame.storage, frame.size, frame.align, frame.descriptor) + if !ok || raw != frame.raw || total != frame.total { + driver.t.Fatalf("release canceled child frame = (%p, %d, %t)", raw, total, ok) + } + return + } driver.requireHandle(handle) driver.destroyCalls++ if driver.destroyCalls != 1 { @@ -385,6 +434,35 @@ func TestCoroProgramTerminalScheduleRetryDoesNotRedestroy(t *testing.T) { runtime.KeepAlive(manifest) } +func TestCoroProgramNormalMainReturnCancelsReadyChild(t *testing.T) { + resetCoroProgramTestStateV1(t) + manifest := newCoroProgramTestManifestV1() + factory := unsafe.Pointer(&manifest.factoryMarker) + gPointer, ok := coroProgramBeginV1(unsafe.Pointer(&manifest.manifest), factory) + if !ok { + t.Fatal("begin command-shutdown program") + } + frame := newCoroProgramTestFrameV1(t, &coroProgramGV1State) + driver := &coroProgramTestDriverV1{t: t, frame: frame, spawnOnMainReturn: true} + activeCoroProgramDriver = driver + if !coroProgramRunV1(gPointer, frame.handle) { + t.Fatal("run command-shutdown program") + } + if coroProgramLifecycleV1State != coroProgramCompleteV1 || driver.doneCalls != 2 || + driver.resumeCalls != 1 || driver.destroyCalls != 1 || driver.cancelDestroyCalls != 1 || + driver.taskReleaseCalls != 1 || driver.child == nil || driver.childFrame == nil || + !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) || + !coro.TerminalG(&coroProgramPV1State, driver.child) { + t.Fatalf("command shutdown = lifecycle:%d done:%d resume:%d mainDestroy:%d childDestroy:%d taskRelease:%d", + coroProgramLifecycleV1State, driver.doneCalls, driver.resumeCalls, driver.destroyCalls, + driver.cancelDestroyCalls, driver.taskReleaseCalls) + } + runtime.KeepAlive(frame.memory) + runtime.KeepAlive(driver.childFrame.memory) + runtime.KeepAlive(driver.child) + runtime.KeepAlive(manifest) +} + func TestCoroProgramV1BeginFailsClosedOnFactoryIdentity(t *testing.T) { resetCoroProgramTestStateV1(t) manifest := newCoroProgramTestManifestV1() diff --git a/runtime/internal/runtime/coro_sched.go b/runtime/internal/runtime/coro_sched.go index 057cc198ca..eefe1b9e6c 100644 --- a/runtime/internal/runtime/coro_sched.go +++ b/runtime/internal/runtime/coro_sched.go @@ -73,11 +73,54 @@ func coroRun(p *coroP, main *coroG) bool { if !coroRunG(p, g) { return false } + if g == main && coroProgramLifecycleV1State == coroProgramMainReturnRequestedV1 && !coro.DeadG(main) { + // The compiler hook is valid only on main's normal continuation + // immediately before the bootstrap root's final suspend. Yielding or + // parking after publishing the marker is an ABI violation. + return false + } if g == main && coro.DeadG(main) { - // Command main must not drain background goroutines after returning. - // Until the runtime can cancel every ready/suspended child safely, only - // a fully terminal P is a supported main-return state. - return coro.TerminalG(p, main) + // Command main never drains background goroutines. The program adapter + // either enters the explicit ready-child cancellation protocol after a + // normal-main hook, or fails closed. + return true + } + } +} + +// coroCancelReady destroys every ready child deepest-to-root. It deliberately +// never calls coro.done or coro.resume: command shutdown owns only suspended +// YieldOnly/AwaitStructured frame chains. +func coroCancelReady(p *coroP) bool { + for { + g, action, ok := coro.NextCommandCancel(p) + if !ok { + return false + } + if g == nil { + return action.Kind == coro.ActionInvalid && action.Handle == nil + } + for { + switch action.Kind { + case coro.ActionCancelDestroy: + coroHandleDestroy(action.Handle) + action, ok = coro.CancelDestroyed(p, g, action) + if !ok { + return false + } + case coro.ActionCancelComplete: + if action.Handle != nil || !coroReleaseCompletedTask(g) { + return false + } + // g may have been physically freed. Never inspect it again. + g = nil + break + default: + return false + } + if g == nil { + break + } } } } diff --git a/runtime/internal/runtime/coro_spawn.go b/runtime/internal/runtime/coro_spawn.go index caafb247d7..58d67ffec9 100644 --- a/runtime/internal/runtime/coro_spawn.go +++ b/runtime/internal/runtime/coro_spawn.go @@ -23,12 +23,10 @@ import ( "github.com/goplus/llgo/runtime/internal/coroalloc" ) -// Safe cancellation of every ready/suspended background G at command-main -// return is not implemented yet. Keep the production C ABI present for -// compiler/link validation, but fail closed before allocating a child. Core and -// adapter tests call the unexported begin/commit functions to exercise the -// complete scheduler transaction without claiming production Go semantics. -const coroSpawnProductionEnabledV1 = false +// Version one is production-enabled only for plans whose spawn targets are +// proven YieldOnly/AwaitStructured. Command shutdown rejects every wait queue +// and directly destroys ready children deepest-to-root. +const coroSpawnProductionEnabledV1 = true func coroSpawnBeginV1(parentPointer unsafe.Pointer) (unsafe.Pointer, bool) { parent := (*coroG)(parentPointer) From 1f2e76c249ba2d0885f2e6d5a4dfc7c02b8fdfea Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 23:27:44 +0800 Subject: [PATCH 21/32] compiler(coro): lower closed static goroutine spawn --- cl/compilation.go | 14 +- cl/compilation_test.go | 11 + cl/compile.go | 3 + cl/coro_abi.go | 41 ++- cl/coro_entry.go | 15 +- cl/coro_spawn.go | 92 ++++++ cl/coro_spawn_test.go | 252 +++++++++++++++++ internal/build/build.go | 265 +++++++++++++++++- internal/build/collect.go | 2 + internal/build/coro_bootstrap.go | 4 + internal/build/coro_bootstrap_factory.go | 14 + internal/build/coro_bootstrap_factory_test.go | 63 ++++- internal/build/coro_plan_test.go | 41 +++ internal/build/coro_spawn_test.go | 232 +++++++++++++++ internal/build/main_module.go | 5 +- internal/build/main_module_test.go | 8 + internal/coro/func_flow.go | 70 +++++ internal/coro/plan_digest.go | 10 +- internal/coro/plan_digest_test.go | 68 +++++ internal/coro/ssa_plan_test.go | 91 ++++++ 20 files changed, 1281 insertions(+), 20 deletions(-) create mode 100644 cl/coro_spawn.go create mode 100644 cl/coro_spawn_test.go create mode 100644 internal/build/coro_spawn_test.go diff --git a/cl/compilation.go b/cl/compilation.go index e44dfae1b1..6ea576d785 100644 --- a/cl/compilation.go +++ b/cl/compilation.go @@ -67,6 +67,10 @@ type Compilation struct { // call is accepted by this capability; every wider dynamic form remains an // unsupported preflight error. EnableCoroPlainDispatch bool + // EnableCoroClosedStaticSpawn permits only the compilation-plan-certified + // closed static spawn transaction. The physical parent G is passed to both + // runtime hooks; no TLS lookup or indirect user callback is permitted. + EnableCoroClosedStaticSpawn bool // EnableCoroProgramBootstrapRun selects the program-root scheduler ABI for // package identities. The factory itself lives in the uncached entry module, // but every linked archive must agree with the runtime driver contract. @@ -108,7 +112,15 @@ func (c *Compilation) validateCoroABIIdentity(required bool) error { if c.EnableCoroChildAwait { wantSchedulerABI = coro.SchedulerChildAwaitABIV0 } - if c.EnableCoroProgramBootstrapRun { + if c.EnableCoroClosedStaticSpawn { + if !c.EnableCoroChildAwait { + return fmt.Errorf("coroutine closed static spawn requires child-await lowering") + } + if !c.EnableCoroProgramBootstrapRun { + return fmt.Errorf("coroutine closed static spawn requires the runnable program-bootstrap v2 scheduler") + } + wantSchedulerABI = coro.SchedulerProgramBootstrapClosedStaticSpawnABIV0 + } else if c.EnableCoroProgramBootstrapRun { if !c.EnableCoroChildAwait { return fmt.Errorf("coroutine program bootstrap runtime requires child-await lowering") } diff --git a/cl/compilation_test.go b/cl/compilation_test.go index c779eadaae..0516805bfd 100644 --- a/cl/compilation_test.go +++ b/cl/compilation_test.go @@ -142,6 +142,17 @@ func TestCompilationCoroABIIdentityValidation(t *testing.T) { if err := programBootstrap.validateCoroABIIdentity(false); err != nil { t.Fatalf("complete program-bootstrap ABI identity: %v", err) } + closedStaticSpawn := newChildAwait() + closedStaticSpawn.EnableCoroProgramBootstrapRun = true + closedStaticSpawn.EnableCoroClosedStaticSpawn = true + closedStaticSpawn.SchedulerABI = coro.SchedulerProgramBootstrapClosedStaticSpawnABIV0 + if err := closedStaticSpawn.validateCoroABIIdentity(false); err != nil { + t.Fatalf("complete closed-static-spawn ABI identity: %v", err) + } + closedStaticSpawn.EnableCoroProgramBootstrapRun = false + if err := closedStaticSpawn.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "runnable program-bootstrap v2") { + t.Fatalf("closed-static-spawn bootstrap dependency error = %v", err) + } programBootstrap.EnableCoroChildAwait = false if err := programBootstrap.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "requires child-await") { t.Fatalf("program-bootstrap dependency error = %v", err) diff --git a/cl/compile.go b/cl/compile.go index 432ec5994b..f238c4ef83 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -1701,6 +1701,9 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { } p.call(b, p.blkInfos[v.Block().Index].Kind, &v.Call) case *ssa.Go: + if p.tryCompileCoroClosedStaticSpawn(b, v) { + return + } p.call(b, llssa.Go, &v.Call) case *ssa.RunDefers: p.recordPanicLocation(b, v.Pos()) diff --git a/cl/coro_abi.go b/cl/coro_abi.go index effd8e0d40..2ca2c64ef5 100644 --- a/cl/coro_abi.go +++ b/cl/coro_abi.go @@ -46,6 +46,8 @@ const ( coroPreemptPollHookV1 = "__llgo_coro_preempt_poll_v1" coroYieldPrepareHookV1 = "__llgo_coro_yield_prepare_v1" coroParkPrepareHookV1 = "__llgo_coro_park_prepare_v1" + coroSpawnBeginHookV1 = "__llgo_coro_spawn_begin_v1" + coroSpawnCommitHookV1 = "__llgo_coro_spawn_commit_v1" coroCompletePrepareHookV1 = "__llgo_coro_complete_prepare_v1" coroFrameFreeHookV1 = "__llgo_coro_frame_free_v1" coroDescriptorPrefixV1 = "__llgo_coro_frame_descriptor_v1." @@ -586,7 +588,7 @@ func (p *context) compileCoroPhysicalBody(b llssa.Builder, fn *ssa.Function, abi } func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, childAwait, programRun bool) error { - return validateCoroPhysicalABIWithUniverse(fn, plan, whole, nil, childAwait, programRun) + return validateCoroPhysicalABIWithUniverseCapabilities(fn, plan, whole, nil, childAwait, programRun, false) } // validateCoroPhysicalABIWithUniverse is the production preflight. The @@ -595,6 +597,10 @@ func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *co // The wrapper above is retained for narrow structural unit tests; active // Compilation paths always call this form with their frozen universe. func validateCoroPhysicalABIWithUniverse(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, universe *EmissionUniverse, childAwait, programRun bool) error { + return validateCoroPhysicalABIWithUniverseCapabilities(fn, plan, whole, universe, childAwait, programRun, false) +} + +func validateCoroPhysicalABIWithUniverseCapabilities(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, universe *EmissionUniverse, childAwait, programRun, staticSpawn bool) error { if !childAwait { return validateCoroLeafPhysicalABI(fn, plan) } @@ -672,6 +678,7 @@ func validateCoroPhysicalABIWithUniverse(fn *ssa.Function, plan coro.FunctionPla returns := 0 awaits := 0 parks := 0 + spawns := 0 infos := blocks.Infos(fn.Blocks) hasCyclicBlock := false for _, info := range infos { @@ -739,6 +746,21 @@ func validateCoroPhysicalABIWithUniverse(fn *ssa.Function, plan coro.FunctionPla if _, _, plainErr := resolveCoroStaticPlainCall(whole, instr); plainErr != nil { return coroLeafInstructionError(fn, plan, instr, "unsupported call: child await: "+err.Error()+"; direct plain: "+plainErr.Error()) } + case *ssa.Go: + if !staticSpawn { + return coroLeafInstructionError(fn, plan, instr, "goroutine spawn requires the closed-static scheduler capability") + } + target, targetPlan, err := whole.ResolveClosedStaticSpawn(instr) + if err != nil { + return coroLeafInstructionError(fn, plan, instr, "unsupported closed static spawn: "+err.Error()) + } + if err := validateCoroLeafPhysicalSignature(targetPlan, target.Signature); err != nil { + return coroLeafInstructionError(fn, plan, instr, "spawn target signature: "+err.Error()) + } + if coroPhysicalSignatureContainsFunctionValue(target.Signature) { + return coroLeafInstructionError(fn, plan, instr, "spawn target function-valued parameters require a later canonical transport capability") + } + spawns++ default: return coroLeafInstructionError(fn, plan, instr, "instruction is outside the CFG physical ABI allowlist") } @@ -756,6 +778,9 @@ func validateCoroPhysicalABIWithUniverse(fn *ssa.Function, plan coro.FunctionPla if parks != 0 && !plan.Effect.Contains(coro.MayPark) { return fail("structured-park body lacks may-park final effect: %s", plan.Effect) } + if spawns != 0 && (!plan.DeclaredEffect.Contains(coro.YieldOnly) || !plan.LocalEffect.Contains(coro.YieldOnly) || !plan.Effect.Contains(coro.YieldOnly)) { + return fail("closed static spawn body lacks its exact yield-only owner seed: declared=%s local=%s final=%s", plan.DeclaredEffect, plan.LocalEffect, plan.Effect) + } if plan.DeclaredEffect.Contains(coro.MayPark) && parks == 0 { return fail("declared may-park effect has no exact structured park intrinsic") } @@ -1115,6 +1140,10 @@ func coroLeafABIDirective(fn *ssa.Function) string { } func validateCoroPhysicalConsumers(plan *coro.SSAPlan, childAwait bool) error { + return validateCoroPhysicalConsumersCapabilities(plan, childAwait, false) +} + +func validateCoroPhysicalConsumersCapabilities(plan *coro.SSAPlan, childAwait, staticSpawn bool) error { coroutineIDs := make(map[coro.FunctionID]struct{}) for _, function := range plan.Functions() { if function.Plan.Emission == coro.EmitCoroutine { @@ -1128,8 +1157,14 @@ func validateCoroPhysicalConsumers(plan *coro.SSAPlan, childAwait bool) error { fn := function.Function for _, block := range fn.Blocks { for _, instr := range block.Instrs { - if _, spawn := instr.(*ssa.Go); spawn { - return coroLeafInstructionError(fn, function.Plan, instr, "goroutine spawn requires scheduler root lowering") + if spawn, ok := instr.(*ssa.Go); ok { + if !staticSpawn { + return coroLeafInstructionError(fn, function.Plan, instr, "goroutine spawn requires scheduler root lowering") + } + if _, _, err := plan.ResolveClosedStaticSpawn(spawn); err != nil { + return coroLeafInstructionError(fn, function.Plan, instr, "unsupported closed static spawn: "+err.Error()) + } + continue } if call, ok := instr.(ssa.CallInstruction); ok { if plan.ElidesCall(call) { diff --git a/cl/coro_entry.go b/cl/coro_entry.go index fdaf09417c..fd2bfc64d4 100644 --- a/cl/coro_entry.go +++ b/cl/coro_entry.go @@ -42,6 +42,7 @@ type plannedFunctionSymbol struct { childAwait bool programRun bool plainDispatch bool + staticSpawn bool coroPlan *coro.SSAPlan emission *EmissionUniverse } @@ -89,6 +90,7 @@ func (p *context) resolveFunctionSymbol(fn *ssa.Function) (plannedFunctionSymbol entry.childAwait = p.compilation.EnableCoroChildAwait entry.programRun = p.compilation.EnableCoroProgramBootstrapRun entry.plainDispatch = p.compilation.EnableCoroPlainDispatch + entry.staticSpawn = p.compilation.EnableCoroClosedStaticSpawn entry.coroPlan = p.compilation.CoroPlan entry.emission = p.compilation.EmissionUniverse if p.compilation.CoroPlan.IgnoresBody(fn) { @@ -181,7 +183,7 @@ func (e plannedFunctionSymbol) checkSupported() error { if err := validateCoroPhysicalFunctionValueABI(e.plan, e.function.Signature, e.plainDispatch); err != nil { return err } - return validateCoroPhysicalABIWithUniverse(e.function, e.plan, e.coroPlan, e.emission, e.childAwait, e.programRun) + return validateCoroPhysicalABIWithUniverseCapabilities(e.function, e.plan, e.coroPlan, e.emission, e.childAwait, e.programRun, e.staticSpawn) } if e.plan.Emission == coro.EmitExternal && e.plan.FuncRep == coro.DirectCoro { return fmt.Errorf("external coroutine emission %q requires coroutine physical ABI lowering", e.plan.ID) @@ -206,6 +208,14 @@ func (c *Compilation) preflightCoroPlan() error { if c.EnableCoroPlainDispatch && !c.EnableCoroEntryResolution { return fmt.Errorf("coroutine plain dispatch requires coroutine entry resolution") } + if c.EnableCoroClosedStaticSpawn { + if !c.EnableCoroChildAwait { + return fmt.Errorf("coroutine closed static spawn requires coroutine child await") + } + if !c.EnableCoroProgramBootstrapRun { + return fmt.Errorf("coroutine closed static spawn requires runnable program bootstrap v2") + } + } if !c.EnableCoroEntryResolution { return nil } @@ -253,6 +263,7 @@ func (c *Compilation) preflightCoroPlan() error { childAwait: c.EnableCoroChildAwait, programRun: c.EnableCoroProgramBootstrapRun, plainDispatch: c.EnableCoroPlainDispatch, + staticSpawn: c.EnableCoroClosedStaticSpawn, coroPlan: c.CoroPlan, emission: c.EmissionUniverse, } @@ -275,7 +286,7 @@ func (c *Compilation) preflightCoroPlan() error { } } if c.EnableCoroPhysicalABI { - c.coroPreflightErr = validateCoroPhysicalConsumers(c.CoroPlan, c.EnableCoroChildAwait) + c.coroPreflightErr = validateCoroPhysicalConsumersCapabilities(c.CoroPlan, c.EnableCoroChildAwait, c.EnableCoroClosedStaticSpawn) if c.coroPreflightErr != nil { return } diff --git a/cl/coro_spawn.go b/cl/coro_spawn.go new file mode 100644 index 0000000000..169ea3ffd5 --- /dev/null +++ b/cl/coro_spawn.go @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "fmt" + "go/token" + "go/types" + + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +func coroSpawnBeginSignature() *types.Signature { + pointer := types.Typ[types.UnsafePointer] + return types.NewSignatureType(nil, nil, nil, + types.NewTuple(types.NewParam(token.NoPos, nil, "parent", pointer)), + types.NewTuple(types.NewParam(token.NoPos, nil, "child", pointer)), + false, + ) +} + +func coroSpawnCommitSignature() *types.Signature { + pointer := types.Typ[types.UnsafePointer] + return types.NewSignatureType(nil, nil, nil, + types.NewTuple( + types.NewParam(token.NoPos, nil, "parent", pointer), + types.NewParam(token.NoPos, nil, "child", pointer), + types.NewParam(token.NoPos, nil, "handle", pointer), + ), nil, false, + ) +} + +// tryCompileCoroClosedStaticSpawn creates exactly one child root to its LLVM +// initial suspend and commits it to the scheduler. Arguments are fully +// materialized before begin mutates scheduler state. The parent then reaches +// an explicit safepoint using its physical G; there is no TLS/current-G +// fallback anywhere in this path. +func (p *context) tryCompileCoroClosedStaticSpawn(b llssa.Builder, spawn *ssa.Go) bool { + if p.compilation == nil || !p.compilation.EnableCoroClosedStaticSpawn || spawn == nil { + return false + } + if p.currentCoro == nil || p.compilation.CoroPlan == nil || b.Func != p.fn { + panic("closed static spawn requires an active planned physical coroutine body") + } + target, targetPlan, err := p.compilation.CoroPlan.ResolveClosedStaticSpawn(spawn) + if err != nil { + caller, _ := p.compilation.CoroPlan.FunctionPlan(p.goFn) + panic(fmt.Sprintf("closed static spawn: function %q: %v", caller.ID, err)) + } + + p.recordCallerLocationForCall(b, &spawn.Call) + p.emitPCLineLabel(b, spawn.Pos()) + // Go SSA already sequences argument-producing instructions. Re-materialize + // every exact operand here, in source order, before the begin transaction. + args := p.compileValues(b, spawn.Call.Args, fnNormal) + + parent := p.currentCoro.task + begin := p.pkg.NewFunc(coroSpawnBeginHookV1, coroSpawnBeginSignature(), llssa.InC) + childG := b.Call(begin.Expr, parent) + null := p.prog.Nil(p.prog.VoidPtr()) + physicalArgs := make([]llssa.Expr, 0, len(args)+2) + physicalArgs = append(physicalArgs, childG, null) + physicalArgs = append(physicalArgs, args...) + + root, _, kind := p.compileFunction(target) + if kind != goFunc { + panic(fmt.Sprintf("closed static spawn: target %q did not resolve to a Go coroutine entry", targetPlan.ID)) + } + if root == nil { + panic(fmt.Sprintf("closed static spawn: target %q has no physical root", targetPlan.ID)) + } + handle := b.Call(root.Expr, physicalArgs...) + commit := p.pkg.NewFunc(coroSpawnCommitHookV1, coroSpawnCommitSignature(), llssa.InC) + b.Call(commit.Expr, parent, childG, handle) + p.currentCoro.pollAndSuspendForPreempt(b) + return true +} diff --git a/cl/coro_spawn_test.go b/cl/coro_spawn_test.go new file mode 100644 index 0000000000..1fd941a231 --- /dev/null +++ b/cl/coro_spawn_test.go @@ -0,0 +1,252 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "bytes" + "regexp" + "strings" + "testing" + + "github.com/goplus/llgo/internal/coro" + "github.com/goplus/llgo/internal/goembed" + llssa "github.com/goplus/llgo/ssa" + "github.com/xgo-dev/llvm" + "golang.org/x/tools/go/ssa" +) + +const coroClosedStaticSpawnTestSource = `package foo + +var Sink uint32 + +func ArgFirst(value uint32) uint32 { return value + 1 } +func ArgSecond(value uint32) uint32 { return value + 2 } +func Plain(first, second uint32) { Sink = first + second } +func Async(value uint32) { Sink = value } + +func Parent(value uint32) { + Plain(value, value) + go Plain(ArgFirst(value), ArgSecond(value)) + go Async(value) +} +` + +func TestCoroClosedStaticSpawnNativeAndWasm32(t *testing.T) { + llssa.Initialize(llssa.InitAll) + for _, test := range []struct { + name string + target *llssa.Target + }{ + {name: "native"}, + {name: "wasm32", target: &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}}, + } { + t.Run(test.name, func(t *testing.T) { + prog, pkg, plan, ssaPkg := compileCoroClosedStaticSpawnFixture(t, test.target) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify closed static spawn before CoroSplit: %v\n%s", err, module.String()) + } + parentPlan, _ := plan.FunctionPlan(ssaPkg.Func("Parent")) + if parentPlan.DeclaredEffect != coro.YieldOnly || !parentPlan.LocalEffect.Contains(coro.YieldOnly) || + !parentPlan.Effect.Contains(coro.YieldOnly) || parentPlan.Emission != coro.EmitCoroutine || + parentPlan.Primary != coro.PrimaryCoroutine || parentPlan.FuncRep != coro.DirectCoro || parentPlan.Demand != coro.AsyncDemand { + t.Fatalf("Parent plan = %+v", parentPlan) + } + plainPlan, _ := plan.FunctionPlan(ssaPkg.Func("Plain")) + if plainPlan.Emission != coro.EmitCoroutine || plainPlan.Primary != coro.PrimaryCoroutine || plainPlan.FuncRep != coro.DirectCoro || + !plainPlan.Effect.Contains(coro.YieldOnly) || plainPlan.Demand != coro.AsyncDemand { + t.Fatalf("Plain sync+spawn plan = %+v", plainPlan) + } + asyncPlan, _ := plan.FunctionPlan(ssaPkg.Func("Async")) + if asyncPlan.Emission != coro.EmitCoroutine || asyncPlan.Primary != coro.PrimaryCoroutine || + asyncPlan.FuncRep != coro.DirectCoro || asyncPlan.Demand != coro.AsyncDemand { + t.Fatalf("Async spawn plan = %+v", asyncPlan) + } + + ir := module.String() + parent := requireCoroPhysicalFunction(t, module, "foo.Parent").String() + if !module.NamedFunction("foo.Plain").IsNil() || module.NamedFunction("foo.Plain"+coroPrimarySuffix).IsNil() { + t.Fatalf("bounded sync+spawn target did not retain exactly one preemptible coroutine primary:\n%s", ir) + } + if !module.NamedFunction("foo.Async").IsNil() || module.NamedFunction("foo.Async"+coroPrimarySuffix).IsNil() { + t.Fatalf("Async did not retain exactly one coroutine primary:\n%s", ir) + } + if strings.Contains(ir, "__llgo_coro_spawn_plain_adapter") { + t.Fatalf("spawn target incorrectly gained a second plain-root adapter body:\n%s", ir) + } + + index := func(pattern string) int { + match := regexp.MustCompile(pattern).FindStringIndex(parent) + if match == nil { + return -1 + } + return match[0] + } + first := index(`call i32 @"?foo\.ArgFirst"?`) + second := index(`call i32 @"?foo\.ArgSecond"?`) + begin := strings.Index(parent, "call ptr @"+coroSpawnBeginHookV1) + plainRoot := -1 + if begin >= 0 { + if relative := regexp.MustCompile(`call ptr @"?foo\.Plain\$coro"?\(`).FindStringIndex(parent[begin:]); relative != nil { + plainRoot = begin + relative[0] + } + } + commit := strings.Index(parent, "call void @"+coroSpawnCommitHookV1) + poll := strings.Index(parent, "call i1 @"+coroPreemptPollHookV1) + if first < 0 || second < 0 || begin < 0 || plainRoot < 0 || commit < 0 || poll < 0 || + !(first < second && second < begin && begin < plainRoot && plainRoot < commit && commit < poll) { + t.Fatalf("argument/begin/root/commit/safepoint order is invalid:\n%s", parent) + } + if got := strings.Count(parent, "call ptr @"+coroSpawnBeginHookV1); got != 2 { + t.Fatalf("spawn begin calls = %d, want two:\n%s", got, parent) + } + if got := strings.Count(parent, "call void @"+coroSpawnCommitHookV1); got != 2 { + t.Fatalf("spawn commit calls = %d, want two:\n%s", got, parent) + } + if got := strings.Count(parent, "call i1 @"+coroPreemptPollHookV1); got != 2 { + t.Fatalf("post-commit explicit preempt polls = %d, want two:\n%s", got, parent) + } + if got := strings.Count(parent, "call void @"+coroYieldPrepareHookV1); got != 2 { + t.Fatalf("post-commit parent yield handoffs = %d, want two:\n%s", got, parent) + } + if !regexp.MustCompile(`call ptr @"?foo\.Async\$coro"?\(`).MatchString(parent) { + t.Fatalf("suspendable target is not called through its unique physical root:\n%s", parent) + } + if strings.Contains(parent[begin:commit], "@llvm.coro.promise") { + t.Fatalf("independent spawned G incorrectly received an await parent-handle link:\n%s", parent[begin:commit]) + } + if got := len(regexp.MustCompile(`call ptr @"?foo\.Plain\$coro"?\(`).FindAllStringIndex(parent, -1)); got != 2 { + t.Fatalf("sync await + spawn calls to the one Plain primary = %d, want two:\n%s", got, parent) + } + for _, forbidden := range []string{"CreateThread", "InitThreadAttr", "DestroyThreadAttr", "._llgo_routine$", "pthread", "AllocRoot"} { + if strings.Contains(ir, forbidden) { + t.Fatalf("closed static spawn leaked legacy native-stack lowering %q:\n%s", forbidden, ir) + } + } + + runCoroABITestPipeline(t, prog, module) + for _, name := range []string{"foo.Parent$coro", "foo.Plain$coro", "foo.Async$coro"} { + for _, suffix := range []string{".resume", ".destroy"} { + if module.NamedFunction(name + suffix).IsNil() { + t.Fatalf("CoroSplit did not create %s%s:\n%s", name, suffix, module.String()) + } + } + } + for _, intrinsic := range []string{"llvm.coro.id", "llvm.coro.begin", "llvm.coro.suspend", "llvm.coro.end"} { + if hasLLVMCall(module.String(), intrinsic) { + t.Fatalf("post-split spawn module still calls %s:\n%s", intrinsic, module.String()) + } + } + object, err := prog.TargetMachine().EmitToMemoryBuffer(module, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit post-CoroSplit spawn object: %v\n%s", err, module.String()) + } + defer object.Dispose() + for _, symbol := range []string{coroSpawnBeginHookV1, coroSpawnCommitHookV1, "foo.Plain$coro"} { + if len(object.Bytes()) == 0 || !bytes.Contains(object.Bytes(), []byte(symbol)) { + t.Fatalf("post-CoroSplit object lost spawn symbol %q", symbol) + } + } + }) + } +} + +func compileCoroClosedStaticSpawnFixture(t *testing.T, target *llssa.Target) ( + llssa.Program, llssa.Package, *coro.SSAPlan, *ssa.Package, +) { + t.Helper() + ssaPkg, _, files := buildGoSSAPkg(t, coroClosedStaticSpawnTestSource) + var prog llssa.Program + if target == nil { + prog = newLLSSAProg(t) + } else { + prog = newLLSSAProgForTarget(t, target) + } + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + parent, plain, async := ssaPkg.Func("Parent"), ssaPkg.Func("Plain"), ssaPkg.Func("Async") + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapClosedStaticSpawnABIV0 + functionIDs.ArchiveReady = true + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: parent, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == parent || fn == plain || fn == async { + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + }) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + compilation := &Compilation{ + CoroPlan: plan, + EmissionUniverse: universe, + EnableCoroEntryResolution: true, + EnableCoroPhysicalABI: true, + EnableCoroChildAwait: true, + EnableCoroClosedStaticSpawn: true, + EnableCoroProgramBootstrapRun: true, + CoroABI: coro.PhysicalABIV1, + SchedulerABI: coro.SchedulerProgramBootstrapClosedStaticSpawnABIV0, + PanicABI: coro.PanicLegacyABIV0, + FuncRepABI: coro.FuncRepABIV0, + } + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + return prog, pkg, plan, ssaPkg +} + +func TestCoroClosedStaticSpawnCompilationCapabilityFailsClosed(t *testing.T) { + compilation := &Compilation{EnableCoroClosedStaticSpawn: true} + if err := compilation.preflightCoroPlan(); err == nil || !strings.Contains(err.Error(), "requires coroutine child await") { + t.Fatalf("capability dependency error = %v", err) + } + compilation = &Compilation{ + EnableCoroEntryResolution: true, + EnableCoroPhysicalABI: true, + EnableCoroChildAwait: true, + EnableCoroClosedStaticSpawn: true, + } + if err := compilation.preflightCoroPlan(); err == nil || !strings.Contains(err.Error(), "requires runnable program bootstrap v2") { + t.Fatalf("runnable-bootstrap dependency error = %v", err) + } +} diff --git a/internal/build/build.go b/internal/build/build.go index 0c6097fc1a..bf20b441f8 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -37,6 +37,7 @@ import ( "sync/atomic" "golang.org/x/tools/go/ssa" + "golang.org/x/tools/go/ssa/ssautil" "github.com/goplus/llgo/cl" "github.com/goplus/llgo/internal/buildenv" @@ -145,6 +146,7 @@ type CoroPlanInput struct { requiredPlain map[*ssa.Function]struct{} requiredDirectPlain []requiredCoroDirectPlainCallArgument requiredClosedDynamic map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate + enableClosedStaticSpawn bool recordAnalysis func(*coro.SSAPlan) } @@ -305,6 +307,83 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. return policy, nil } } + // A source `go f(args)` is a scheduler boundary even though CallSpawn + // deliberately does not taint its owner in the generic effect graph. The + // no-TLS lowering must retain the owner's exact G explicitly. The spawned + // target is also a coroutine primary even when its source body is currently + // bounded: otherwise a future CPU-heavy/looping version could run forever in + // a synchronous plain adapter with no preemption cut. Static sync callers are + // then tainted through the ordinary effect graph and await this same unique + // target body. + if in.enableClosedStaticSpawn { + seeded := make(map[*ssa.Function]struct{}) + var functions []*ssa.Function + if in.EmissionUniverse != nil { + functions = in.EmissionUniverse.Functions() + } else { + for fn := range ssautil.AllFunctions(in.Program) { + functions = append(functions, fn) + } + slices.SortFunc(functions, func(left, right *ssa.Function) int { + if left == nil { + if right == nil { + return 0 + } + return -1 + } + if right == nil { + return 1 + } + return strings.Compare(left.String(), right.String()) + }) + } + for _, fn := range functions { + if fn == nil { + continue + } + if in.functionBackground != nil { + background, classified, err := in.functionBackground(fn) + if err != nil { + return nil, fmt.Errorf("classify closed static spawn owner %q frontend ABI: %w", fn.Name(), err) + } + if classified && background != llssa.InGo { + continue + } + } + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + spawn, ok := instruction.(*ssa.Go) + if !ok { + continue + } + target, err := in.closedStaticSpawnTarget(fn, spawn) + if err != nil { + return nil, fmt.Errorf("closed static spawn in %q: %w", fn.Name(), err) + } + seeded[fn] = struct{}{} + seeded[target] = struct{}{} + } + } + } + classify := config.ClassifyFunction + config.ClassifyFunction = func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + var policy coro.SSAFunctionPolicy + var err error + if classify != nil { + policy, err = classify(fn) + if err != nil { + return coro.SSAFunctionPolicy{}, err + } + } + if _, required := seeded[fn]; required { + if policy.IgnoreBody { + return coro.SSAFunctionPolicy{}, fmt.Errorf("closed static spawn function %q is not a Go-emitted body", fn.Name()) + } + policy.Effect = policy.Effect.Join(coro.YieldOnly) + } + return policy, nil + } + } if len(in.requiredPlain) != 0 { classify := config.ClassifyFunction config.ClassifyFunction = func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { @@ -513,12 +592,138 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. if err == nil { err = validateRequiredCoroClosedDynamicCalls(plan, in.requiredClosedDynamic) } + if err == nil && in.enableClosedStaticSpawn { + err = validateClosedStaticSpawnPlan(plan) + } if err == nil && in.recordAnalysis != nil { in.recordAnalysis(plan) } return plan, err } +func (in CoroPlanInput) closedStaticSpawnTarget(owner *ssa.Function, spawn *ssa.Go) (*ssa.Function, error) { + if owner == nil || spawn == nil || spawn.Common() == nil || spawn.Parent() != owner { + return nil, fmt.Errorf("requires an exact owner and call site") + } + common := spawn.Common() + raw, direct := common.Value.(*ssa.Function) + if !direct || raw == nil || common.IsInvoke() || common.Method != nil || common.StaticCallee() != raw { + return nil, fmt.Errorf("requires a direct static function operand; closures, methods, interfaces, and function values are unsupported") + } + target, ok := in.ResolveFunction(raw) + if !ok || target == nil { + return nil, fmt.Errorf("target %q is outside the frozen emission universe", raw.Name()) + } + if target.Parent() != nil || len(target.FreeVars) != 0 || target.Synthetic != "" || target.Origin() != nil || len(target.TypeArgs()) != 0 { + return nil, fmt.Errorf("target %q is not an exact non-capturing top-level function", target.Name()) + } + if params := target.TypeParams(); params != nil && params.Len() != 0 { + return nil, fmt.Errorf("target %q is a generic declaration", target.Name()) + } + sig := target.Signature + if sig == nil || sig.Recv() != nil || sig.Variadic() || sig.Results().Len() != 0 || + typeParamLen(sig.TypeParams()) != 0 || typeParamLen(sig.RecvTypeParams()) != 0 { + return nil, fmt.Errorf("target %q must have a non-method, non-variadic, zero-result signature", target.Name()) + } + if len(target.Blocks) == 0 { + return nil, fmt.Errorf("target %q has no defined Go body", target.Name()) + } + if in.functionBackground != nil { + background, classified, err := in.functionBackground(target) + if err != nil { + return nil, fmt.Errorf("classify target %q frontend ABI: %w", target.Name(), err) + } + if !classified || background != llssa.InGo { + return nil, fmt.Errorf("target %q is not one frozen Go-emitted body", target.Name()) + } + } + return target, nil +} + +func validateClosedStaticSpawnPlan(plan *coro.SSAPlan) error { + if plan == nil { + return fmt.Errorf("closed static spawn validation requires a coroutine plan") + } + for _, owner := range plan.Functions() { + if owner.Function == nil || owner.Plan.Emission == coro.EmitNone || plan.IgnoresBody(owner.Function) { + continue + } + for _, block := range owner.Function.Blocks { + for _, instruction := range block.Instrs { + spawn, ok := instruction.(*ssa.Go) + if !ok { + continue + } + if _, _, err := plan.ResolveClosedStaticSpawn(spawn); err != nil { + return fmt.Errorf("closed static spawn in %q: %w", owner.Plan.ID, err) + } + } + } + } + return nil +} + +func coroPlanContainsSpawn(plan *coro.SSAPlan) bool { + if plan == nil { + return false + } + for _, owner := range plan.Functions() { + if owner.Function == nil || owner.Plan.Emission == coro.EmitNone || plan.IgnoresBody(owner.Function) { + continue + } + for _, block := range owner.Function.Blocks { + for _, instruction := range block.Instrs { + if _, spawn := instruction.(*ssa.Go); spawn { + return true + } + } + } + } + return false +} + +func validateCoroClosedStaticSpawnRunGate(conf *Config, plan *coro.SSAPlan) error { + if conf == nil || !conf.EnableCoroClosedStaticSpawn { + return nil + } + if !conf.EnableCoroProgramBootstrapRun { + return fmt.Errorf("validate coroutine closed static spawn: runnable program bootstrap v2 is required") + } + if plan == nil { + return fmt.Errorf("validate coroutine closed static spawn: runnable capability requires a coroutine plan") + } + // Main-return cancellation can safely retire ready/yielded children and a + // structured await tree. Platform, host, foreign, channel/select and opaque + // waits need separate producer quiescence/cancellation protocols, so keep + // those targets outside this first production slice. + allowed := coro.YieldOnly | coro.AwaitStructured + for _, owner := range plan.Functions() { + if owner.Function == nil || owner.Plan.Emission == coro.EmitNone || plan.IgnoresBody(owner.Function) { + continue + } + for _, block := range owner.Function.Blocks { + for _, instruction := range block.Instrs { + spawn, ok := instruction.(*ssa.Go) + if !ok { + continue + } + _, target, err := plan.ResolveClosedStaticSpawn(spawn) + if err != nil { + return fmt.Errorf("validate coroutine closed static spawn in %q: %w", owner.Plan.ID, err) + } + effect := target.Effect.Normalize() + if !effect.Contains(coro.YieldOnly) || effect&^allowed != 0 { + return fmt.Errorf( + "validate coroutine closed static spawn in %q: target %q effect %s is outside the production main-return cancellation subset %s", + owner.Plan.ID, target.ID, effect, allowed, + ) + } + } + } + } + return nil +} + func sameExactCoroFunctionReferences(left, right []*ssa.Function) bool { if len(left) != len(right) { return false @@ -700,6 +905,13 @@ type Config struct { // coroutine, interface, reflect, method, go/defer, aggregate, or captured // closure dispatch. EnableCoroPlainDispatch bool + // EnableCoroClosedStaticSpawn enables only an exact source `go f(args)` + // whose operand is one closed, top-level static function. This first + // capability accepts only zero-result targets; that is a lowering gate, not + // a Go language restriction. It requires the runnable program-bootstrap v2 + // scheduler (including the v1 physical/child-await ABI) and never gives the + // runtime a user callback. + EnableCoroClosedStaticSpawn bool // EnableCoroProgramBootstrapABI emits the target-neutral v1 startup table // for an executable after the exact init/main entries have been validated // against the frozen whole-program plan. It does not replace the legacy @@ -1163,6 +1375,14 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { if ctx == nil || ctx.buildConf == nil { return nil } + if ctx.buildConf.EnableCoroClosedStaticSpawn { + if !ctx.buildConf.EnableCoroProgramBootstrapRun { + return fmt.Errorf("enable coroutine closed static spawn: runnable program bootstrap v2 is required") + } + if !ctx.buildConf.EnableCoroChildAwait { + return fmt.Errorf("enable coroutine closed static spawn: coroutine child await is required") + } + } if err := validateCoroProgramBootstrapConfig(ctx.buildConf); err != nil { return err } @@ -1219,11 +1439,12 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { } requiredRoots = append(requiredRoots, managedEntryRoots...) input := CoroPlanInput{ - Program: ctx.progSSA, - requiredRoots: requiredRoots, - requiredPlain: requiredPlain, - requiredDirectPlain: requiredDirectPlain, - requiredClosedDynamic: requiredClosedDynamic, + Program: ctx.progSSA, + requiredRoots: requiredRoots, + requiredPlain: requiredPlain, + requiredDirectPlain: requiredDirectPlain, + requiredClosedDynamic: requiredClosedDynamic, + enableClosedStaticSpawn: ctx.buildConf.EnableCoroClosedStaticSpawn, recordAnalysis: func(plan *coro.SSAPlan) { if plan != nil { analyzedPlansMu.Lock() @@ -1272,6 +1493,9 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { return fmt.Errorf("validate coroutine plan coverage: %w", err) } } + if err := validateCoroClosedStaticSpawnRunGate(ctx.buildConf, plan); err != nil { + return err + } var metadata coro.PlanDigestMetadata var digest string if ctx.buildConf.EnableCoroEntryResolution { @@ -1294,6 +1518,7 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { EnableCoroPhysicalABI: ctx.buildConf.EnableCoroPhysicalABI, EnableCoroChildAwait: ctx.buildConf.EnableCoroChildAwait, EnableCoroPlainDispatch: ctx.buildConf.EnableCoroPlainDispatch, + EnableCoroClosedStaticSpawn: ctx.buildConf.EnableCoroClosedStaticSpawn, EnableCoroProgramBootstrapRun: ctx.buildConf.EnableCoroProgramBootstrapRun, CoroPlanDigest: digest, CoroABI: metadata.CoroABI, @@ -1590,6 +1815,9 @@ func requiredCoroProgramManagedEntryRoots(ctx *context) (coro.Roots, error) { } func activeCoroSchedulerABIVersion(conf *Config) string { + if conf != nil && conf.EnableCoroClosedStaticSpawn { + return coro.SchedulerProgramBootstrapClosedStaticSpawnABIV0 + } if conf != nil && conf.EnableCoroProgramBootstrapRun { return coro.SchedulerProgramBootstrapABIV2 } @@ -1614,7 +1842,13 @@ func activeCoroFuncRepABIVersion(conf *Config) string { // summary. Their fallback SSA stubs remain ignored; ordinary C declarations // outside this compiler-owned closure stay unknown foreign. func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function]struct{}, []requiredCoroDirectPlainCallArgument, map[ssa.CallInstruction]coro.SSAClosedDynamicCallCertificate, error) { - if ctx == nil || ctx.buildConf == nil || !ctx.buildConf.EnableCoroChildAwait { + if ctx == nil || ctx.buildConf == nil { + return nil, nil, nil, nil, nil + } + if ctx.buildConf.EnableCoroClosedStaticSpawn && !ctx.buildConf.EnableCoroProgramBootstrapRun { + return nil, nil, nil, nil, fmt.Errorf("coroutine closed static spawn runtime roots require runnable program bootstrap v2") + } + if !ctx.buildConf.EnableCoroChildAwait { return nil, nil, nil, nil, nil } if ctx.coroSSAEmission == nil || ctx.coroEmission == nil { @@ -1645,6 +1879,10 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function coroFrameAllocatorBootstrapSymbolV1, coroProgramBeginSymbolV1, coroProgramRunSymbolV1, + ) + } + if ctx.buildConf.EnableCoroProgramBootstrapRun { + names = append(names, "__llgo_coro_frame_alloc_v1", "__llgo_coro_frame_publish_v1", "__llgo_coro_await_prepare_v1", @@ -1654,10 +1892,17 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function "__llgo_coro_complete_prepare_v1", "__llgo_coro_frame_free_v1", ) - for _, name := range names[1:] { - demandByName[name] = coro.SyncDemand - plainRootByName[name] = true - } + } + if ctx.buildConf.EnableCoroClosedStaticSpawn { + names = append(names, + "__llgo_coro_spawn_begin_v1", + "__llgo_coro_spawn_commit_v1", + coroProgramMainReturnSymbolV1, + ) + } + for _, name := range names[1:] { + demandByName[name] = coro.SyncDemand + plainRootByName[name] = true } byName := make(map[string]*ssa.Function, len(names)) wanted := make(map[string]struct{}, len(names)) diff --git a/internal/build/collect.go b/internal/build/collect.go index b6b282592f..48bc9279d7 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -378,6 +378,8 @@ func (c *context) canUsePackageCache() bool { c.clCompilation.EnableCoroPhysicalABI == c.buildConf.EnableCoroPhysicalABI && c.clCompilation.EnableCoroChildAwait == c.buildConf.EnableCoroChildAwait && c.clCompilation.EnableCoroPlainDispatch == c.buildConf.EnableCoroPlainDispatch && + c.clCompilation.EnableCoroClosedStaticSpawn == c.buildConf.EnableCoroClosedStaticSpawn && + c.clCompilation.EnableCoroProgramBootstrapRun == c.buildConf.EnableCoroProgramBootstrapRun && c.clCompilation.CoroABI == metadata.CoroABI && c.clCompilation.SchedulerABI == metadata.SchedulerABI && c.clCompilation.PanicABI == metadata.PanicABI && diff --git a/internal/build/coro_bootstrap.go b/internal/build/coro_bootstrap.go index 166e7792f9..65c2949d0e 100644 --- a/internal/build/coro_bootstrap.go +++ b/internal/build/coro_bootstrap.go @@ -42,6 +42,7 @@ const ( coroProgramPublicRuntimeNoopIDV2 coro.FunctionID = "llgo.bootstrap.v2.public-runtime-init.noop" coroProgramBeginSymbolV1 = "__llgo_coro_program_begin_v1" coroProgramRunSymbolV1 = "__llgo_coro_program_run_v1" + coroProgramMainReturnSymbolV1 = "__llgo_coro_program_main_return_v1" // Step kinds and semantic roles are part of the cross-target bootstrap ABI. // Keep these numeric values synchronized with ssa and runtime/internal/coro. @@ -88,6 +89,9 @@ func validateCoroProgramBootstrapConfig(conf *Config) error { if conf == nil { return nil } + if conf.EnableCoroClosedStaticSpawn && !conf.EnableCoroProgramBootstrapRun { + return fmt.Errorf("enable coroutine closed static spawn: runnable program bootstrap v2 is required") + } if conf.EnableCoroProgramBootstrapRun && !conf.EnableCoroProgramBootstrapABI { return fmt.Errorf("enable coroutine program bootstrap runtime: program bootstrap ABI is required") } diff --git a/internal/build/coro_bootstrap_factory.go b/internal/build/coro_bootstrap_factory.go index 2c17df0e8e..96cb087d5e 100644 --- a/internal/build/coro_bootstrap_factory.go +++ b/internal/build/coro_bootstrap_factory.go @@ -177,6 +177,7 @@ func emitCoroProgramBootstrapFactoryV2( bootstrap *coroProgramBootstrapV1, targets []coroProgramBootstrapFactoryTargetV2, finalHash [16]byte, + notifyMainReturn bool, ) llssa.Function { validateCoroProgramBootstrapFactoryV2(pkg, bootstrap, targets) @@ -226,6 +227,12 @@ func emitCoroProgramBootstrapFactoryV2( free := pkg.NewFunc(coroProgramFrameFreeHookV1, newSignature( []types.Type{pointer, pointer, types.Typ[types.Uintptr], types.Typ[types.Uintptr], pointer}, nil, ), llssa.InC) + var mainReturn llssa.Function + if notifyMainReturn { + mainReturn = pkg.NewFunc(coroProgramMainReturnSymbolV1, newSignature( + []types.Type{pointer}, nil, + ), llssa.InC) + } frame := llssa.CoroFrameOps{ Alloc: func(b llssa.Builder, size, align llssa.Expr) llssa.Expr { @@ -302,6 +309,13 @@ func emitCoroProgramBootstrapFactoryV2( b.Store(b.FieldAddr(header, coroProgramHeaderSuspendReasonV1), prog.IntVal(coroProgramSuspendNoneV1, prog.Uint16())) b.Store(b.FieldAddr(header, coroProgramHeaderLifecycleV1), prog.IntVal(coroProgramLifecycleActiveV1, prog.Uint16())) } + // This is deliberately the normal continuation of the exact V2 main + // step, not an entry-module call after program_run. A panic or Goexit + // terminal path never returns through this point, so it cannot be + // mistaken for command-main return and cannot cancel background Gs. + if mainReturn != nil && step.Role == coroProgramStepRoleMainV2 { + b.Call(mainReturn.Expr, g) + } } b.Store(b.FieldAddr(header, coroProgramHeaderSuspendReasonV1), prog.IntVal(coroProgramSuspendFrameCompleteV1, prog.Uint16())) diff --git a/internal/build/coro_bootstrap_factory_test.go b/internal/build/coro_bootstrap_factory_test.go index 5cfb554460..d6ac1f5d4c 100644 --- a/internal/build/coro_bootstrap_factory_test.go +++ b/internal/build/coro_bootstrap_factory_test.go @@ -112,7 +112,7 @@ func TestCoroProgramBootstrapFactoryV2MixedNativeAndWasm(t *testing.T) { defer pkg.Module().Dispose() bootstrap, targets, tableSteps, finalHash := newCoroProgramBootstrapFactoryFixtureV2(pkg) - factory := emitCoroProgramBootstrapFactoryV2(pkg, bootstrap, targets, finalHash) + factory := emitCoroProgramBootstrapFactoryV2(pkg, bootstrap, targets, finalHash, false) pkg.NewCoroProgramBootstrap("__llgo_test_program_bootstrap_v2", llssa.CoroProgramBootstrapOptions{ Version: coroProgramBootstrapVersionV2, ABIHash: finalHash, @@ -159,6 +159,64 @@ func TestCoroProgramBootstrapFactoryV2MixedNativeAndWasm(t *testing.T) { } } +func TestCoroProgramBootstrapFactoryV2MainReturnIsOnlyOnCoroMainContinuation(t *testing.T) { + llssa.Initialize(llssa.InitAll) + prog := llssa.NewProgram(nil) + defer prog.Dispose() + pkg := prog.NewPackage("entry", "entry") + defer pkg.Module().Dispose() + + bootstrap, targets, _, finalHash := newCoroProgramBootstrapFactoryFixtureV2(pkg) + const anchor = "__llgo_coro_root_package_v1.0123456789abcdef0123456789abcdef" + bootstrap.Steps[4] = coroProgramBootstrapStepV1{ + Kind: coroProgramStepCoroRootV1, Role: coroProgramStepRoleMainV2, + FunctionID: "main-coro-id", Target: "example.com/program.main$coro", + Owner: "example.com/program", CatalogTarget: anchor, Aux: 1, + } + targets[4] = coroProgramBootstrapFactoryTargetV2{Anchor: targets[3].Anchor} + factory := emitCoroProgramBootstrapFactoryV2(pkg, bootstrap, targets, finalHash, true) + body := pkg.Module().NamedFunction(factory.Name()).String() + if got := strings.Count(body, "call void @__llgo_coro_await_prepare_v1"); got != 3 { + t.Fatalf("coroutine-main await calls = %d, want 3:\n%s", got, body) + } + if got := strings.Count(body, "call void @"+coroProgramMainReturnSymbolV1); got != 1 { + t.Fatalf("coroutine-main return calls = %d, want 1:\n%s", got, body) + } + lastAwait := strings.LastIndex(body, "call void @__llgo_coro_await_prepare_v1") + mainReturn := strings.Index(body, "call void @"+coroProgramMainReturnSymbolV1) + complete := strings.Index(body, "call void @"+coroProgramCompletePrepareHookV1) + if lastAwait < 0 || mainReturn < 0 || complete < 0 || !(lastAwait < mainReturn && mainReturn < complete) { + t.Fatalf("main-return cancellation is not on the normal post-await continuation:\n%s", body) + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify coroutine-main return factory: %v\n%s", err, pkg.Module().String()) + } +} + +func TestCoroProgramBootstrapFactoryV2MainReturnFollowsPlainMain(t *testing.T) { + llssa.Initialize(llssa.InitAll) + prog := llssa.NewProgram(nil) + defer prog.Dispose() + pkg := prog.NewPackage("entry", "entry") + defer pkg.Module().Dispose() + + bootstrap, targets, _, finalHash := newCoroProgramBootstrapFactoryFixtureV2(pkg) + factory := emitCoroProgramBootstrapFactoryV2(pkg, bootstrap, targets, finalHash, true) + body := pkg.Module().NamedFunction(factory.Name()).String() + if got := strings.Count(body, "call void @"+coroProgramMainReturnSymbolV1); got != 1 { + t.Fatalf("plain-main return calls = %d, want 1:\n%s", got, body) + } + plainMain := strings.Index(body, "call void @\"example.com/program.main\"()") + mainReturn := strings.Index(body, "call void @"+coroProgramMainReturnSymbolV1) + complete := strings.Index(body, "call void @"+coroProgramCompletePrepareHookV1) + if plainMain < 0 || mainReturn < 0 || complete < 0 || !(plainMain < mainReturn && mainReturn < complete) { + t.Fatalf("main-return cancellation is not on the normal post-plain-main continuation:\n%s", body) + } + if err := llvm.VerifyModule(pkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify plain-main return factory: %v\n%s", err, pkg.Module().String()) + } +} + func TestCoroProgramBootstrapFactoryV1RejectsNonCanonicalInputs(t *testing.T) { llssa.Initialize(llssa.InitAll) tests := []struct { @@ -407,6 +465,9 @@ func assertCoroProgramBootstrapFactoryPresplitV2(t *testing.T, ir, uintptrIR str if got := strings.Count(body, "call ptr %"); got != 2 { t.Fatalf("mixed v2 bootstrap indirect child factory calls = %d, want 2:\n%s", got, body) } + if strings.Contains(body, coroProgramMainReturnSymbolV1) { + t.Fatalf("V2 factory without closed-static spawn emitted main-return cancellation:\n%s", body) + } assertInOrder(t, body, "call void @"+coroProgramFramePublishHookV1, "call i8 @llvm.coro.suspend", diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index d5a6056fc5..b08dde0db9 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -417,6 +417,9 @@ func __llgo_coro_yield_prepare_v1() {} func __llgo_coro_park_prepare_v1() {} func __llgo_coro_complete_prepare_v1() {} func __llgo_coro_frame_free_v1() {} +func __llgo_coro_spawn_begin_v1() {} +func __llgo_coro_spawn_commit_v1() {} +func __llgo_coro_program_main_return_v1() {} func bootstrapHelper() { closureLoop(); externalABI(); inlineIntrinsic("bootstrap") } func closureLoop() { for i := 0; i < 2; i++ {} } func unrelatedLoop() { for {} } @@ -485,6 +488,38 @@ func atomicExchange(*uint32, uint32) uint32 t.Fatalf("required root %d = %+v, want %s/%s", index, root, wantRoots[index], wantDemand) } } + spawnCtx := *ctx + spawnCtx.buildConf = &Config{ + EnableCoroChildAwait: true, + EnableCoroProgramBootstrapRun: true, + EnableCoroClosedStaticSpawn: true, + } + spawnRoots, spawnPlain, _, _, err := requiredCoroProgramRuntimePlan(&spawnCtx) + if err != nil { + t.Fatal(err) + } + if len(spawnRoots) != len(wantRoots)+3 { + t.Fatalf("closed-static-spawn runtime roots = %d, want %d", len(spawnRoots), len(wantRoots)+3) + } + for _, name := range []string{"__llgo_coro_spawn_begin_v1", "__llgo_coro_spawn_commit_v1", coroProgramMainReturnSymbolV1} { + fn := ssaPkg.Func(name) + if fn == nil { + t.Fatalf("closed-static-spawn runtime hook %q is absent", name) + } + if _, ok := spawnPlain[fn]; !ok { + t.Fatalf("closed-static-spawn runtime hook %q is not a required plain root", name) + } + found := false + for _, root := range spawnRoots { + if root.Function == fn && root.Demand == coro.SyncDemand { + found = true + break + } + } + if !found { + t.Fatalf("closed-static-spawn runtime hook %q has no sync root", name) + } + } if _, ok := requiredPlain[ssaPkg.Func("init")]; ok { t.Fatal("managed runtime.init leaked into the native required-plain island") } @@ -1759,6 +1794,7 @@ func TestActiveCoroABIVersions(t *testing.T) { {"physical leaf", &Config{EnableCoroPhysicalABI: true}, coro.PhysicalABIV0, coro.SchedulerNoneABIV0, coro.FuncRepABIV0}, {"plain dispatch", &Config{EnableCoroPlainDispatch: true}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0, coro.FuncRepABIV1}, {"child await", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true}, coro.PhysicalABIV1, coro.SchedulerChildAwaitABIV0, coro.FuncRepABIV0}, + {"closed static spawn", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroClosedStaticSpawn: true, EnableCoroProgramBootstrapRun: true}, coro.PhysicalABIV1, coro.SchedulerProgramBootstrapClosedStaticSpawnABIV0, coro.FuncRepABIV0}, {"program bootstrap runtime with plain dispatch", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroPlainDispatch: true, EnableCoroProgramBootstrapRun: true}, coro.PhysicalABIV1, coro.SchedulerProgramBootstrapABIV2, coro.FuncRepABIV1}, } for _, test := range tests { @@ -2173,6 +2209,11 @@ func TestCoroEntryResolutionUsesPlanMatchedPackageCache(t *testing.T) { if dispatchCtx.canUsePackageCache() { t.Fatal("plain-dispatch capability mismatch unexpectedly permits package cache") } + bootstrapMismatch := newContext(digestA) + bootstrapMismatch.clCompilation.EnableCoroProgramBootstrapRun = true + if bootstrapMismatch.canUsePackageCache() { + t.Fatal("program-bootstrap-run capability mismatch unexpectedly permits package cache") + } if !matchingPkg.NeedRt || !matchingPkg.NeedPyInit { t.Fatalf("cache metadata runtime flags = %v/%v, want true/true", matchingPkg.NeedRt, matchingPkg.NeedPyInit) } diff --git a/internal/build/coro_spawn_test.go b/internal/build/coro_spawn_test.go new file mode 100644 index 0000000000..46608bf751 --- /dev/null +++ b/internal/build/coro_spawn_test.go @@ -0,0 +1,232 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "strings" + "testing" + + "github.com/goplus/llgo/internal/coro" + "golang.org/x/tools/go/ssa" +) + +func TestCoroPlanInputClosedStaticSpawnSeedsOwnerAndPreservesTargetPrimary(t *testing.T) { + ssaPkg, _ := buildCoroPlanTestPackage(t, "example.com/spawn", `package spawn +var channel chan int +func plain(value int) { _ = value } +func suspending() { <-channel } +func launchPlain(value int) { plain(value); go plain(value) } +func launchSuspending() { go suspending() } +`, nil) + launchPlain := ssaPkg.Func("launchPlain") + launchSuspending := ssaPkg.Func("launchSuspending") + input := CoroPlanInput{Program: ssaPkg.Prog, enableClosedStaticSpawn: true} + plan, err := input.Analyze(coro.Roots{ + {Function: launchPlain, Demand: coro.AsyncDemand}, + {Function: launchSuspending, Demand: coro.AsyncDemand}, + }, coro.SSAConfig{MaxPlainInstructions: -1}) + if err != nil { + t.Fatal(err) + } + + plainPlan, _ := plan.FunctionPlan(ssaPkg.Func("plain")) + if plainPlan.Emission != coro.EmitCoroutine || plainPlan.Primary != coro.PrimaryCoroutine || plainPlan.FuncRep != coro.DirectCoro || + plainPlan.Demand != coro.AsyncDemand || !plainPlan.Effect.Contains(coro.YieldOnly) { + t.Fatalf("plain sync+spawn target = %+v", plainPlan) + } + suspendingPlan, _ := plan.FunctionPlan(ssaPkg.Func("suspending")) + if suspendingPlan.Emission != coro.EmitCoroutine || suspendingPlan.Primary != coro.PrimaryCoroutine || + suspendingPlan.FuncRep != coro.DirectCoro || suspendingPlan.Demand != coro.AsyncDemand { + t.Fatalf("suspending spawn target = %+v", suspendingPlan) + } + for _, owner := range []*ssa.Function{launchPlain, launchSuspending} { + ownerPlan, _ := plan.FunctionPlan(owner) + if ownerPlan.DeclaredEffect != coro.YieldOnly || !ownerPlan.LocalEffect.Contains(coro.YieldOnly) || + !ownerPlan.Effect.Contains(coro.YieldOnly) || ownerPlan.Emission != coro.EmitCoroutine || + ownerPlan.Primary != coro.PrimaryCoroutine || ownerPlan.FuncRep != coro.DirectCoro || ownerPlan.Demand != coro.AsyncDemand { + t.Fatalf("owner %s = %+v", owner.Name(), ownerPlan) + } + for _, call := range coroPlanTestCalls(owner) { + spawn, ok := call.(*ssa.Go) + if !ok { + continue + } + if _, _, err := plan.ResolveClosedStaticSpawn(spawn); err != nil { + t.Fatalf("resolve %s spawn: %v", owner.Name(), err) + } + callPlan, ok := plan.CallPlan(spawn) + if !ok || callPlan.Kind != coro.CallSpawn || callPlan.Open || callPlan.MayBeNil || len(callPlan.Targets) != 1 { + t.Fatalf("owner %s spawn CallPlan = %+v, present=%v", owner.Name(), callPlan, ok) + } + } + } + if !coroPlanContainsSpawn(plan) { + t.Fatal("emitted plan lost its spawn site") + } + if err := validateCoroClosedStaticSpawnRunGate(&Config{EnableCoroClosedStaticSpawn: true}, plan); err == nil || !strings.Contains(err.Error(), "runnable program bootstrap v2") { + t.Fatalf("non-runnable spawn gate error = %v", err) + } + err = validateCoroClosedStaticSpawnRunGate(&Config{ + EnableCoroClosedStaticSpawn: true, + EnableCoroProgramBootstrapRun: true, + }, plan) + if err == nil || !strings.Contains(err.Error(), "may-park") || !strings.Contains(err.Error(), "main-return cancellation subset") { + t.Fatalf("runnable spawn gate error = %v", err) + } +} + +func TestCoroClosedStaticSpawnRunGateEffectSubset(t *testing.T) { + tests := []struct { + name string + effect coro.Effect + wantOK bool + wantDetail string + }{ + {name: "yield", effect: coro.YieldOnly, wantOK: true}, + {name: "structured await", effect: coro.YieldOnly | coro.AwaitStructured, wantOK: true}, + {name: "missing yield", effect: coro.AwaitStructured, wantDetail: "await-structured"}, + {name: "park", effect: coro.YieldOnly | coro.MayPark, wantDetail: "may-park"}, + {name: "platform wait", effect: coro.YieldOnly | coro.WaitPlatform, wantDetail: "wait-platform"}, + {name: "host wait", effect: coro.YieldOnly | coro.WaitHost, wantDetail: "wait-host"}, + {name: "foreign wait", effect: coro.YieldOnly | coro.WaitForeign, wantDetail: "wait-foreign"}, + {name: "opaque", effect: coro.OpaqueSuspend, wantDetail: "opaque-suspend"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ssaPkg, _ := buildCoroPlanTestPackage(t, "example.com/spawngate", `package spawngate +func target() {} +func launch() { go target() } +`, nil) + launch, target := ssaPkg.Func("launch"), ssaPkg.Func("target") + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: launch, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + switch fn { + case launch: + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + case target: + return coro.SSAFunctionPolicy{Effect: test.effect}, nil + default: + return coro.SSAFunctionPolicy{}, nil + } + }, + }) + if err != nil { + t.Fatal(err) + } + err = validateCoroClosedStaticSpawnRunGate(&Config{ + EnableCoroClosedStaticSpawn: true, + EnableCoroProgramBootstrapRun: true, + }, plan) + if test.wantOK { + if err != nil { + t.Fatalf("safe runnable spawn rejected: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.wantDetail) { + t.Fatalf("gate error = %v, want detail %q", err, test.wantDetail) + } + }) + } +} + +func TestCoroPlanInputClosedStaticSpawnFailsClosedOnUnsupportedShapes(t *testing.T) { + tests := []struct { + name string + source string + want string + }{ + { + name: "captured closure", + source: `package spawn; func launch(value int) { go func() { _ = value }() }`, + want: "closures, methods, interfaces, and function values", + }, + { + name: "method", + source: `package spawn +type worker int +func (worker) run() {} +func launch(value worker) { go value.run() } +`, + want: "non-method", + }, + { + name: "dynamic function value", + source: `package spawn; func launch(fn func()) { go fn() }`, + want: "closures, methods, interfaces, and function values", + }, + { + name: "discarded result capability", + source: `package spawn; func worker() int { return 1 }; func launch() { go worker() }`, + want: "zero-result signature", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ssaPkg, _ := buildCoroPlanTestPackage(t, "example.com/spawn", test.source, nil) + input := CoroPlanInput{Program: ssaPkg.Prog, enableClosedStaticSpawn: true} + _, err := input.Analyze(coro.Roots{{Function: ssaPkg.Func("launch"), Demand: coro.AsyncDemand}}, coro.SSAConfig{MaxPlainInstructions: -1}) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %v, want substring %q", err, test.want) + } + }) + } +} + +func TestBuildCoroPlanClosedStaticSpawnCapabilityDependencies(t *testing.T) { + tests := []struct { + name string + conf *Config + want string + }{ + { + name: "runnable bootstrap", + conf: &Config{EnableCoroClosedStaticSpawn: true}, + want: "runnable program bootstrap v2 is required", + }, + { + name: "bootstrap ABI", + conf: &Config{ + EnableCoroClosedStaticSpawn: true, + EnableCoroProgramBootstrapRun: true, + EnableCoroChildAwait: true, + }, + want: "program bootstrap ABI is required", + }, + { + name: "child await", + conf: &Config{ + BuildMode: BuildModeExe, + EnableCoroEntryResolution: true, EnableCoroPhysicalABI: true, + EnableCoroProgramBootstrapABI: true, EnableCoroProgramBootstrapRun: true, + EnableCoroClosedStaticSpawn: true, + }, + want: "coroutine child await is required", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := buildCoroPlan(&context{buildConf: test.conf}) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("dependency error = %v, want %q", err, test.want) + } + }) + } +} diff --git a/internal/build/main_module.go b/internal/build/main_module.go index e5c30d51d1..4fc4bb6cc3 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -289,7 +289,10 @@ func emitCoroProgramManifest(ctx *context, pkg llssa.Package, cfg *genConfig) co } } if ctx.buildConf.EnableCoroProgramBootstrapRun { - factory = emitCoroProgramBootstrapFactoryV2(pkg, cfg.coroBootstrap, targets, cfg.coroManifestHash) + factory = emitCoroProgramBootstrapFactoryV2( + pkg, cfg.coroBootstrap, targets, cfg.coroManifestHash, + ctx.buildConf.EnableCoroClosedStaticSpawn, + ) } } else { targets := make([]llssa.Function, len(cfg.coroBootstrap.Steps)) diff --git a/internal/build/main_module_test.go b/internal/build/main_module_test.go index e4fb8b6b08..b03b8b5830 100644 --- a/internal/build/main_module_test.go +++ b/internal/build/main_module_test.go @@ -397,6 +397,7 @@ func TestGenMainModuleCoroProgramBootstrapV2MixedNativeAndWasm(t *testing.T) { EnableCoroChildAwait: true, EnableCoroProgramBootstrapABI: true, EnableCoroProgramBootstrapRun: true, + EnableCoroClosedStaticSpawn: true, }, } const anchor = "__llgo_coro_root_package_v1.0123456789abcdef0123456789abcdef" @@ -485,6 +486,9 @@ func TestGenMainModuleCoroProgramBootstrapV2MixedNativeAndWasm(t *testing.T) { if got := strings.Count(factoryBody, "call void @__llgo_coro_await_prepare_v1"); got != 2 { t.Fatalf("mixed v2 main-module factory await calls = %d, want 2:\n%s", got, factoryBody) } + if got := strings.Count(factoryBody, "call void @"+coroProgramMainReturnSymbolV1); got != 1 { + t.Fatalf("mixed v2 main-module main-return calls = %d, want 1:\n%s", got, factoryBody) + } assertInOrder(t, factoryBody, "call ptr %", "call void @__llgo_coro_await_prepare_v1", @@ -493,6 +497,7 @@ func TestGenMainModuleCoroProgramBootstrapV2MixedNativeAndWasm(t *testing.T) { "call ptr %", "call void @__llgo_coro_await_prepare_v1", "call void @\"example.com/foo.main\"()", + "call void @"+coroProgramMainReturnSymbolV1, "call void @"+coroProgramCompletePrepareHookV1, ) @@ -584,6 +589,9 @@ func TestGenMainModuleCoroProgramBootstrapV2DefinesOnlyOwnedPublicRuntimeNoop(t if function := module.NamedFunction("syscall.init"); !function.IsNil() { t.Fatalf("managed V2 entry retained a weak syscall.init interception body:\n%s", module.String()) } + if function := module.NamedFunction(coroProgramMainReturnSymbolV1); !function.IsNil() { + t.Fatalf("V2 bootstrap without closed-static spawn declared main-return cancellation:\n%s", module.String()) + } if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { t.Fatalf("verify absent-public-runtime v2 module: %v\n%s", err, module.String()) } diff --git a/internal/coro/func_flow.go b/internal/coro/func_flow.go index 88bd81e615..af3aeb044e 100644 --- a/internal/coro/func_flow.go +++ b/internal/coro/func_flow.go @@ -109,6 +109,76 @@ func (p *SSAPlan) CallPlan(call ssa.CallInstruction) (SSACallPlan, bool) { return plan, true } +// ResolveClosedStaticSpawn proves the exact source and whole-plan shape used +// by the first stackless goroutine-spawn lowering. The target is selected by +// the immutable CallPlan, never by a display name or a runtime callback. The +// target must have one coroutine primary even when its source body is bounded: +// this preserves preemption if that goroutine becomes CPU-heavy and lets sync +// callers reuse the same body through ordinary async-effect propagation. +func (p *SSAPlan) ResolveClosedStaticSpawn(call *ssa.Go) (*ssa.Function, FunctionPlan, error) { + if p == nil || call == nil || call.Common() == nil { + return nil, FunctionPlan{}, fmt.Errorf("requires a compilation CallPlan") + } + common := call.Common() + raw, direct := common.Value.(*ssa.Function) + if !direct || raw == nil || common.IsInvoke() || common.Method != nil || common.StaticCallee() != raw { + return nil, FunctionPlan{}, fmt.Errorf("requires an exact static top-level function operand") + } + callPlan, ok := p.CallPlan(call) + if !ok { + return nil, FunctionPlan{}, fmt.Errorf("spawn has no compilation CallPlan") + } + if callPlan.Kind != CallSpawn || callPlan.Open || callPlan.MayBeNil || len(callPlan.Targets) != 1 { + return nil, FunctionPlan{}, fmt.Errorf( + "requires one closed non-nil spawn target, got kind=%v open=%t may-be-nil=%t targets=%d", + callPlan.Kind, callPlan.Open, callPlan.MayBeNil, len(callPlan.Targets), + ) + } + target, ok := p.Function(callPlan.Targets[0]) + if !ok || target == nil { + return nil, FunctionPlan{}, fmt.Errorf("spawn target %q is absent from the compilation plan", callPlan.Targets[0]) + } + targetPlan, ok := p.FunctionPlan(target) + if !ok || targetPlan.ID != callPlan.Targets[0] { + return nil, FunctionPlan{}, fmt.Errorf("spawn target %q has no canonical function plan", callPlan.Targets[0]) + } + if target.Parent() != nil || len(target.FreeVars) != 0 || target.Synthetic != "" || target.Origin() != nil || len(target.TypeArgs()) != 0 { + return nil, FunctionPlan{}, fmt.Errorf("target %q is not an exact non-capturing top-level function", targetPlan.ID) + } + if params := target.TypeParams(); params != nil && params.Len() != 0 { + return nil, FunctionPlan{}, fmt.Errorf("target %q is a generic declaration", targetPlan.ID) + } + sig := target.Signature + if sig == nil || sig.Recv() != nil || sig.Variadic() || sig.Results().Len() != 0 || + (sig.TypeParams() != nil && sig.TypeParams().Len() != 0) || + (sig.RecvTypeParams() != nil && sig.RecvTypeParams().Len() != 0) { + return nil, FunctionPlan{}, fmt.Errorf("target %q must have one non-method, non-variadic, zero-result signature", targetPlan.ID) + } + if targetPlan.External != Defined || targetPlan.Demand != AsyncDemand { + return nil, FunctionPlan{}, fmt.Errorf( + "target %q is not one demanded defined async root (external=%s demand=%s)", + targetPlan.ID, targetPlan.External, targetPlan.Demand, + ) + } + if targetPlan.Emission != EmitCoroutine || targetPlan.Primary != PrimaryCoroutine || targetPlan.FuncRep != DirectCoro || + !targetPlan.Effect.Contains(YieldOnly) || callPlan.Rep != DirectCoro { + return nil, FunctionPlan{}, fmt.Errorf( + "target %q is not one preemptible direct coroutine primary (emission=%s primary=%s representation=%s effect=%s call-representation=%s)", + targetPlan.ID, targetPlan.Emission, targetPlan.Primary, targetPlan.FuncRep, targetPlan.Effect, callPlan.Rep, + ) + } + caller := call.Parent() + callerPlan, ok := p.FunctionPlan(caller) + if !ok || callerPlan.Emission != EmitCoroutine || callerPlan.Primary != PrimaryCoroutine || callerPlan.FuncRep != DirectCoro || + callerPlan.Demand != AsyncDemand || !callerPlan.Effect.Contains(YieldOnly) { + return nil, FunctionPlan{}, fmt.Errorf( + "spawn owner is not one async-only contextful coroutine primary (emission=%s primary=%s representation=%s demand=%s effect=%s)", + callerPlan.Emission, callerPlan.Primary, callerPlan.FuncRep, callerPlan.Demand, callerPlan.Effect, + ) + } + return target, targetPlan, nil +} + // ElidesCall reports whether trusted frontend policy proved that the exact SSA // declaration call emits no callable edge. The source operation may be omitted, // lowered inline, or replaced by separately frozen lowered calls. Elided calls diff --git a/internal/coro/plan_digest.go b/internal/coro/plan_digest.go index 86bcc754b8..d84f80a453 100644 --- a/internal/coro/plan_digest.go +++ b/internal/coro/plan_digest.go @@ -54,8 +54,14 @@ const ( // contract. It still does not claim spawn, park, timers, or a production // source of concurrent runnable Gs. SchedulerProgramBootstrapABIV2 = "llgo.coro.scheduler.program-bootstrap.v2" - PanicLegacyABIV0 = "llgo.coro.panic.legacy.v0" - FuncRepABIV0 = "llgo.coro.func-rep.v0" + // SchedulerProgramBootstrapClosedStaticSpawnABIV0 is the explicit superset + // of SchedulerProgramBootstrapABIV2 that adds compiler-owned begin/commit + // for one exact closed static `go f(args)` target and normal-main-return + // cancellation. The runtime never receives a user callback; the compiler + // creates the child only to its initial suspend before commit. + SchedulerProgramBootstrapClosedStaticSpawnABIV0 = "llgo.coro.scheduler.program-bootstrap.v2.closed-static-spawn.v0" + PanicLegacyABIV0 = "llgo.coro.panic.legacy.v0" + FuncRepABIV0 = "llgo.coro.func-rep.v0" // FuncRepABIV1 introduces an explicit descriptor/context representation for // dynamically consumed Go function values. The first producer/consumer slice // supports only one no-capture, non-suspending plain body; unsupported value diff --git a/internal/coro/plan_digest_test.go b/internal/coro/plan_digest_test.go index 09b62a6173..32954d32ab 100644 --- a/internal/coro/plan_digest_test.go +++ b/internal/coro/plan_digest_test.go @@ -146,6 +146,74 @@ func TestCoroPlanDigestDeterministicCompleteAndDomainSeparated(t *testing.T) { } } +func TestCoroPlanDigestRecordsClosedStaticSpawnConsumerAndOwnerSeed(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "spawn_digest.go", `package coroid +func worker(value int) { _ = value } +func launch(value int) { go worker(value) } +`) + launch := packageFunction(t, pkg, "launch") + worker := packageFunction(t, pkg, "worker") + build := func(seed bool) *SSAPlan { + config := planDigestSSAConfig() + config.FunctionIDs.CoroABI = PhysicalABIV1 + config.FunctionIDs.SchedulerABI = SchedulerProgramBootstrapClosedStaticSpawnABIV0 + config.MaxPlainInstructions = -1 + if seed { + config.ClassifyFunction = func(fn *ssa.Function) (SSAFunctionPolicy, error) { + if fn == launch || fn == worker { + return SSAFunctionPolicy{Effect: YieldOnly}, nil + } + return SSAFunctionPolicy{}, nil + } + } + plan, err := AnalyzeSSA(prog, Roots{{Function: launch, Demand: AsyncDemand}}, config) + if err != nil { + t.Fatal(err) + } + return plan + } + seeded := build(true) + again := build(true) + unseeded := build(false) + metadata := validPlanDigestMetadata() + metadata.CoroABI = PhysicalABIV1 + metadata.SchedulerABI = SchedulerProgramBootstrapClosedStaticSpawnABIV0 + digest, err := seeded.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + againDigest, err := again.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if digest != againDigest { + t.Fatalf("closed static spawn digest is unstable: %s != %s", digest, againDigest) + } + unseededDigest, err := unseeded.CoroPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + if digest == unseededDigest { + t.Fatal("spawn owner YieldOnly/contextful-primary seed is absent from the digest") + } + document, err := seeded.canonicalPlanDigest(metadata) + if err != nil { + t.Fatal(err) + } + found := false + for _, call := range document.Calls { + if CallKind(call.Kind) == CallSpawn { + found = true + if call.Open || call.MayBeNil || len(call.Targets) != 1 { + t.Fatalf("spawn digest call = %+v", call) + } + } + } + if !found { + t.Fatal("canonical plan digest has no exact CallSpawn consumer") + } +} + func TestCoroPlanDigestRecordsIgnoredPhysicalBodySemantics(t *testing.T) { prog, pkg := buildCoroTestSSA(t, "ignored_digest.go", `package coroid func external() {} diff --git a/internal/coro/ssa_plan_test.go b/internal/coro/ssa_plan_test.go index 98123763d3..6ff4cb1914 100644 --- a/internal/coro/ssa_plan_test.go +++ b/internal/coro/ssa_plan_test.go @@ -116,6 +116,97 @@ func send(ch chan int) { ch <- 1 } } } +func TestSSAPlanResolvesOnlyClosedStaticSpawnAndKeepsOnePreemptibleTargetPrimary(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "spawn.go", `package coroid +var ch chan int +func plain(value int) { _ = value } +func suspending() { <-ch } +func launchPlain(value int) { plain(value); go plain(value) } +func launchSuspending() { go suspending() } +`) + plain := packageFunction(t, pkg, "plain") + suspending := packageFunction(t, pkg, "suspending") + launchPlain := packageFunction(t, pkg, "launchPlain") + launchSuspending := packageFunction(t, pkg, "launchSuspending") + plan, err := AnalyzeSSA(prog, Roots{ + {Function: launchPlain, Demand: AsyncDemand}, + {Function: launchSuspending, Demand: AsyncDemand}, + }, SSAConfig{ + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (SSAFunctionPolicy, error) { + if fn == launchPlain || fn == launchSuspending || fn == plain || fn == suspending { + return SSAFunctionPolicy{Effect: YieldOnly}, nil + } + return SSAFunctionPolicy{}, nil + }, + }) + if err != nil { + t.Fatal(err) + } + if got := functionPlanFor(t, plan, plain); got.Emission != EmitCoroutine || got.Primary != PrimaryCoroutine || got.FuncRep != DirectCoro || + got.Demand != AsyncDemand || !got.Effect.Contains(YieldOnly) { + t.Fatalf("bounded target plan = %+v, want one sync+spawn preemptible coroutine primary", got) + } + if got := functionPlanFor(t, plan, suspending); got.Emission != EmitCoroutine || got.Primary != PrimaryCoroutine || + got.FuncRep != DirectCoro || got.Demand != AsyncDemand { + t.Fatalf("suspending target plan = %+v, want one async coroutine primary", got) + } + for _, owner := range []*ssa.Function{launchPlain, launchSuspending} { + ownerPlan := functionPlanFor(t, plan, owner) + if ownerPlan.DeclaredEffect != YieldOnly || !ownerPlan.LocalEffect.Contains(YieldOnly) || !ownerPlan.Effect.Contains(YieldOnly) || + ownerPlan.Emission != EmitCoroutine || ownerPlan.FuncRep != DirectCoro || ownerPlan.Demand != AsyncDemand { + t.Fatalf("spawn owner %s plan = %+v", owner.Name(), ownerPlan) + } + var spawn *ssa.Go + for _, block := range owner.Blocks { + for _, instruction := range block.Instrs { + if candidate, ok := instruction.(*ssa.Go); ok { + spawn = candidate + } + } + } + if spawn == nil { + t.Fatalf("spawn owner %s has no ssa.Go", owner.Name()) + } + target, targetPlan, err := plan.ResolveClosedStaticSpawn(spawn) + if err != nil { + t.Fatalf("resolve spawn in %s: %v", owner.Name(), err) + } + callPlan, ok := plan.CallPlan(spawn) + if !ok || callPlan.Kind != CallSpawn || callPlan.Open || callPlan.MayBeNil || len(callPlan.Targets) != 1 || + targetPlan.Demand != AsyncDemand { + t.Fatalf("spawn in %s call/target plan = %+v / %+v", owner.Name(), callPlan, targetPlan) + } + if owner == launchPlain && target != plain || owner == launchSuspending && target != suspending { + t.Fatalf("spawn in %s target = %v", owner.Name(), target) + } + } + + bothPlan, err := AnalyzeSSA(prog, Roots{{Function: launchPlain, Demand: BothDemand}}, SSAConfig{ + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (SSAFunctionPolicy, error) { + if fn == launchPlain || fn == plain { + return SSAFunctionPolicy{Effect: YieldOnly}, nil + } + return SSAFunctionPolicy{}, nil + }, + }) + if err != nil { + t.Fatal(err) + } + var bothSpawn *ssa.Go + for _, block := range launchPlain.Blocks { + for _, instruction := range block.Instrs { + if spawn, ok := instruction.(*ssa.Go); ok { + bothSpawn = spawn + } + } + } + if _, _, err := bothPlan.ResolveClosedStaticSpawn(bothSpawn); err == nil || !strings.Contains(err.Error(), "async-only") { + t.Fatalf("BothDemand spawn owner error = %v, want async-only fail-closed", err) + } +} + func TestSSAPlanRootsCanonicalJoinedSortedAndDefensive(t *testing.T) { prog, pkg := buildCoroTestSSA(t, "roots.go", `package coroid func original() {} From 14fb36e9c7753b6ec3f29c4af8f04650f3a3f5de Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 23:29:10 +0800 Subject: [PATCH 22/32] docs(coro): record closed static spawn prototype --- .github/workflows/coroutine.yml | 4 ++-- doc/llvm-coro-runtime-design.md | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/coroutine.yml b/.github/workflows/coroutine.yml index 1b92bbdea4..0a1c4ba1b9 100644 --- a/.github/workflows/coroutine.yml +++ b/.github/workflows/coroutine.yml @@ -59,14 +59,14 @@ jobs: ./internal/runtime/coro_program.go \ ./internal/runtime/coro_sched.go \ ./internal/runtime/coro_program_test.go \ - -run '^TestCoroProgram(V1|V2)' -count=1 + -run '^TestCoroProgram' -count=1 GOOS=js GOARCH=wasm CGO_ENABLED=0 go test \ -tags=coro_runtime_adapter_test \ -exec="$(go env GOROOT)/lib/wasm/go_js_wasm_exec" \ ./internal/runtime/coro_program.go \ ./internal/runtime/coro_sched.go \ ./internal/runtime/coro_program_test.go \ - -run '^TestCoroProgram(V1|V2)' -count=1 + -run '^TestCoroProgram' -count=1 - name: Link named freestanding WebAssembly targets if: matrix.llvm == 19 && matrix.go == '1.24.2' diff --git a/doc/llvm-coro-runtime-design.md b/doc/llvm-coro-runtime-design.md index ef3efa09cb..7d4b3117bf 100644 --- a/doc/llvm-coro-runtime-design.md +++ b/doc/llvm-coro-runtime-design.md @@ -1778,7 +1778,7 @@ Pure sync library/archive不需要链接scheduler。Executable一旦选择 `-sch 验收：纯 sync chain 只有 `F`；纯 async chain 只有 `F$coro`；动态 escape 才出现 descriptor/adapter；所有 `go` root和可挂起call都以LLVM-coro frame表示。 -当前落地状态（2026-07-16，实验 physical ABI v0/v1；scheduler ABI `llgo.coro.scheduler.program-bootstrap.v2`）： +当前落地状态（2026-07-16，实验 physical ABI v0/v1；scheduler ABI 已扩展到 `llgo.coro.scheduler.program-bootstrap.v2.closed-static-spawn.v0`）： - 全程序 SSA 的 Effect、Demand、FuncRep、稳定 FunctionID、精确 emission universe、单 primary symbol 选择和 `CoroPlanDigest` 已落地。明确 plain 或 coro 的函数仍只有一个主体；仅真正动态的 func/`any`/interface consumer 才进入 descriptor/dispatch。缺失、过期或目标布局不匹配的计划与 cache manifest 均 fail closed。 - LLGo 已固定使用 `cpunion/llvm` PR #5 的 LLVM 19–22 绑定。该分支吸收上游 LLVM 22 的完整 switch API 变更，并保留 LLGo 所需的 switched-resume builder/CoroSplit API；19、20、21、22 CI 均通过。LLGo 不再覆盖 LLVM 19 以下版本。 @@ -1789,14 +1789,16 @@ Pure sync library/archive不需要链接scheduler。Executable一旦选择 `-sch - frozen foreign `//llgo:coro noblock` certificate 当前只授予已审计的 `time`、`pthread_self`、`pthread_mutex_init` 和 `pthread_mutex_unlock`。证书只移除未知阻塞，`IRQUnsafe` 仍保留但允许在普通 G 上执行。真实 runtime init 仍被 `pthread_key_create`、`rand`/`srand`、`GC_malloc`、mutex lock、Memcpy/Memset 等未完成边界挡住。 - legacy PanicABI 仍是完整启动链的正式 blocker。exact proof 可追踪 `runtime.Panic → Rethrow → TracePanic → printany`，并在动态 `error.Error` 调用处停止；这里必须落地 non-legacy task-local PanicABI/descriptor dispatch，不能把动态调用误标为 plain。 - 多基本块 CFG、聚合值、PHI 和抢占 lowering 已完成。自然循环、循环入口及每 64 条有效指令的长直线块插入 poll；scheduler 的 P 级原子 request 只有在 slow path 才执行 publish/yield/`llvm.coro.suspend`，fast path 不切换。LLVM 19–22 上均有 native64/wasm32 pre-/post-CoroSplit 与 object 测试。 +- 第一条 production `go` 路径已经落地：严格限定为 closed static、top-level、非捕获、非泛型、非变参、零返回的 `go f(args)`。编译器先按 Go 顺序完整求值参数，再以显式 parent G 执行 begin，调用 target 唯一的 `DirectCoro` primary 到 LLVM initial suspend，commit 后在 parent 上 poll/yield；runtime 不接收用户 callback，也不依赖 TLS。owner 与 target 都由精确 `YieldOnly` seed 进入 effect 传播，因此 target 即使当前很短也保留抢占点，普通同步 caller 则透明 await 同一主体。 +- Command `main` 的正常 continuation 现在显式通知 runtime。main root 完成后，single-P shutdown 先整体校验 ready/wait/current/action 状态，再封闭调度 gate，按 FIFO 取 ready G、按 active-child 到 root 顺序直接 `llvm.coro.destroy`，最后每个 task storage 只释放一次。该 v1 路径只接收 `YieldOnly|AwaitStructured` target 且拒绝非空 wait set；panic/Goexit 不经过正常 main-return hook。 - park/wake handshake 已落地 32-bit 原子 `WaitToken`、generation ticket、early/late completion、唯一 waiter claim、ABA 范围校验及 terminal gate。精确 intrinsic `llgo.coroPark(token, ticket)` 被 Effect 分析识别为 `MayPark`，并在调用者当前 LLVM frame 中生成 park prepare、stateID、`coro.suspend` 和恢复路径；没有隐藏在普通同步 helper 中。channel/timer/syscall 的 submit/retry producer 尚未接入。 - wait/preempt core 要求目标提供可靠的 32-bit atomic load/store/CAS。WASM 可直接满足；带 A 扩展的 RISC-V 可满足；ESP32-C3 RV32IMC 当前会在链接时缺少 `__atomic_*_4`，直到平台用 IRQ critical section 提供单核适配。这里故意不使用非原子 fallback。 - `wasip1`、`wasip2` 和 `wasm-unknown` 明确选择 leaking/nogc frame backend，不依赖 libuv 或 BDWGC。`wasip2` 与 `wasm-unknown` 已通过真实 `llgo build -target=...`、wasm magic/symbol closure、无 `GC_*`/undefined 检查，并由 wasmtime 运行返回 0。当前 `wasip2` 产物是 Preview 2 目标的 core module，尚不是 WIT component。 - frame allocator 已有 conservative BDWGC、nogc/WASM malloc 和 tinygogc/baremetal 后端。跨 suspend 的 pointer 目前只在 conservative 或 non-collecting 配置下安全；精确 frame root map、write barrier、STW、weak timer/finalizer 与 cleanup 语义尚未实现，不能据此宣称完整 Go GC 兼容。 -- deterministic single-P runtime 已能管理多个 frame、ready queue、preempt request、park/wake 和 terminal idle/requested/disabled 状态，但 production program 目前仍只有静态 bootstrap G。尚无 `go` spawn/newG、真实 tick/alarm request source、channel/select/sync slow path、timer/netpoll、异步 syscall submit/retry、task-local panic/defer/recover/Goexit 或多 P。 +- deterministic single-P runtime 已能管理多个 frame、ready queue、preempt request、park/wake、closed-static spawned G、正常 main-return ready-child cancellation 和 terminal idle/requested/stopping/disabled 状态。尚无动态/closure/method `go` target、等待中 G 的 producer 解注册与取消、真实 tick/alarm request source、channel/select/sync slow path、timer/netpoll、异步 syscall submit/retry、task-local panic/defer/recover/Goexit 或多 P。 - 完整真实 `entry → allocator → v2 factory → runtime/package init → main → scheduler` linked smoke 仍受上述 runtime/Panic/foreign blockers 限制；现有 runtime adapter 测试和 freestanding wasm CLI fixture 分别证明 scheduler ABI 与目标链接，不能合并表述为完整 Go runtime 已经端到端运行。 - 当前 cache digest 只解决同一完整程序计划下的内部 package cache；未知未来 caller 可复用的预编译 archive/标准库仍需 producer summary、canonical boundary Dispatch 和 linker ABI 校验。 -- 后续依赖顺序是：closed static `go f(args)`/newG 与真实 platform request source；随后接 channel/timer/syscall producer 并跑完整 linked smoke；并行实现 non-legacy PanicABI；再补 suspended-frame GC、defer/recover/Goexit、多 P 与各 target event backend。所有阶段保持无栈、单 primary 和未证明即 fail closed。 +- 后续依赖顺序是：先解除完整 runtime 链的 non-legacy PanicABI/动态 `error.Error` blocker，并为 WaitToken 增加可注销、可静默迟到 completion 的稳定 registration；再接真实 platform request source 与 channel/timer/syscall producer并跑完整 linked smoke；随后补 suspended-frame GC、defer/recover/Goexit、多 P 与各 target event backend。动态/closure/method `go` target只在 canonical descriptor transport 完成后开启。所有阶段保持无栈、单 primary 和未证明即 fail closed。 ### Phase 1：单 P deterministic scheduler From b61c8bb7c2892dce12970beec21c3e932daa7047 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 23:37:39 +0800 Subject: [PATCH 23/32] test(coro): avoid copying build context lock --- internal/build/coro_plan_test.go | 33 ++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index b08dde0db9..720995a394 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -488,13 +488,16 @@ func atomicExchange(*uint32, uint32) uint32 t.Fatalf("required root %d = %+v, want %s/%s", index, root, wantRoots[index], wantDemand) } } - spawnCtx := *ctx - spawnCtx.buildConf = &Config{ - EnableCoroChildAwait: true, - EnableCoroProgramBootstrapRun: true, - EnableCoroClosedStaticSpawn: true, + spawnCtx := &context{ + buildConf: &Config{ + EnableCoroChildAwait: true, + EnableCoroProgramBootstrapRun: true, + EnableCoroClosedStaticSpawn: true, + }, + coroEmission: ctx.coroEmission, + coroSSAEmission: ctx.coroSSAEmission, } - spawnRoots, spawnPlain, _, _, err := requiredCoroProgramRuntimePlan(&spawnCtx) + spawnRoots, spawnPlain, _, _, err := requiredCoroProgramRuntimePlan(spawnCtx) if err != nil { t.Fatal(err) } @@ -1788,14 +1791,17 @@ func TestActiveCoroABIVersions(t *testing.T) { config *Config coroABI string scheduler string + panicABI string funcRep string }{ - {"entry resolution", &Config{}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0, coro.FuncRepABIV0}, - {"physical leaf", &Config{EnableCoroPhysicalABI: true}, coro.PhysicalABIV0, coro.SchedulerNoneABIV0, coro.FuncRepABIV0}, - {"plain dispatch", &Config{EnableCoroPlainDispatch: true}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0, coro.FuncRepABIV1}, - {"child await", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true}, coro.PhysicalABIV1, coro.SchedulerChildAwaitABIV0, coro.FuncRepABIV0}, - {"closed static spawn", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroClosedStaticSpawn: true, EnableCoroProgramBootstrapRun: true}, coro.PhysicalABIV1, coro.SchedulerProgramBootstrapClosedStaticSpawnABIV0, coro.FuncRepABIV0}, - {"program bootstrap runtime with plain dispatch", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroPlainDispatch: true, EnableCoroProgramBootstrapRun: true}, coro.PhysicalABIV1, coro.SchedulerProgramBootstrapABIV2, coro.FuncRepABIV1}, + {"nil defaults", nil, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0, coro.PanicLegacyABIV0, coro.FuncRepABIV0}, + {"entry resolution", &Config{}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0, coro.PanicLegacyABIV0, coro.FuncRepABIV0}, + {"physical leaf", &Config{EnableCoroPhysicalABI: true}, coro.PhysicalABIV0, coro.SchedulerNoneABIV0, coro.PanicLegacyABIV0, coro.FuncRepABIV0}, + {"explicit status panic", &Config{EnableCoroExplicitStatusPanicABI: true}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0, coro.PanicExplicitStatusABIV0, coro.FuncRepABIV0}, + {"plain dispatch", &Config{EnableCoroPlainDispatch: true}, coro.EntryResolutionABIV0, coro.SchedulerNoneABIV0, coro.PanicLegacyABIV0, coro.FuncRepABIV1}, + {"child await", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true}, coro.PhysicalABIV1, coro.SchedulerChildAwaitABIV0, coro.PanicLegacyABIV0, coro.FuncRepABIV0}, + {"closed static spawn", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroClosedStaticSpawn: true, EnableCoroProgramBootstrapRun: true}, coro.PhysicalABIV1, coro.SchedulerProgramBootstrapClosedStaticSpawnABIV0, coro.PanicLegacyABIV0, coro.FuncRepABIV0}, + {"program bootstrap runtime with plain dispatch", &Config{EnableCoroPhysicalABI: true, EnableCoroChildAwait: true, EnableCoroPlainDispatch: true, EnableCoroProgramBootstrapRun: true}, coro.PhysicalABIV1, coro.SchedulerProgramBootstrapABIV2, coro.PanicLegacyABIV0, coro.FuncRepABIV1}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -1805,6 +1811,9 @@ func TestActiveCoroABIVersions(t *testing.T) { if got := activeCoroSchedulerABIVersion(test.config); got != test.scheduler { t.Fatalf("scheduler ABI = %q, want %q", got, test.scheduler) } + if got := activeCoroPanicABIVersion(test.config); got != test.panicABI { + t.Fatalf("panic ABI = %q, want %q", got, test.panicABI) + } if got := activeCoroFuncRepABIVersion(test.config); got != test.funcRep { t.Fatalf("function representation ABI = %q, want %q", got, test.funcRep) } From b2c855589357bad900101a71a2f1ef6b677d7ecd Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 23:44:57 +0800 Subject: [PATCH 24/32] compiler(coro): reserve explicit status panic ABI --- cl/compilation.go | 14 ++++++- cl/compilation_test.go | 27 ++++++++++++++ cl/coro_entry.go | 7 ++++ internal/build/build.go | 53 +++++++++++++++++++-------- internal/build/collect.go | 4 +- internal/build/collect_test.go | 14 +++++++ internal/build/coro_plan_test.go | 55 ++++++++++++++++++++++++++++ internal/build/coro_registry.go | 1 + internal/build/coro_registry_test.go | 9 +++++ internal/coro/plan_digest.go | 7 +++- internal/coro/plan_digest_test.go | 18 +++++++++ internal/coro/summary_test.go | 5 ++- 12 files changed, 194 insertions(+), 20 deletions(-) diff --git a/cl/compilation.go b/cl/compilation.go index 6ea576d785..a692a7c704 100644 --- a/cl/compilation.go +++ b/cl/compilation.go @@ -51,6 +51,11 @@ type Compilation struct { SchedulerABI string PanicABI string FuncRepABI string + // EnableCoroExplicitStatusPanicABI selects the reserved target-wide + // explicit-status panic identity. This slice does not implement its hidden + // outcome, cleanup, or runtime protocol, so active code generation remains + // fail-closed when the capability is selected. + EnableCoroExplicitStatusPanicABI bool // EnableCoroPhysicalABI permits the conservative leaf-only coroutine ABI // lowering implemented by the current experimental slice. It requires entry // resolution and does not by itself enable await, dispatch, roots, or a @@ -129,6 +134,13 @@ func (c *Compilation) validateCoroABIIdentity(required bool) error { if c.EnableCoroPlainDispatch && !c.EnableCoroEntryResolution { return fmt.Errorf("coroutine plain dispatch requires coroutine entry resolution") } + if c.EnableCoroExplicitStatusPanicABI && !c.EnableCoroEntryResolution { + return fmt.Errorf("coroutine explicit-status panic ABI requires coroutine entry resolution") + } + wantPanicABI := coro.PanicLegacyABIV0 + if c.EnableCoroExplicitStatusPanicABI { + wantPanicABI = coro.PanicExplicitStatusABIV0 + } wantFuncRepABI := coro.FuncRepABIV0 if c.EnableCoroPlainDispatch { wantFuncRepABI = coro.FuncRepABIV1 @@ -140,7 +152,7 @@ func (c *Compilation) validateCoroABIIdentity(required bool) error { }{ {"coroutine", c.CoroABI, wantCoroABI}, {"scheduler", c.SchedulerABI, wantSchedulerABI}, - {"panic", c.PanicABI, coro.PanicLegacyABIV0}, + {"panic", c.PanicABI, wantPanicABI}, {"function representation", c.FuncRepABI, wantFuncRepABI}, } if !required { diff --git a/cl/compilation_test.go b/cl/compilation_test.go index 0516805bfd..91dc8163ee 100644 --- a/cl/compilation_test.go +++ b/cl/compilation_test.go @@ -121,6 +121,33 @@ func TestCompilationCoroABIIdentityValidation(t *testing.T) { if err := withoutEntry.preflightCoroPlan(); err == nil || !strings.Contains(err.Error(), "requires coroutine entry resolution") { t.Fatalf("plain-dispatch preflight dependency error = %v", err) } + newExplicitStatus := func() *Compilation { + compilation := newPhysical() + compilation.EnableCoroExplicitStatusPanicABI = true + compilation.PanicABI = coro.PanicExplicitStatusABIV0 + return compilation + } + explicitStatus := newExplicitStatus() + if err := explicitStatus.validateCoroABIIdentity(false); err != nil { + t.Fatalf("complete explicit-status panic ABI identity: %v", err) + } + legacyIdentity := newExplicitStatus() + legacyIdentity.PanicABI = coro.PanicLegacyABIV0 + if err := legacyIdentity.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "panic ABI") { + t.Fatalf("explicit-status panic ABI mismatch = %v", err) + } + withoutExplicitStatusEntry := newExplicitStatus() + withoutExplicitStatusEntry.EnableCoroEntryResolution = false + if err := withoutExplicitStatusEntry.validateCoroABIIdentity(false); err == nil || !strings.Contains(err.Error(), "requires coroutine entry resolution") { + t.Fatalf("explicit-status panic ABI dependency error = %v", err) + } + if err := withoutExplicitStatusEntry.preflightCoroPlan(); err == nil || !strings.Contains(err.Error(), "requires coroutine entry resolution") { + t.Fatalf("explicit-status panic ABI preflight dependency error = %v", err) + } + if err := explicitStatus.preflightCoroPlan(); err == nil || + !strings.Contains(err.Error(), "identity-only") || !strings.Contains(err.Error(), "runtime semantics are not implemented") { + t.Fatalf("explicit-status panic ABI active preflight error = %v", err) + } newChildAwait := func() *Compilation { return &Compilation{ EnableCoroEntryResolution: true, diff --git a/cl/coro_entry.go b/cl/coro_entry.go index fd2bfc64d4..719c6842ca 100644 --- a/cl/coro_entry.go +++ b/cl/coro_entry.go @@ -208,6 +208,9 @@ func (c *Compilation) preflightCoroPlan() error { if c.EnableCoroPlainDispatch && !c.EnableCoroEntryResolution { return fmt.Errorf("coroutine plain dispatch requires coroutine entry resolution") } + if c.EnableCoroExplicitStatusPanicABI && !c.EnableCoroEntryResolution { + return fmt.Errorf("coroutine explicit-status panic ABI requires coroutine entry resolution") + } if c.EnableCoroClosedStaticSpawn { if !c.EnableCoroChildAwait { return fmt.Errorf("coroutine closed static spawn requires coroutine child await") @@ -224,6 +227,10 @@ func (c *Compilation) preflightCoroPlan() error { c.coroPreflightErr = err return } + if c.EnableCoroExplicitStatusPanicABI { + c.coroPreflightErr = fmt.Errorf("coroutine explicit-status panic ABI %q is identity-only: lowering and runtime semantics are not implemented", coro.PanicExplicitStatusABIV0) + return + } if c.CoroPlan == nil { c.coroPreflightErr = fmt.Errorf("coroutine entry resolution requires a compilation CoroPlan") return diff --git a/internal/build/build.go b/internal/build/build.go index bf20b441f8..9a539838ba 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -890,6 +890,11 @@ type Config struct { // leaving it false preserves report-only behavior. Package archives are // reused only when their complete plan/ABI/target fingerprint matches. EnableCoroEntryResolution bool + // EnableCoroExplicitStatusPanicABI selects the reserved target-wide + // explicit-status panic identity. Hidden outcomes, cleanup edges, and the + // runtime protocol are not implemented by this slice; active builds select + // the identity for validation and then fail closed before code generation. + EnableCoroExplicitStatusPanicABI bool // EnableCoroPhysicalABI enables the experimental LLVM coroutine physical ABI. // It requires EnableCoroEntryResolution and remains leaf-only unless a more // specific lowering capability is enabled. @@ -1396,6 +1401,9 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { if ctx.buildConf.EnableCoroPlainDispatch && !ctx.buildConf.EnableCoroEntryResolution { return fmt.Errorf("enable coroutine plain dispatch: coroutine entry resolution is required") } + if ctx.buildConf.EnableCoroExplicitStatusPanicABI && !ctx.buildConf.EnableCoroEntryResolution { + return fmt.Errorf("enable coroutine explicit-status panic ABI: coroutine entry resolution is required") + } if ctx.buildConf.EnableCoroChildAwait && ctx.buildConf.BuildMode == BuildModeCArchive { return fmt.Errorf("enable coroutine child await: c-archive requires flattened package members and an explicit host bootstrap extraction contract") } @@ -1512,20 +1520,28 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { ctx.coroPlanDigest = digest ctx.coroPlanMetadata = metadata ctx.clCompilation = &cl.Compilation{ - CoroPlan: plan, - CoroPlanObserver: ctx.buildConf.CoroPlanObserver, - EnableCoroEntryResolution: ctx.buildConf.EnableCoroEntryResolution, - EnableCoroPhysicalABI: ctx.buildConf.EnableCoroPhysicalABI, - EnableCoroChildAwait: ctx.buildConf.EnableCoroChildAwait, - EnableCoroPlainDispatch: ctx.buildConf.EnableCoroPlainDispatch, - EnableCoroClosedStaticSpawn: ctx.buildConf.EnableCoroClosedStaticSpawn, - EnableCoroProgramBootstrapRun: ctx.buildConf.EnableCoroProgramBootstrapRun, - CoroPlanDigest: digest, - CoroABI: metadata.CoroABI, - SchedulerABI: metadata.SchedulerABI, - PanicABI: metadata.PanicABI, - FuncRepABI: metadata.FuncRepABI, - EmissionUniverse: ctx.coroEmission, + CoroPlan: plan, + CoroPlanObserver: ctx.buildConf.CoroPlanObserver, + EnableCoroEntryResolution: ctx.buildConf.EnableCoroEntryResolution, + EnableCoroExplicitStatusPanicABI: ctx.buildConf.EnableCoroExplicitStatusPanicABI, + EnableCoroPhysicalABI: ctx.buildConf.EnableCoroPhysicalABI, + EnableCoroChildAwait: ctx.buildConf.EnableCoroChildAwait, + EnableCoroPlainDispatch: ctx.buildConf.EnableCoroPlainDispatch, + EnableCoroClosedStaticSpawn: ctx.buildConf.EnableCoroClosedStaticSpawn, + EnableCoroProgramBootstrapRun: ctx.buildConf.EnableCoroProgramBootstrapRun, + CoroPlanDigest: digest, + CoroABI: metadata.CoroABI, + SchedulerABI: metadata.SchedulerABI, + PanicABI: metadata.PanicABI, + FuncRepABI: metadata.FuncRepABI, + EmissionUniverse: ctx.coroEmission, + } + if ctx.buildConf.EnableCoroExplicitStatusPanicABI { + ctx.coroPlan = nil + ctx.coroPlanDigest = "" + ctx.coroPlanMetadata = coro.PlanDigestMetadata{} + ctx.clCompilation = nil + return fmt.Errorf("enable coroutine explicit-status panic ABI %q: identity-only capability; lowering and runtime semantics are not implemented", metadata.PanicABI) } if ctx.buildConf.EnableCoroProgramBootstrapABI { bootstraps, err := prepareCoroProgramBootstrapsV1(ctx) @@ -1827,6 +1843,13 @@ func activeCoroSchedulerABIVersion(conf *Config) string { return coro.SchedulerNoneABIV0 } +func activeCoroPanicABIVersion(conf *Config) string { + if conf != nil && conf.EnableCoroExplicitStatusPanicABI { + return coro.PanicExplicitStatusABIV0 + } + return coro.PanicLegacyABIV0 +} + func activeCoroFuncRepABIVersion(conf *Config) string { if conf != nil && conf.EnableCoroPlainDispatch { return coro.FuncRepABIV1 @@ -2243,7 +2266,7 @@ func buildCoroPlanDigestMetadata(ctx *context) (coro.PlanDigestMetadata, error) return coro.PlanDigestMetadata{ CoroABI: activeCoroABIVersion(ctx.buildConf), SchedulerABI: activeCoroSchedulerABIVersion(ctx.buildConf), - PanicABI: coro.PanicLegacyABIV0, + PanicABI: activeCoroPanicABIVersion(ctx.buildConf), FuncRepABI: activeCoroFuncRepABIVersion(ctx.buildConf), TargetTriple: target.Triple, TargetCPU: target.CPU, diff --git a/internal/build/collect.go b/internal/build/collect.go index 48bc9279d7..9108a160a3 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -27,7 +27,6 @@ import ( "sort" "strings" - "github.com/goplus/llgo/internal/coro" "github.com/goplus/llgo/internal/env" "github.com/goplus/llgo/internal/packages" intllvm "github.com/goplus/llgo/internal/xtool/llvm" @@ -378,6 +377,7 @@ func (c *context) canUsePackageCache() bool { c.clCompilation.EnableCoroPhysicalABI == c.buildConf.EnableCoroPhysicalABI && c.clCompilation.EnableCoroChildAwait == c.buildConf.EnableCoroChildAwait && c.clCompilation.EnableCoroPlainDispatch == c.buildConf.EnableCoroPlainDispatch && + c.clCompilation.EnableCoroExplicitStatusPanicABI == c.buildConf.EnableCoroExplicitStatusPanicABI && c.clCompilation.EnableCoroClosedStaticSpawn == c.buildConf.EnableCoroClosedStaticSpawn && c.clCompilation.EnableCoroProgramBootstrapRun == c.buildConf.EnableCoroProgramBootstrapRun && c.clCompilation.CoroABI == metadata.CoroABI && @@ -386,7 +386,7 @@ func (c *context) canUsePackageCache() bool { c.clCompilation.FuncRepABI == metadata.FuncRepABI && metadata.CoroABI == activeCoroABIVersion(c.buildConf) && metadata.SchedulerABI == activeCoroSchedulerABIVersion(c.buildConf) && - metadata.PanicABI == coro.PanicLegacyABIV0 && + metadata.PanicABI == activeCoroPanicABIVersion(c.buildConf) && metadata.FuncRepABI == activeCoroFuncRepABIVersion(c.buildConf) && metadata.TargetTriple != "" && metadata.PointerBits > 0 && (metadata.Endianness == "little" || metadata.Endianness == "big") && diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 4ddf6510f6..234ce19c2a 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -60,6 +60,20 @@ func TestCoroutinePlanInputsAffectFingerprint(t *testing.T) { return manifest.Fingerprint() } baseline := fingerprint(strings.Repeat("1", 64), base) + explicitStatus := base + explicitStatus.PanicABI = coro.PanicExplicitStatusABIV0 + if got := fingerprint(strings.Repeat("1", 64), explicitStatus); got == baseline { + t.Fatal("explicit-status panic ABI did not domain-separate the package fingerprint") + } + explicitManifest := newManifestBuilder() + (&context{ + buildConf: &Config{Goos: "linux", Goarch: "amd64", EnableCoroEntryResolution: true, EnableCoroExplicitStatusPanicABI: true}, + coroPlanDigest: strings.Repeat("1", 64), + coroPlanMetadata: explicitStatus, + }).collectCommonInputs(explicitManifest) + if got := explicitManifest.common.CoroPanicABI; got != coro.PanicExplicitStatusABIV0 { + t.Fatalf("manifest panic ABI = %q, want %q", got, coro.PanicExplicitStatusABIV0) + } if got := fingerprint(strings.Repeat("2", 64), base); got == baseline { t.Fatal("CoroPlanDigest did not affect the package fingerprint") } diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index 720995a394..bc3a13f27c 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -1237,6 +1237,31 @@ func TestBuildCoroPlanInstallsArchiveDigest(t *testing.T) { t.Fatalf("manifest coroutine inputs = %+v", manifest.common) } + explicitProg := llssa.NewProgram(nil) + defer explicitProg.Dispose() + explicitCtx := &context{ + progSSA: ssaPkg.Prog, + prog: explicitProg, + buildConf: &Config{ + EnableCoroEntryResolution: true, + EnableCoroExplicitStatusPanicABI: true, + CoroPlanBuilder: func(input CoroPlanInput) (*coro.SSAPlan, error) { + return input.Analyze(coro.Roots{{Function: ssaPkg.Func("F"), Demand: coro.SyncDemand}}, coro.SSAConfig{ + MaxPlainInstructions: -1, + }) + }, + }, + } + if err := buildCoroPlan(explicitCtx, aPkg); err == nil || + !strings.Contains(err.Error(), coro.PanicExplicitStatusABIV0) || + !strings.Contains(err.Error(), "lowering and runtime semantics are not implemented") { + t.Fatalf("explicit-status panic ABI build error = %v", err) + } + if explicitCtx.coroPlan != nil || explicitCtx.clCompilation != nil || explicitCtx.coroPlanDigest != "" || explicitCtx.coroPlanMetadata.PanicABI != "" { + t.Fatalf("identity-only explicit-status panic build retained active state: plan=%v compilation=%v digest=%q metadata=%+v", + explicitCtx.coroPlan, explicitCtx.clCompilation, explicitCtx.coroPlanDigest, explicitCtx.coroPlanMetadata) + } + badProg := llssa.NewProgram(nil) defer badProg.Dispose() badCtx := &context{ @@ -1732,6 +1757,13 @@ func external() if err := validateCoroUnwindOnlyLoweredCalls(plainPlan, coro.PanicLegacyABIV0); err != nil { t.Fatalf("bounded plain unwind helper rejected: %v", err) } + if err := validateCoroUnwindOnlyLoweredCalls(plainPlan, coro.PanicExplicitStatusABIV0); err == nil || + !strings.Contains(err.Error(), "has no certified unwind-helper call contract") { + t.Fatalf("identity-only explicit-status unwind helper error = %v", err) + } + if err := validateCoroUnwindOnlyLoweredCalls(plainPlan, coro.PanicLegacyABIV0); err != nil { + t.Fatalf("explicit-status rejection changed the legacy bounded-plain certificate: %v", err) + } forged := coroLegacyPanicPlainCertificate{owner: owner, logicalName: "runtime.Helper", target: suspending} if err := forged.validate(plainPlan); err == nil || !strings.Contains(err.Error(), "not bound to an exact frozen unwind-only target") { t.Fatalf("name-only retargeted certificate error = %v", err) @@ -1891,6 +1923,17 @@ func TestBuildCoroPlanErrors(t *testing.T) { } }) + t.Run("explicit-status panic ABI requires entry resolution", func(t *testing.T) { + ctx := &context{buildConf: &Config{EnableCoroExplicitStatusPanicABI: true}} + err := buildCoroPlan(ctx) + if err == nil || !strings.Contains(err.Error(), "entry resolution is required") { + t.Fatalf("buildCoroPlan error = %v, want explicit-status entry-resolution requirement", err) + } + if ctx.coroPlan != nil || ctx.clCompilation != nil { + t.Fatal("invalid explicit-status panic ABI configuration installed coroutine compilation state") + } + }) + t.Run("child await requires physical ABI", func(t *testing.T) { ctx := &context{buildConf: &Config{ EnableCoroEntryResolution: true, @@ -2218,6 +2261,18 @@ func TestCoroEntryResolutionUsesPlanMatchedPackageCache(t *testing.T) { if dispatchCtx.canUsePackageCache() { t.Fatal("plain-dispatch capability mismatch unexpectedly permits package cache") } + explicitStatusCtx := newContext(digestA) + explicitStatusCtx.buildConf.EnableCoroExplicitStatusPanicABI = true + explicitStatusCtx.clCompilation.EnableCoroExplicitStatusPanicABI = true + explicitStatusCtx.clCompilation.PanicABI = coro.PanicExplicitStatusABIV0 + explicitStatusCtx.coroPlanMetadata.PanicABI = coro.PanicExplicitStatusABIV0 + if !explicitStatusCtx.canUsePackageCache() { + t.Fatal("matching explicit-status panic ABI identity unexpectedly disabled package cache") + } + explicitStatusCtx.clCompilation.EnableCoroExplicitStatusPanicABI = false + if explicitStatusCtx.canUsePackageCache() { + t.Fatal("explicit-status panic capability mismatch unexpectedly permits package cache") + } bootstrapMismatch := newContext(digestA) bootstrapMismatch.clCompilation.EnableCoroProgramBootstrapRun = true if bootstrapMismatch.canUsePackageCache() { diff --git a/internal/build/coro_registry.go b/internal/build/coro_registry.go index 2855858470..a09a510963 100644 --- a/internal/build/coro_registry.go +++ b/internal/build/coro_registry.go @@ -79,6 +79,7 @@ func coroProgramManifestHashV1(ctx *context, anchors []string, bootstrap ...*cor write(ctx.coroPlanDigest) write(activeCoroABIVersion(ctx.buildConf)) write(activeCoroSchedulerABIVersion(ctx.buildConf)) + write(activeCoroPanicABIVersion(ctx.buildConf)) write(target.Triple) write(target.CPU) write(target.Features) diff --git a/internal/build/coro_registry_test.go b/internal/build/coro_registry_test.go index e0b5572d44..706e0ba558 100644 --- a/internal/build/coro_registry_test.go +++ b/internal/build/coro_registry_test.go @@ -76,6 +76,15 @@ func TestCoroProgramManifestHashV1StableAndComplete(t *testing.T) { if first != again { t.Fatalf("manifest hash is unstable: %x != %x", first, again) } + ctx.buildConf.EnableCoroExplicitStatusPanicABI = true + explicitStatus, err := coroProgramManifestHashV1(ctx, []string{a, b}) + if err != nil { + t.Fatal(err) + } + ctx.buildConf.EnableCoroExplicitStatusPanicABI = false + if explicitStatus == first { + t.Fatal("manifest hash ignored the active panic ABI") + } changed, err := coroProgramManifestHashV1(ctx, []string{a}) if err != nil { t.Fatal(err) diff --git a/internal/coro/plan_digest.go b/internal/coro/plan_digest.go index d84f80a453..9d7e339a41 100644 --- a/internal/coro/plan_digest.go +++ b/internal/coro/plan_digest.go @@ -61,7 +61,12 @@ const ( // creates the child only to its initial suspend before commit. SchedulerProgramBootstrapClosedStaticSpawnABIV0 = "llgo.coro.scheduler.program-bootstrap.v2.closed-static-spawn.v0" PanicLegacyABIV0 = "llgo.coro.panic.legacy.v0" - FuncRepABIV0 = "llgo.coro.func-rep.v0" + // PanicExplicitStatusABIV0 reserves the target-wide identity for the first + // compiler-carried panic outcome ABI. The identity is intentionally wired + // before its lowering and runtime protocol: selecting it must remain + // fail-closed until those semantics are implemented. + PanicExplicitStatusABIV0 = "llgo.coro.panic.explicit-status.v0" + FuncRepABIV0 = "llgo.coro.func-rep.v0" // FuncRepABIV1 introduces an explicit descriptor/context representation for // dynamically consumed Go function values. The first producer/consumer slice // supports only one no-capture, non-suspending plain body; unsupported value diff --git a/internal/coro/plan_digest_test.go b/internal/coro/plan_digest_test.go index 32954d32ab..4c3b717909 100644 --- a/internal/coro/plan_digest_test.go +++ b/internal/coro/plan_digest_test.go @@ -676,6 +676,24 @@ func TestCoroPlanDigestMetadataMutationsChangeDigest(t *testing.T) { } } +func TestCoroPlanDigestExplicitStatusPanicABIDomainSeparation(t *testing.T) { + plan, _ := buildPlanDigestTestPlan(t, ssa.SanityCheckFunctions|ssa.InstantiateGenerics) + legacy := validPlanDigestMetadata() + legacyDigest, err := plan.CoroPlanDigest(legacy) + if err != nil { + t.Fatal(err) + } + explicitStatus := legacy + explicitStatus.PanicABI = PanicExplicitStatusABIV0 + explicitStatusDigest, err := plan.CoroPlanDigest(explicitStatus) + if err != nil { + t.Fatal(err) + } + if explicitStatusDigest == legacyDigest { + t.Fatalf("panic ABI identities share a plan digest: %s", legacyDigest) + } +} + func TestCoroPlanDigestCanonicalEmptyArrays(t *testing.T) { prog, _ := buildCoroTestSSA(t, "empty.go", `package coroid; func root() {}`) plan, err := AnalyzeSSA(prog, nil, planDigestSSAConfig()) diff --git a/internal/coro/summary_test.go b/internal/coro/summary_test.go index 3a83b0c045..591565547a 100644 --- a/internal/coro/summary_test.go +++ b/internal/coro/summary_test.go @@ -52,7 +52,7 @@ func TestSummaryStableAcrossInsertionOrder(t *testing.T) { return plan.Summary(SummaryMetadata{ CoroABI: "v1", SchedulerABI: "v1", - PanicABI: "explicit-status-v1", + PanicABI: PanicExplicitStatusABIV0, TargetTriple: "wasm32-unknown-unknown", }) } @@ -84,6 +84,9 @@ func TestSummaryStableAcrossInsertionOrder(t *testing.T) { if !strings.Contains(string(aData), `"effect":"await-structured,wait-platform"`) { t.Fatalf("summary does not use stable effect spelling: %s", aData) } + if !strings.Contains(string(aData), `"panic_abi":"`+PanicExplicitStatusABIV0+`"`) { + t.Fatalf("summary does not preserve the explicit-status panic ABI identity: %s", aData) + } if !strings.Contains(string(aData), `"emission":"none"`) || !strings.Contains(string(aData), `"emission":"external"`) { t.Fatalf("summary does not encode body emission: %s", aData) } From 825e984fdc2862417f91c85b7e9926d36f676062 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 23:47:24 +0800 Subject: [PATCH 25/32] test(coro): link and run native static spawn island --- internal/build/coro_spawn_native_e2e_test.go | 460 +++++++++++++++++++ 1 file changed, 460 insertions(+) create mode 100644 internal/build/coro_spawn_native_e2e_test.go diff --git a/internal/build/coro_spawn_native_e2e_test.go b/internal/build/coro_spawn_native_e2e_test.go new file mode 100644 index 0000000000..fc91920212 --- /dev/null +++ b/internal/build/coro_spawn_native_e2e_test.go @@ -0,0 +1,460 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + stdcontext "context" + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + "time" + + "github.com/goplus/llgo/cl" + "github.com/goplus/llgo/internal/coro" + "github.com/goplus/llgo/internal/goembed" + "github.com/goplus/llgo/internal/packages" + llssa "github.com/goplus/llgo/ssa" + llvm "github.com/xgo-dev/llvm" + "golang.org/x/tools/go/ssa" +) + +const ( + coroSpawnNativeE2EPackage = "example.com/llgo-coro-spawn-e2e" + coroSpawnNativeE2EEntry = "__llgo_coro_spawn_e2e_entry" +) + +const coroSpawnNativeE2ESource = `package main + +var Before uint32 +var After uint32 +var Leaf uint32 + +func leaf() { Leaf = 1 } + +func child() { + Before = 1 + go leaf() + After = 1 +} + +func main() { go child() } + +func Check() int32 { + if Before != 1 { + return 11 + } + if After != 0 { + return 12 + } + if Leaf != 0 { + return 13 + } + return 0 +} +` + +// TestCoroClosedStaticSpawnNativeNoStdlibRuntimeE2E is deliberately a +// scheduler-island smoke test, not a claim that the complete standard-library +// runtime startup or its legacy PanicABI is coroutine-safe. The compiler emits +// the real closed-static-go lowering and the real V2 entry/factory/control +// wrappers. The first four V2 init stages are bounded no-ops, while the linked +// production coroutine adapter/core uses its native nogc allocator backend. +// +// The two nested spawns make the result deterministic without a timer source: +// main yields to child, child publishes leaf and yields back behind main, and +// main then returns with leaf initial-suspended and child yield-suspended. +// Command shutdown must destroy both instead of resuming either one. +func TestCoroClosedStaticSpawnNativeNoStdlibRuntimeE2E(t *testing.T) { + if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + t.Skip("native coroutine link smoke requires Darwin or Linux") + } + clang, err := exec.LookPath("clang") + if err != nil { + t.Skip("clang is unavailable") + } + ar, err := exec.LookPath("llvm-ar") + if err != nil { + ar, err = exec.LookPath("ar") + if err != nil { + t.Skip("llvm-ar/ar is unavailable") + } + } + + llssa.Initialize(llssa.InitAll) + temp := t.TempDir() + prog := llssa.NewProgram(nil) + defer prog.Dispose() + + userObject, anchor, checkSymbol := buildCoroSpawnNativeE2EUser(t, prog, temp) + entryObject := buildCoroSpawnNativeE2EEntry(t, prog, temp, anchor) + driverObject := buildCoroSpawnNativeE2EDriver(t, prog, temp, checkSymbol) + runtimeObjects := buildCoroSpawnNativeE2ERuntimeIsland(t, temp) + runtimeArchive := filepath.Join(temp, "libllgo-coro-runtime-island.a") + arArgs := append([]string{"rcs", runtimeArchive}, runtimeObjects...) + if output, err := exec.Command(ar, arArgs...).CombinedOutput(); err != nil { + t.Fatalf("archive coroutine runtime island: %v\n%s", err, output) + } + + executable := filepath.Join(temp, "coro-spawn-e2e") + linkArgs := []string{driverObject, entryObject, userObject, runtimeArchive, "-o", executable} + if runtime.GOOS == "darwin" { + linkArgs = append(linkArgs, "-Wl,-dead_strip") + } else { + linkArgs = append(linkArgs, "-Wl,--gc-sections") + } + if output, err := exec.Command(clang, linkArgs...).CombinedOutput(); err != nil { + t.Fatalf("link native coroutine spawn/shutdown smoke: %v\n%s", err, output) + } + assertCoroSpawnNativeE2ELinkedSymbols(t, executable) + + runCtx, cancel := stdcontext.WithTimeout(stdcontext.Background(), 10*time.Second) + defer cancel() + output, err := exec.CommandContext(runCtx, executable).CombinedOutput() + if runCtx.Err() != nil { + t.Fatalf("native coroutine spawn/shutdown smoke timed out: %v\n%s", runCtx.Err(), output) + } + if err != nil { + t.Fatalf("native coroutine spawn/shutdown smoke failed: %v\n%s", err, output) + } +} + +func buildCoroSpawnNativeE2EUser(t *testing.T, prog llssa.Program, temp string) (object, anchor, checkSymbol string) { + t.Helper() + ssaPkg, files := buildCoroPlanTestPackage(t, coroSpawnNativeE2EPackage, coroSpawnNativeE2ESource, nil) + universe, err := cl.PrepareEmissionUniverse(prog, nil, []cl.EmissionPackage{{ + SSA: ssaPkg, Files: files, Identity: coroSpawnNativeE2EPackage, + }}) + if err != nil { + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + t.Fatal(err) + } + mainFn, childFn, leafFn, checkFn := ssaPkg.Func("main"), ssaPkg.Func("child"), ssaPkg.Func("leaf"), ssaPkg.Func("Check") + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapClosedStaticSpawnABIV0 + functionIDs.ArchiveReady = true + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{ + {Function: mainFn, Demand: coro.AsyncDemand}, + {Function: checkFn, Demand: coro.SyncDemand}, + }, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + switch fn { + case mainFn, childFn, leafFn: + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + default: + return coro.SSAFunctionPolicy{}, nil + } + }, + }) + if err != nil { + t.Fatal(err) + } + compilation := &cl.Compilation{ + CoroPlan: plan, + EnableCoroEntryResolution: true, + EnableCoroPhysicalABI: true, + EnableCoroChildAwait: true, + EnableCoroClosedStaticSpawn: true, + EnableCoroProgramBootstrapRun: true, + CoroABI: coro.PhysicalABIV1, + SchedulerABI: coro.SchedulerProgramBootstrapClosedStaticSpawnABIV0, + PanicABI: coro.PanicLegacyABIV0, + FuncRepABI: coro.FuncRepABIV0, + EmissionUniverse: universe, + } + pkg, _, err := cl.NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + cl.PackageOptions{Compilation: compilation}, + ) + if err != nil { + t.Fatal(err) + } + module := pkg.Module() + runCoroSpawnNativeE2EPasses(t, prog, module) + ir := module.String() + match := regexp.MustCompile(`@"?(__llgo_coro_root_package_v1\.[0-9a-f]{32})"?\s*=`).FindStringSubmatch(ir) + if len(match) != 2 { + t.Fatalf("compiled E2E user module has no root package anchor:\n%s", ir) + } + checkSymbol = coroSpawnNativeE2EPackage + ".Check" + if module.NamedFunction(checkSymbol).IsNil() { + t.Fatalf("compiled E2E user module has no plain checker %q:\n%s", checkSymbol, ir) + } + return emitCoroSpawnNativeE2EObject(t, prog, module, filepath.Join(temp, "user.o")), match[1], checkSymbol +} + +func buildCoroSpawnNativeE2EEntry(t *testing.T, prog llssa.Program, temp, anchor string) string { + t.Helper() + conf := &Config{ + BuildMode: BuildModeExe, + Goos: runtime.GOOS, + Goarch: runtime.GOARCH, + EnableCoroEntryResolution: true, + EnableCoroPhysicalABI: true, + EnableCoroChildAwait: true, + EnableCoroClosedStaticSpawn: true, + EnableCoroProgramBootstrapABI: true, + EnableCoroProgramBootstrapRun: true, + } + ctx := &context{prog: prog, buildConf: conf} + bootstrap := &coroProgramBootstrapV1{ + Version: coroProgramBootstrapVersionV2, + Steps: []coroProgramBootstrapStepV1{ + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleRuntimeInitV2, FunctionID: "e2e-runtime-init", Target: "__llgo_coro_e2e_runtime_init"}, + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleABIInitV2, FunctionID: "e2e-abi-init", Target: "init$abitypes"}, + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRolePublicRuntimeInitV2, FunctionID: coroProgramPublicRuntimeNoopIDV2, Target: coroProgramPublicRuntimeNoopSymbolV2}, + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRolePackageInitV2, FunctionID: "e2e-package-init", Target: "__llgo_coro_e2e_package_init"}, + { + Kind: coroProgramStepCoroRootV1, Role: coroProgramStepRoleMainV2, + FunctionID: "e2e-main", Target: coroSpawnNativeE2EPackage + ".main$coro", + Owner: coroSpawnNativeE2EPackage, CatalogTarget: anchor, Aux: 0, + }, + }, + } + var programHash [16]byte + for i := range programHash { + programHash[i] = byte(i + 1) + } + entry := genMainModule(ctx, llssa.PkgRuntime, &packages.Package{ + ID: coroSpawnNativeE2EPackage, PkgPath: coroSpawnNativeE2EPackage, ExportFile: "coro-spawn-e2e.a", + }, &genConfig{ + coroRootAnchors: []string{anchor}, + coroManifestHash: programHash, + coroBootstrap: bootstrap, + }) + for _, name := range []string{"__llgo_coro_e2e_runtime_init", "__llgo_coro_e2e_package_init"} { + fn := entry.LPkg.FuncOf(name) + if fn == nil { + t.Fatalf("entry module has no bounded E2E init declaration %q", name) + } + if !fn.HasBody() { + body := fn.MakeBody(1) + body.Return() + } + } + entryMain := entry.LPkg.Module().NamedFunction("main") + if entryMain.IsNil() { + t.Fatalf("entry module has no native main:\n%s", entry.LPkg.String()) + } + entryMain.SetName(coroSpawnNativeE2EEntry) + if err := lowerCoroControlWrappers(ctx, entry.LPkg); err != nil { + t.Fatal(err) + } + return emitCoroSpawnNativeE2EObject(t, prog, entry.LPkg.Module(), filepath.Join(temp, "entry.o")) +} + +func buildCoroSpawnNativeE2EDriver(t *testing.T, prog llssa.Program, temp, checkSymbol string) string { + t.Helper() + pkg := prog.NewPackage("coro-spawn-e2e-driver", "coro-spawn-e2e-driver") + defer pkg.Module().Dispose() + pointer := types.Typ[types.UnsafePointer] + entry := pkg.NewFunc(coroSpawnNativeE2EEntry, newSignature( + []types.Type{types.Typ[types.Int32], pointer}, []types.Type{types.Typ[types.Int32]}, + ), llssa.InC) + check := pkg.NewFunc(checkSymbol, newSignature(nil, []types.Type{types.Typ[types.Int32]}), llssa.InGo) + // The production scheduler core is intentionally compiled without the full + // standard-library runtime package in its coroutine plan. LLGo's ordinary + // pointer checks name this legacy helper even though every valid scheduler + // path passes false. Keep the test island fail-stop without pulling the + // legacy panic/printing closure into the final executable. + abort := pkg.NewFunc("abort", newSignature(nil, nil), llssa.InC) + assertNil := pkg.NewFunc(llssa.PkgRuntime+".AssertNilDeref", newSignature( + []types.Type{types.Typ[types.Bool]}, nil, + ), llssa.InGo) + assertBody := assertNil.MakeBody(3) + fail, valid := assertNil.Block(1), assertNil.Block(2) + assertBody.If(assertNil.Param(0), fail, valid) + assertBody.SetBlock(fail).Call(abort.Expr) + assertBody.Return() + assertBody.SetBlock(valid).Return() + // Compiling the complete production core object also leaves relocations for + // ordinary runtime allocation helpers in currently unreachable panic-status + // code. Resolve those helpers directly to libc so archive extraction cannot + // pull the unrelated legacy runtime/Panic/printing object into this island. + // Frame and task storage still go through the production coroalloc backend. + uintptrType := types.Typ[types.Uintptr] + malloc := pkg.NewFunc("malloc", newSignature( + []types.Type{uintptrType}, []types.Type{pointer}, + ), llssa.InC) + calloc := pkg.NewFunc("calloc", newSignature( + []types.Type{uintptrType, uintptrType}, []types.Type{pointer}, + ), llssa.InC) + allocU := pkg.NewFunc(llssa.PkgRuntime+".AllocU", newSignature( + []types.Type{uintptrType}, []types.Type{pointer}, + ), llssa.InGo) + allocUBody := allocU.MakeBody(1) + allocUBody.Return(allocUBody.Call(malloc.Expr, allocU.Param(0))) + allocZ := pkg.NewFunc(llssa.PkgRuntime+".AllocZ", newSignature( + []types.Type{uintptrType}, []types.Type{pointer}, + ), llssa.InGo) + allocZBody := allocZ.MakeBody(1) + allocZBody.Return(allocZBody.Call(calloc.Expr, prog.IntVal(1, prog.Uintptr()), allocZ.Param(0))) + main := pkg.NewFunc("main", newSignature( + []types.Type{types.Typ[types.Int32], pointer}, []types.Type{types.Typ[types.Int32]}, + ), llssa.InC) + body := main.MakeBody(1) + body.Call(entry.Expr, main.Param(0), main.Param(1)) + body.Return(body.Call(check.Expr)) + pkg.MaterializePreserveSyms() + return emitCoroSpawnNativeE2EObject(t, prog, pkg.Module(), filepath.Join(temp, "driver.o")) +} + +func buildCoroSpawnNativeE2ERuntimeIsland(t *testing.T, temp string) []string { + t.Helper() + files := []string{ + filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_allocator.go"), + filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_frame.go"), + filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_program.go"), + filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_sched.go"), + filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_spawn.go"), + } + conf := NewDefaultConf(ModeGen) + conf.ForceRebuild = true + conf.Tags = "nogc" + allowed := map[string]bool{ + "command-line-arguments": true, + "github.com/goplus/llgo/runtime/internal/coro": true, + "github.com/goplus/llgo/runtime/internal/coroalloc": true, + } + seen := make(map[string]bool, len(allowed)) + var objects []string + conf.ModuleHook = func(pkg Package) { + if pkg.LPkg == nil || pkg.LPkg.Prog == nil { + return + } + if !allowed[pkg.ID] { + return + } + if seen[pkg.ID] { + t.Fatalf("production coroutine runtime island emitted duplicate module %q", pkg.ID) + } + seen[pkg.ID] = true + module := pkg.LPkg.Module() + if module.IsNil() { + return + } + name := fmt.Sprintf("runtime-%03d-%s.o", len(objects), sanitizeCoroSpawnNativeE2EObjectName(pkg.ID)) + objects = append(objects, emitCoroSpawnNativeE2EObject( + t, pkg.LPkg.Prog, module, filepath.Join(temp, name), + )) + } + pkgs, err := Do(files, conf) + if err != nil { + t.Fatalf("compile production coroutine runtime island in nogc mode: %v", err) + } + if len(pkgs) == 0 || pkgs[0].LPkg == nil { + t.Fatal("production coroutine runtime island produced no root package") + } + pkgs[0].LPkg.Prog.Dispose() + for id := range allowed { + if !seen[id] { + t.Fatalf("production coroutine runtime island did not emit required module %q", id) + } + } + if len(objects) != len(allowed) { + t.Fatalf("production coroutine runtime island objects = %d, want exactly %d", len(objects), len(allowed)) + } + return objects +} + +func sanitizeCoroSpawnNativeE2EObjectName(name string) string { + return strings.NewReplacer("/", "_", "\\", "_", ":", "_", " ", "_").Replace(name) +} + +func assertCoroSpawnNativeE2ELinkedSymbols(t *testing.T, executable string) { + t.Helper() + nm, err := exec.LookPath("nm") + if err != nil { + t.Skip("nm is unavailable for linked coroutine island audit") + } + output, err := exec.Command(nm, executable).CombinedOutput() + if err != nil { + t.Fatalf("inspect linked coroutine island: %v\n%s", err, output) + } + symbols := string(output) + for _, required := range []string{ + "__llgo_coro_spawn_begin_v1", + "__llgo_coro_spawn_commit_v1", + "github.com/goplus/llgo/runtime/internal/coro.CommitSpawn", + "github.com/goplus/llgo/runtime/internal/coro.BeginCommandShutdown", + } { + if !strings.Contains(symbols, required) { + t.Fatalf("linked coroutine island is missing production symbol %q:\n%s", required, symbols) + } + } + for _, forbidden := range []string{ + "github.com/goplus/llgo/runtime/internal/runtime.Panic", + "github.com/goplus/llgo/runtime/internal/runtime.Rethrow", + "github.com/goplus/llgo/runtime/internal/runtime.TracePanic", + "github.com/goplus/llgo/runtime/internal/runtime.printany", + } { + if strings.Contains(symbols, forbidden) { + t.Fatalf("test-only coroutine island unexpectedly extracted legacy PanicABI symbol %q", forbidden) + } + } +} + +func runCoroSpawnNativeE2EPasses(t *testing.T, prog llssa.Program, module llvm.Module) { + t.Helper() + module.SetDataLayout(prog.DataLayout()) + module.SetTarget(prog.TargetSpec().Triple) + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify E2E coroutine module before CoroSplit: %v\n%s", err, module.String()) + } + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + options.SetVerifyEach(true) + const pipeline = "coro-early,cgscc(coro-split),coro-cleanup" + if err := module.RunPasses(pipeline, prog.TargetMachine(), options); err != nil { + t.Fatalf("run E2E %s: %v\n%s", pipeline, err, module.String()) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify E2E coroutine module after CoroSplit: %v\n%s", err, module.String()) + } +} + +func emitCoroSpawnNativeE2EObject(t *testing.T, prog llssa.Program, module llvm.Module, path string) string { + t.Helper() + module.SetDataLayout(prog.DataLayout()) + module.SetTarget(prog.TargetSpec().Triple) + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify %s before object emission: %v\n%s", filepath.Base(path), err, module.String()) + } + object, err := prog.TargetMachine().EmitToMemoryBuffer(module, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit %s: %v\n%s", filepath.Base(path), err, module.String()) + } + defer object.Dispose() + if err := os.WriteFile(path, object.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + return path +} From 020a8aa1102479a9e3b17744465a06888c9660b0 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 23:50:10 +0800 Subject: [PATCH 26/32] runtime(coro): add terminal explicit panic state machine --- runtime/internal/coro/explicit_status.go | 237 +++++++++++ runtime/internal/coro/explicit_status_test.go | 400 ++++++++++++++++++ runtime/internal/coro/frame.go | 6 + runtime/internal/coro/scheduler.go | 41 +- runtime/internal/coro/spawn.go | 3 +- .../internal/runtime/coro_explicit_status.go | 40 ++ runtime/internal/runtime/coro_sched.go | 24 ++ 7 files changed, 747 insertions(+), 4 deletions(-) create mode 100644 runtime/internal/coro/explicit_status.go create mode 100644 runtime/internal/coro/explicit_status_test.go create mode 100644 runtime/internal/runtime/coro_explicit_status.go diff --git a/runtime/internal/coro/explicit_status.go b/runtime/internal/coro/explicit_status.go new file mode 100644 index 0000000000..e5b9f45f09 --- /dev/null +++ b/runtime/internal/coro/explicit_status.go @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import "unsafe" + +// ExplicitStatus is a terminal completion published by compiler-generated +// code. Version zero intentionally supports only an explicit panic. Normal +// return continues to use PrepareComplete; Goexit and implicit faults require +// distinct cleanup/producer protocols and are rejected. +type ExplicitStatus uint32 + +const ( + ExplicitStatusNone ExplicitStatus = iota + ExplicitStatusPanic + ExplicitStatusReturn + ExplicitStatusGoexit + ExplicitStatusImplicitFault +) + +const ( + explicitStatusPublishing uint32 = 0x80000000 + iota + explicitStatusRejected +) + +// PanicRecord is embedded in G, so its two interface words remain rooted after +// the active LLVM frame and all suspended-await ancestors have been destroyed. +// status is the one-time publication word. The runtime must not inspect either +// payload word until it observes ExplicitStatusPanic with acquire semantics. +type PanicRecord struct { + status uint32 + typeWord unsafe.Pointer + dataWord unsafe.Pointer +} + +// PanicRecordSnapshot is the stable adapter-facing copy of a published +// task-local record. Neither payload word is read from a frame/header/handle. +type PanicRecordSnapshot struct { + Status ExplicitStatus + TypeWord unsafe.Pointer + DataWord unsafe.Pointer +} + +func emptyPanicRecord(record *PanicRecord) bool { + return record != nil && preemptLoad(&record.status) == uint32(ExplicitStatusNone) && + record.typeWord == nil && record.dataWord == nil +} + +func publishedPanicRecord(record *PanicRecord) bool { + return record != nil && preemptLoad(&record.status) == uint32(ExplicitStatusPanic) +} + +// LoadPanicRecord takes an acquire snapshot after one successful publication. +// The record is deliberately not consumed: terminal reporting must retain a GC +// root until a later, separately designed fatal/recover ownership protocol. +func LoadPanicRecord(g *G) (PanicRecordSnapshot, bool) { + if !ValidG(g) || !publishedPanicRecord(&g.panicRecord) { + return PanicRecordSnapshot{}, false + } + return PanicRecordSnapshot{ + Status: ExplicitStatusPanic, + TypeWord: g.panicRecord.typeWord, + DataWord: g.panicRecord.dataWord, + }, true +} + +// PrepareExplicitStatus is the independently testable core of the future +// compiler hook. Only ExplicitStatusPanic is accepted. The first caller owns +// the publication attempt; any malformed winner permanently poisons the record +// instead of allowing execution to continue with ambiguous terminal state. +// +// HeaderV1.Flags must be zero. Thus cleanup/recover/Goexit/implicit-fault +// shapes cannot be smuggled through an unversioned flag convention. An +// untyped nil panic is also rejected; the compiler must first materialize the +// Go-version-appropriate non-nil panic type word. +func PrepareExplicitStatus( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + status ExplicitStatus, + typeWord, dataWord unsafe.Pointer, +) bool { + if g == nil || !ValidG(g) { + return false + } + record := &g.panicRecord + if !preemptCompareAndSwap(&record.status, uint32(ExplicitStatusNone), explicitStatusPublishing) { + return false + } + reject := func() bool { + preemptStore(&record.status, explicitStatusRejected) + return false + } + if status != ExplicitStatusPanic || typeWord == nil || handle == nil || header == nil || header.Flags != 0 || + g.state != GRunning || g.active == nil || g.root == nil || g.runP == nil || + g.runP.current != g || !g.runP.inResume || !expectedAction(g.runP, g, g.runP.action, ActionResume) || + g.pending.kind != pendingNone || g.pending.from != nil || g.pending.target != nil || + g.pending.wait != nil || g.pending.ticket != 0 || g.destroyTarget != nil || g.destroyRoot || + g.queued || g.nextReady != nil || g.waitToken != nil || g.waitTicket != 0 || + g.nextWait != nil || g.waiting || g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil || + g.panicUnwind { + return reject() + } + frame := findFrame(g, handle) + if frame == nil || frame != g.active || frame.owner != g || frame.header != header || + frame.state != FrameActive || header.G != unsafe.Pointer(g) || + header.SuspendReason != uint16(SuspendPanic) || + header.Lifecycle != uint16(FrameFinalSuspended) { + return reject() + } + + // The winner is the only writer. Publish pending ownership before the + // release-store of status so a post-resume scheduler and any snapshot reader + // can never observe a partially initialized record. + record.typeWord = typeWord + record.dataWord = dataWord + g.pending = pendingTransition{kind: pendingPanic, from: frame} + preemptStore(&record.status, uint32(ExplicitStatusPanic)) + return true +} + +// PreparePanic is the intended runtime hook shape. It carries the physical G +// explicitly and never consults TLS or a process-global current-G variable. +func PreparePanic(g *G, handle unsafe.Pointer, header *HeaderV1, typeWord, dataWord unsafe.Pointer) bool { + return PrepareExplicitStatus(g, handle, header, ExplicitStatusPanic, typeWord, dataWord) +} + +func preparePanicAncestor(p *P, g *G, frame *Frame) (Action, bool) { + if p == nil || g == nil || frame == nil || p.current != g || g.state != GPanicking || + !g.panicUnwind || !publishedPanicRecord(&g.panicRecord) || g.destroyTarget != nil || + frame != g.active || frame.owner != g || frame.handle == nil || frame.header == nil || + frame.state != FrameSuspended || frame.header.G != unsafe.Pointer(g) || + frame.header.SuspendReason != uint16(SuspendCall) || + frame.header.Lifecycle != uint16(FrameSuspended) { + return Action{}, false + } + handle := frame.handle + g.active = frame.parent + g.destroyRoot = frame == g.root + frame.state = FrameDestroyPending + frame.header.Lifecycle = uint16(FrameDestroyPending) + g.destroyTarget = frame + return setAction(p, ActionPanicDestroy, handle) +} + +func finishPanicG(p *P, g *G, wasRoot bool) (Action, bool) { + if p == nil || g == nil || !wasRoot || g.active != nil || g.frames != nil || + !g.panicUnwind || !publishedPanicRecord(&g.panicRecord) || + !validReadyQueue(p) || !validWaitQueue(p) { + return Action{}, false + } + schedule := preemptLoad(&p.schedule) + if schedule != scheduleIdle && schedule != scheduleRequested { + return Action{}, false + } + // Match normal terminal linearization when this is the last G. With peers, + // retain the P gate: the runtime will surface the panic immediately, but no + // child/peer ownership is silently discarded by this core transition. + if p.readyHead == nil && p.waitHead == nil && + !preemptCompareAndSwap(&p.schedule, scheduleIdle, scheduleDisabled) { + return Action{}, false + } + g.destroyRoot = false + g.root = nil + g.panicUnwind = false + preemptStore(preemptAddress(g), preemptDisabled) + g.state = GDead + g.runP = nil + p.current = nil + p.action = Action{} + return Action{Kind: ActionPanicComplete}, true +} + +// commitInitialPanicDestroyed is entered only after the active final-suspended +// panic frame passed coro.done, was directly destroyed, and its free hook +// unlinked it. A suspended-await ancestor is never checked or resumed. +func commitInitialPanicDestroyed(p *P, g *G, wasRoot bool) (Action, bool) { + if g == nil || !g.panicUnwind || !publishedPanicRecord(&g.panicRecord) { + return Action{}, false + } + if g.active != nil { + if wasRoot { + return Action{}, false + } + g.destroyRoot = false + g.state = GPanicking + return preparePanicAncestor(p, g, g.active) + } + return finishPanicG(p, g, wasRoot) +} + +// PanicDestroyed commits one direct ancestor destroy. ReleaseFrame must have +// already removed the frame. The next action is either another deepest parent +// destroy or terminal PanicComplete; no normal continuation is resumed. +func PanicDestroyed(p *P, g *G, action Action) (Action, bool) { + if !expectedAction(p, g, action, ActionPanicDestroy) || p.inResume || + g.state != GPanicking || !g.panicUnwind || !publishedPanicRecord(&g.panicRecord) || + g.destroyTarget != nil { + return Action{}, false + } + wasRoot := g.destroyRoot + if g.active != nil { + if wasRoot { + return Action{}, false + } + g.destroyRoot = false + return preparePanicAncestor(p, g, g.active) + } + return finishPanicG(p, g, wasRoot) +} + +// AcknowledgePanicTerminalSchedule consumes the only legal failed terminal +// commit after the last handle was already destroyed: RequestSchedule won the +// idle-to-disabled race. The adapter may then retry PanicDestroyed without +// calling llvm.coro.destroy twice. +func AcknowledgePanicTerminalSchedule(p *P, g *G, action Action) bool { + return expectedAction(p, g, action, ActionPanicDestroy) && !p.inResume && + g.state == GPanicking && g.panicUnwind && publishedPanicRecord(&g.panicRecord) && + g.destroyTarget == nil && g.destroyRoot && g.active == nil && g.frames == nil && + p.readyHead == nil && p.readyTail == nil && p.waitHead == nil && p.waitTail == nil && + validReadyQueue(p) && validWaitQueue(p) && + preemptCompareAndSwap(&p.schedule, scheduleRequested, scheduleIdle) +} diff --git a/runtime/internal/coro/explicit_status_test.go b/runtime/internal/coro/explicit_status_test.go new file mode 100644 index 0000000000..30fac729ef --- /dev/null +++ b/runtime/internal/coro/explicit_status_test.go @@ -0,0 +1,400 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "runtime" + "sync" + "testing" + "unsafe" +) + +type explicitPanicFixture struct { + p *P + g *G + frames []*testFrame + byHandle map[unsafe.Pointer]*testFrame + action Action +} + +func newExplicitPanicFixture(t *testing.T, depth int) *explicitPanicFixture { + t.Helper() + if depth < 1 { + t.Fatal("explicit panic fixture requires at least one frame") + } + g := new(G) + if !InitG(g) { + t.Fatal("initialize explicit panic G") + } + frames := make([]*testFrame, depth) + byHandle := make(map[unsafe.Pointer]*testFrame, depth) + var parent unsafe.Pointer + for index := range frames { + handle := unsafe.Pointer(new(byte)) + frames[index] = newTestFrame(t, g, handle, parent) + frames[index].header.StateID = uint32(index + 1) + byHandle[handle] = frames[index] + parent = handle + } + if !AdoptRoot(g, frames[0].handle) { + t.Fatal("adopt explicit panic root") + } + p := new(P) + if !Enqueue(p, g) { + t.Fatal("enqueue explicit panic G") + } + if got, ok := NextRunnable(p); !ok || got != g { + t.Fatalf("dequeue explicit panic G = (%p, %t)", got, ok) + } + action, ok := BeginRunG(p, g) + if !ok || action.Kind != ActionCheckResume { + t.Fatalf("begin explicit panic G = (%+v, %t)", action, ok) + } + action, ok = Checked(p, g, action, false) + if !ok || action.Kind != ActionResume { + t.Fatalf("activate explicit panic root = (%+v, %t)", action, ok) + } + frames[0].header.SuspendReason = uint16(SuspendNone) + frames[0].header.Lifecycle = uint16(FrameActive) + + for index := 0; index+1 < len(frames); index++ { + current, child := frames[index], frames[index+1] + current.header.SuspendReason = uint16(SuspendCall) + current.header.Lifecycle = uint16(FrameSuspended) + if !PrepareAwait(g, current.handle, child.handle) { + t.Fatalf("prepare explicit panic await %d", index) + } + action, ok = Resumed(p, g, action) + if !ok || action.Kind != ActionCheckResume || action.Handle != child.handle { + t.Fatalf("dispatch explicit panic child %d = (%+v, %t)", index, action, ok) + } + action, ok = Checked(p, g, action, false) + if !ok || action.Kind != ActionResume || action.Handle != child.handle { + t.Fatalf("activate explicit panic child %d = (%+v, %t)", index, action, ok) + } + child.header.SuspendReason = uint16(SuspendNone) + child.header.Lifecycle = uint16(FrameActive) + } + return &explicitPanicFixture{p: p, g: g, frames: frames, byHandle: byHandle, action: action} +} + +func (fixture *explicitPanicFixture) publish(t *testing.T, typeWord, dataWord unsafe.Pointer) { + t.Helper() + leaf := fixture.frames[len(fixture.frames)-1] + leaf.header.SuspendReason = uint16(SuspendPanic) + leaf.header.Lifecycle = uint16(FrameFinalSuspended) + if !PreparePanic(fixture.g, leaf.handle, leaf.header, typeWord, dataWord) { + t.Fatal("publish explicit panic") + } +} + +func (fixture *explicitPanicFixture) beginPanicDestroy(t *testing.T) Action { + t.Helper() + action, ok := Resumed(fixture.p, fixture.g, fixture.action) + if !ok || action.Kind != ActionCheckDestroy || action.Handle != fixture.frames[len(fixture.frames)-1].handle { + t.Fatalf("panic active-frame completion = (%+v, %t)", action, ok) + } + action, ok = Checked(fixture.p, fixture.g, action, true) + if !ok || action.Kind != ActionDestroy { + t.Fatalf("panic active-frame done check = (%+v, %t)", action, ok) + } + return action +} + +func (fixture *explicitPanicFixture) release(t *testing.T, action Action) { + t.Helper() + frame := fixture.byHandle[action.Handle] + if frame == nil { + t.Fatalf("release unknown panic handle %p", action.Handle) + } + releaseTestFrame(t, fixture.g, frame) +} + +func (fixture *explicitPanicFixture) commitDestroyed(action Action) (Action, bool) { + switch action.Kind { + case ActionDestroy: + return Destroyed(fixture.p, fixture.g, action) + case ActionPanicDestroy: + return PanicDestroyed(fixture.p, fixture.g, action) + default: + return Action{}, false + } +} + +func (fixture *explicitPanicFixture) acknowledgeTerminalSchedule(action Action) bool { + switch action.Kind { + case ActionDestroy: + return AcknowledgeTerminalSchedule(fixture.p, fixture.g, action) + case ActionPanicDestroy: + return AcknowledgePanicTerminalSchedule(fixture.p, fixture.g, action) + default: + return false + } +} + +func (fixture *explicitPanicFixture) finish(t *testing.T, action Action) ([]unsafe.Pointer, Action) { + t.Helper() + destroyed := make([]unsafe.Pointer, 0, len(fixture.frames)) + for { + switch action.Kind { + case ActionDestroy: + destroyed = append(destroyed, action.Handle) + fixture.release(t, action) + var ok bool + action, ok = fixture.commitDestroyed(action) + if !ok { + t.Fatalf("commit panic active destroy = (%+v, %t)", action, ok) + } + case ActionPanicDestroy: + destroyed = append(destroyed, action.Handle) + fixture.release(t, action) + var ok bool + action, ok = fixture.commitDestroyed(action) + if !ok { + t.Fatalf("commit panic ancestor destroy = (%+v, %t)", action, ok) + } + case ActionPanicComplete: + return destroyed, action + default: + t.Fatalf("panic path resumed or emitted unexpected action %+v", action) + } + } +} + +func TestExplicitPanicPublishOnceRace(t *testing.T) { + fixture := newExplicitPanicFixture(t, 1) + leaf := fixture.frames[0] + leaf.header.SuspendReason = uint16(SuspendPanic) + leaf.header.Lifecycle = uint16(FrameFinalSuspended) + + const contenders = 32 + typeWords := new([contenders]byte) + dataWords := new([contenders]byte) + winners := make(chan int, contenders) + start := make(chan struct{}) + var group sync.WaitGroup + group.Add(contenders) + for index := 0; index < contenders; index++ { + go func(index int) { + defer group.Done() + <-start + if PreparePanic(fixture.g, leaf.handle, leaf.header, + unsafe.Pointer(&typeWords[index]), unsafe.Pointer(&dataWords[index])) { + winners <- index + } + }(index) + } + close(start) + group.Wait() + close(winners) + winner := -1 + for index := range winners { + if winner != -1 { + t.Fatalf("multiple explicit panic publishers won: %d and %d", winner, index) + } + winner = index + } + if winner < 0 { + t.Fatal("no explicit panic publisher won") + } + record, ok := LoadPanicRecord(fixture.g) + if !ok || record.Status != ExplicitStatusPanic || + record.TypeWord != unsafe.Pointer(&typeWords[winner]) || record.DataWord != unsafe.Pointer(&dataWords[winner]) { + t.Fatalf("published panic record = (%+v, %t), winner=%d", record, ok, winner) + } + destroyed, action := fixture.finish(t, fixture.beginPanicDestroy(t)) + if len(destroyed) != 1 || destroyed[0] != leaf.handle || action.Kind != ActionPanicComplete { + t.Fatalf("single-frame panic destroy = %v / %+v", destroyed, action) + } + if TerminalG(fixture.p, fixture.g) || ReclaimableG(fixture.g) { + t.Fatal("published panic was misclassified as ordinary completion") + } + runtime.KeepAlive(typeWords) + runtime.KeepAlive(dataWords) + runtime.KeepAlive(leaf.memory) +} + +func explicitFramePermutations() [][3]int { + return [][3]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}} +} + +func TestExplicitPanicDestroysDeepestToRootAcrossFrameListShuffle(t *testing.T) { + for _, permutation := range explicitFramePermutations() { + permutation := permutation + t.Run(string(rune('0'+permutation[0]))+string(rune('0'+permutation[1]))+string(rune('0'+permutation[2])), func(t *testing.T) { + fixture := newExplicitPanicFixture(t, 3) + metadata := make([]*Frame, len(fixture.frames)) + for index, frame := range fixture.frames { + metadata[index] = FrameFromStorage(frame.storage) + } + for index, source := range permutation { + metadata[source].next = nil + if index+1 < len(permutation) { + metadata[source].next = metadata[permutation[index+1]] + } + } + fixture.g.frames = metadata[permutation[0]] + + typeWord, dataWord := new(byte), new(byte) + fixture.publish(t, unsafe.Pointer(typeWord), unsafe.Pointer(dataWord)) + destroyed, action := fixture.finish(t, fixture.beginPanicDestroy(t)) + want := []unsafe.Pointer{fixture.frames[2].handle, fixture.frames[1].handle, fixture.frames[0].handle} + if len(destroyed) != len(want) { + t.Fatalf("destroy order = %v, want %v", destroyed, want) + } + for index := range want { + if destroyed[index] != want[index] { + t.Fatalf("destroy order = %v, want deepest-to-root %v", destroyed, want) + } + } + if action.Kind != ActionPanicComplete || action.Handle != nil || fixture.g.state != GDead || + fixture.g.root != nil || fixture.g.active != nil || fixture.g.frames != nil || fixture.g.panicUnwind { + t.Fatalf("panic terminal state = action:%+v state:%d root:%p active:%p frames:%p unwind:%t", + action, fixture.g.state, fixture.g.root, fixture.g.active, fixture.g.frames, fixture.g.panicUnwind) + } + if record, ok := LoadPanicRecord(fixture.g); !ok || record.TypeWord != unsafe.Pointer(typeWord) || record.DataWord != unsafe.Pointer(dataWord) { + t.Fatalf("post-destroy task-local record = (%+v, %t)", record, ok) + } + for _, frame := range fixture.frames { + runtime.KeepAlive(frame.memory) + } + }) + } +} + +func TestExplicitStatusUnsupportedShapesFailClosed(t *testing.T) { + tests := []struct { + name string + status ExplicitStatus + typeWord bool + flags uint32 + }{ + {name: "normal return", status: ExplicitStatusReturn, typeWord: true}, + {name: "goexit", status: ExplicitStatusGoexit, typeWord: true}, + {name: "implicit fault", status: ExplicitStatusImplicitFault, typeWord: true}, + {name: "explicit nil", status: ExplicitStatusPanic}, + {name: "cleanup", status: ExplicitStatusPanic, typeWord: true, flags: 1 << 0}, + {name: "recover", status: ExplicitStatusPanic, typeWord: true, flags: 1 << 1}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixture := newExplicitPanicFixture(t, 1) + leaf := fixture.frames[0] + leaf.header.SuspendReason = uint16(SuspendPanic) + leaf.header.Lifecycle = uint16(FrameFinalSuspended) + leaf.header.Flags = test.flags + var typeWord unsafe.Pointer + if test.typeWord { + typeWord = unsafe.Pointer(new(byte)) + } + if PrepareExplicitStatus(fixture.g, leaf.handle, leaf.header, test.status, typeWord, unsafe.Pointer(new(byte))) { + t.Fatal("unsupported explicit terminal shape accepted") + } + if fixture.g.pending.kind != pendingNone || fixture.g.panicUnwind { + t.Fatal("rejected explicit terminal shape mutated scheduler transition") + } + if record, ok := LoadPanicRecord(fixture.g); ok || record != (PanicRecordSnapshot{}) { + t.Fatalf("rejected explicit terminal shape published record (%+v, %t)", record, ok) + } + leaf.header.Flags = 0 + if PreparePanic(fixture.g, leaf.handle, leaf.header, unsafe.Pointer(new(byte)), unsafe.Pointer(new(byte))) { + t.Fatal("poisoned one-shot record accepted a later supported panic") + } + runtime.KeepAlive(leaf.memory) + }) + } +} + +func TestExplicitPanicTerminalScheduleRaceDoesNotRedestroy(t *testing.T) { + tests := []struct { + name string + depth int + }{ + {name: "active root", depth: 1}, + {name: "suspended ancestor root", depth: 2}, + } + const iterations = 250 + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for iteration := 0; iteration < iterations; iteration++ { + fixture := newExplicitPanicFixture(t, test.depth) + fixture.publish(t, unsafe.Pointer(new(byte)), unsafe.Pointer(new(byte))) + action := fixture.beginPanicDestroy(t) + for action.Handle != fixture.frames[0].handle { + fixture.release(t, action) + var ok bool + action, ok = fixture.commitDestroyed(action) + if !ok { + t.Fatalf("iteration %d: prepare panic root destroy = (%+v, %t)", iteration, action, ok) + } + } + if test.depth == 1 && action.Kind != ActionDestroy || + test.depth > 1 && action.Kind != ActionPanicDestroy { + t.Fatalf("iteration %d: panic root destroy kind = %+v", iteration, action) + } + fixture.release(t, action) + + start := make(chan struct{}) + requestResult := make(chan bool, 1) + commitResult := make(chan struct { + action Action + ok bool + }, 1) + go func() { + <-start + requestResult <- RequestSchedule(fixture.p) + }() + go func() { + <-start + next, committed := fixture.commitDestroyed(action) + commitResult <- struct { + action Action + ok bool + }{next, committed} + }() + close(start) + requested := <-requestResult + committed := <-commitResult + if committed.ok { + if committed.action.Kind != ActionPanicComplete || requested || + preemptLoad(&fixture.p.schedule) != scheduleDisabled { + t.Fatalf("iteration %d: terminal winner = action:%+v request:%t schedule:%d", + iteration, committed.action, requested, preemptLoad(&fixture.p.schedule)) + } + } else { + if !requested || preemptLoad(&fixture.p.schedule) != scheduleRequested || + !fixture.g.destroyRoot || fixture.g.frames != nil || fixture.g.active != nil { + t.Fatalf("iteration %d: request winner partially committed terminal state", iteration) + } + if !fixture.acknowledgeTerminalSchedule(action) { + t.Fatalf("iteration %d: acknowledge panic terminal schedule", iteration) + } + committed.action, committed.ok = fixture.commitDestroyed(action) + if !committed.ok || committed.action.Kind != ActionPanicComplete { + t.Fatalf("iteration %d: retry panic terminal commit = (%+v, %t)", iteration, committed.action, committed.ok) + } + } + if _, ok := LoadPanicRecord(fixture.g); !ok || TerminalG(fixture.p, fixture.g) || ReclaimableG(fixture.g) { + t.Fatalf("iteration %d: terminal panic record/state invalid", iteration) + } + for _, frame := range fixture.frames { + runtime.KeepAlive(frame.memory) + } + } + }) + } +} diff --git a/runtime/internal/coro/frame.go b/runtime/internal/coro/frame.go index 4008b4c774..7b2cd1f05c 100644 --- a/runtime/internal/coro/frame.go +++ b/runtime/internal/coro/frame.go @@ -64,6 +64,11 @@ const ( // versioned WaitTicket is completed. The platform event source owns only // the ticket; it never resumes an LLVM handle or mutates scheduler queues. SuspendPark + // SuspendPanic is the terminal-only ExplicitStatus prototype. The active + // frame has published its two-word panic value into its owning G and reached + // final suspend. It is never used for cleanup, recover, Goexit, or an + // implicit hardware fault in this first fail-closed slice. + SuspendPanic ) // FrameState values deliberately match the lifecycle field emitted by cl. @@ -89,6 +94,7 @@ const ( pendingComplete pendingYield pendingPark + pendingPanic ) type pendingTransition struct { diff --git a/runtime/internal/coro/scheduler.go b/runtime/internal/coro/scheduler.go index 6b8829411d..3f70900521 100644 --- a/runtime/internal/coro/scheduler.go +++ b/runtime/internal/coro/scheduler.go @@ -29,6 +29,9 @@ const ( GCanceling GWaiting GDead + // GPanicking destroys suspended-await ancestors deepest-to-root after the + // active final-suspended panic frame has passed its normal done check. + GPanicking ) // G owns the stackless frame chain for one logical Go task. @@ -65,6 +68,12 @@ type G struct { taskStorage unsafe.Pointer taskSize uintptr taskState taskStorageState + + // panicRecord is task-local. It must never be discovered through TLS or a + // process-global current-G slot. panicUnwind is scheduler-thread-only and is + // set only after the published active frame returns from llvm.coro.resume. + panicRecord PanicRecord + panicUnwind bool } const ( @@ -131,6 +140,13 @@ const ( // ActionCancelComplete transfers one fully destroyed spawned G to the task // storage reclaimer. ActionCancelComplete + // ActionPanicDestroy directly destroys one suspended-await ancestor after + // the active panic frame has already gone through CheckDestroy/Destroy. + // It must never be preceded by coro.done or followed by coro.resume. + ActionPanicDestroy + // ActionPanicComplete exposes a stable task-local PanicRecord to the runtime + // adapter after every frame has been destroyed deepest-to-root. + ActionPanicComplete ) // Action is one deterministic scheduler operation or control event. Handle is @@ -142,7 +158,8 @@ type Action struct { } func setAction(p *P, kind ActionKind, handle unsafe.Pointer) (Action, bool) { - if p == nil || kind == ActionInvalid || kind == ActionComplete || kind == ActionYield || kind == ActionPark || kind == ActionCancelComplete || handle == nil { + if p == nil || kind == ActionInvalid || kind == ActionComplete || kind == ActionYield || kind == ActionPark || + kind == ActionCancelComplete || kind == ActionPanicComplete || handle == nil { return Action{}, false } action := Action{Kind: kind, Handle: handle} @@ -162,7 +179,8 @@ func InitG(g *G) bool { g.destroyTarget != nil || g.destroyRoot || g.nextReady != nil || g.queued || g.waitToken != nil || g.waitTicket != 0 || g.nextWait != nil || g.waiting || g.runP != nil || g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil || - g.taskStorage != nil || g.taskSize != 0 || g.taskState != taskStorageStatic { + g.taskStorage != nil || g.taskSize != 0 || g.taskState != taskStorageStatic || + !emptyPanicRecord(&g.panicRecord) || g.panicUnwind { return false } g.magic = gMagic @@ -529,6 +547,19 @@ func dispatchPending(g *G, resumed *Frame) (destroy *Frame, yielded bool, ok boo g.waitToken = pending.wait g.waitTicket = pending.ticket return nil, false, true + case pendingPanic: + if pending.target != nil || pending.wait != nil || pending.ticket != 0 || resumed.header == nil || + resumed.header.SuspendReason != uint16(SuspendPanic) || + resumed.header.Lifecycle != uint16(FrameFinalSuspended) || + g.panicUnwind || !publishedPanicRecord(&g.panicRecord) { + return nil, false, false + } + g.active = resumed.parent + resumed.state = FrameDestroyPending + resumed.header.Lifecycle = uint16(FrameDestroyPending) + g.destroyTarget = resumed + g.panicUnwind = true + return resumed, false, true default: return nil, false, false } @@ -656,6 +687,9 @@ func Destroyed(p *P, g *G, action Action) (Action, bool) { return Action{}, false } isRoot := g.destroyRoot + if g.panicUnwind { + return commitInitialPanicDestroyed(p, g, isRoot) + } if isRoot { if g.active != nil || g.frames != nil || !validReadyQueue(p) || !validWaitQueue(p) { return Action{}, false @@ -716,5 +750,6 @@ func TerminalG(p *P, g *G) bool { g.pending.kind == pendingNone && g.pending.from == nil && g.pending.target == nil && g.pending.wait == nil && g.pending.ticket == 0 && g.destroyTarget == nil && !g.destroyRoot && g.nextReady == nil && !g.queued && g.waitToken == nil && g.waitTicket == 0 && g.nextWait == nil && !g.waiting && g.runP == nil && - g.spawnChild == nil && g.spawnParent == nil && g.spawnP == nil && validTerminalTaskStorage(g) + g.spawnChild == nil && g.spawnParent == nil && g.spawnP == nil && validTerminalTaskStorage(g) && + emptyPanicRecord(&g.panicRecord) && !g.panicUnwind } diff --git a/runtime/internal/coro/spawn.go b/runtime/internal/coro/spawn.go index 41862e3047..abd37b4fdf 100644 --- a/runtime/internal/coro/spawn.go +++ b/runtime/internal/coro/spawn.go @@ -233,7 +233,8 @@ func ReclaimableG(g *G) bool { g.pending.wait == nil && g.pending.ticket == 0 && g.destroyTarget == nil && !g.destroyRoot && g.nextReady == nil && !g.queued && g.waitToken == nil && g.waitTicket == 0 && g.nextWait == nil && !g.waiting && g.runP == nil && - g.spawnChild == nil && g.spawnParent == nil && g.spawnP == nil && validLiveTaskStorage(g) + g.spawnChild == nil && g.spawnParent == nil && g.spawnP == nil && validLiveTaskStorage(g) && + emptyPanicRecord(&g.panicRecord) && !g.panicUnwind } // TaskStorageOwned reports the only two legal storage states at ActionComplete. diff --git a/runtime/internal/runtime/coro_explicit_status.go b/runtime/internal/runtime/coro_explicit_status.go new file mode 100644 index 0000000000..42ee0bd838 --- /dev/null +++ b/runtime/internal/runtime/coro_explicit_status.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/internal/coro" +) + +// coroPrepareExplicitPanicPrototype is intentionally not exported as a C ABI: +// compiler lowering does not yet publish SuspendPanic or prove the absence of +// cleanup/recover/Goexit/implicit-fault shapes. It demonstrates the future +// no-TLS hook boundary using only the physical G passed by generated code. +func coroPrepareExplicitPanicPrototype( + g *coroG, + handle unsafe.Pointer, + header *coro.HeaderV1, + typeWord, dataWord unsafe.Pointer, +) bool { + return coro.PreparePanic(g, handle, header, typeWord, dataWord) +} + +func coroLoadExplicitPanicPrototype(g *coroG) (coro.PanicRecordSnapshot, bool) { + return coro.LoadPanicRecord(g) +} diff --git a/runtime/internal/runtime/coro_sched.go b/runtime/internal/runtime/coro_sched.go index eefe1b9e6c..7ae041f869 100644 --- a/runtime/internal/runtime/coro_sched.go +++ b/runtime/internal/runtime/coro_sched.go @@ -156,6 +156,30 @@ func coroRunActions(p *coroP, g *coroG, action coro.Action) bool { // Retry only the scheduler commit. The LLVM handle was already // destroyed exactly once before entering this loop. } + case coro.ActionPanicDestroy: + coroHandleDestroy(action.Handle) + for { + next, committed := coro.PanicDestroyed(p, g, action) + if committed { + action, ok = next, true + break + } + if !coro.AcknowledgePanicTerminalSchedule(p, g, action) { + ok = false + break + } + // Retry only the state commit. The suspended ancestor handle was + // already destroyed exactly once. + } + case coro.ActionPanicComplete: + // The core has retained a stable task-local two-word record and has + // destroyed every frame. Printing/fatal ownership and compiler-side + // cleanup/recover semantics are not part of this prototype, so stop + // here instead of misclassifying panic as ordinary G completion. + if _, published := coro.LoadPanicRecord(g); !published { + return false + } + return false default: return false } From f3bdeeaa4c6f1f1d58959a83ac0d7588f7a4afc0 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 23:51:18 +0800 Subject: [PATCH 27/32] docs(coro): record runnable spawn and panic core --- .github/workflows/coroutine.yml | 3 ++- doc/llvm-coro-runtime-design.md | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/coroutine.yml b/.github/workflows/coroutine.yml index 0a1c4ba1b9..f0f599c9f5 100644 --- a/.github/workflows/coroutine.yml +++ b/.github/workflows/coroutine.yml @@ -106,7 +106,8 @@ jobs: if: matrix.llvm == 19 # Keep the focused workflow exhaustive for the build-side coroutine # contract. This includes park effect seeding, frozen foreign noblock - # certificates, IRQUnsafe handling, and the exact legacy PanicABI stop. + # certificates, IRQUnsafe handling, the exact legacy PanicABI stop, and + # the native linked static-spawn scheduler-island execution smoke. run: go test ./internal/build -run 'Coro|Coroutine' -timeout=10m -count=1 - name: Test coroutine compiler integration diff --git a/doc/llvm-coro-runtime-design.md b/doc/llvm-coro-runtime-design.md index 7d4b3117bf..66a15bfcda 100644 --- a/doc/llvm-coro-runtime-design.md +++ b/doc/llvm-coro-runtime-design.md @@ -1787,18 +1787,20 @@ Pure sync library/archive不需要链接scheduler。Executable一旦选择 `-sch - `program-bootstrap.v2` 在 codegen 前冻结五阶段表：`[internal runtime.init, init$abitypes, public runtime.init, selected main-package init, main.main]`。managed Go 阶段根据唯一 primary 选择 `DirectPlain` 或 `CoroRoot`；public runtime init 若存在则必须使用其 exact managed body，不存在时才由 compiler 生成 no-op。Coro 表项只绑定 package anchor/descriptor index，不复制函数体，也不把 catalog 当启动列表。 - planner 已把 internal runtime init、selected package init 和 `main.main` 注入 managed demand。普通同步 Go/标准库调用风格不变，调用者根据精确 effect 自动被染成 coro；scheduler-stack hook closure 则是单独审计的 NoSuspend island，不能通过强改 demand 或放宽 trusted closure 绕过。 - frozen foreign `//llgo:coro noblock` certificate 当前只授予已审计的 `time`、`pthread_self`、`pthread_mutex_init` 和 `pthread_mutex_unlock`。证书只移除未知阻塞，`IRQUnsafe` 仍保留但允许在普通 G 上执行。真实 runtime init 仍被 `pthread_key_create`、`rand`/`srand`、`GC_malloc`、mutex lock、Memcpy/Memset 等未完成边界挡住。 -- legacy PanicABI 仍是完整启动链的正式 blocker。exact proof 可追踪 `runtime.Panic → Rethrow → TracePanic → printany`，并在动态 `error.Error` 调用处停止；这里必须落地 non-legacy task-local PanicABI/descriptor dispatch，不能把动态调用误标为 plain。 +- legacy PanicABI 仍是完整启动链的正式 blocker。exact proof 可追踪 `runtime.Panic → Rethrow → TracePanic → printany`，并在动态 `error.Error` 调用处停止；不能把该动态调用误标为 plain。新的 `llgo.coro.panic.explicit-status.v0` 已进入 digest、summary、cache、manifest 和 package/root ABI hash，但 active compiler build 仍全局 fail closed，直到下述 runtime core 有对应 compiler lowering。 - 多基本块 CFG、聚合值、PHI 和抢占 lowering 已完成。自然循环、循环入口及每 64 条有效指令的长直线块插入 poll；scheduler 的 P 级原子 request 只有在 slow path 才执行 publish/yield/`llvm.coro.suspend`，fast path 不切换。LLVM 19–22 上均有 native64/wasm32 pre-/post-CoroSplit 与 object 测试。 - 第一条 production `go` 路径已经落地：严格限定为 closed static、top-level、非捕获、非泛型、非变参、零返回的 `go f(args)`。编译器先按 Go 顺序完整求值参数，再以显式 parent G 执行 begin，调用 target 唯一的 `DirectCoro` primary 到 LLVM initial suspend，commit 后在 parent 上 poll/yield；runtime 不接收用户 callback，也不依赖 TLS。owner 与 target 都由精确 `YieldOnly` seed 进入 effect 传播，因此 target 即使当前很短也保留抢占点，普通同步 caller 则透明 await 同一主体。 - Command `main` 的正常 continuation 现在显式通知 runtime。main root 完成后，single-P shutdown 先整体校验 ready/wait/current/action 状态，再封闭调度 gate，按 FIFO 取 ready G、按 active-child 到 root 顺序直接 `llvm.coro.destroy`，最后每个 task storage 只释放一次。该 v1 路径只接收 `YieldOnly|AwaitStructured` target 且拒绝非空 wait set；panic/Goexit 不经过正常 main-return hook。 +- terminal-only ExplicitStatus runtime core 已有 task-local 两字 `PanicRecord` 和原子 once publication。active panic frame 先经过 `coro.done` 验证并 destroy，之后 suspended-await ancestor 不再 resume，而是从深到 root 直接 destroy；最终保留 record 并返回独立 `PanicComplete`。该原型明确拒绝 nil type word、cleanup/recover flags、Goexit、implicit fault 和重复发布；尚未导出 compiler C hook，也未实现 defer/recover 或用户 `Error/String` 报告。 - park/wake handshake 已落地 32-bit 原子 `WaitToken`、generation ticket、early/late completion、唯一 waiter claim、ABA 范围校验及 terminal gate。精确 intrinsic `llgo.coroPark(token, ticket)` 被 Effect 分析识别为 `MayPark`，并在调用者当前 LLVM frame 中生成 park prepare、stateID、`coro.suspend` 和恢复路径；没有隐藏在普通同步 helper 中。channel/timer/syscall 的 submit/retry producer 尚未接入。 - wait/preempt core 要求目标提供可靠的 32-bit atomic load/store/CAS。WASM 可直接满足；带 A 扩展的 RISC-V 可满足；ESP32-C3 RV32IMC 当前会在链接时缺少 `__atomic_*_4`，直到平台用 IRQ critical section 提供单核适配。这里故意不使用非原子 fallback。 - `wasip1`、`wasip2` 和 `wasm-unknown` 明确选择 leaking/nogc frame backend，不依赖 libuv 或 BDWGC。`wasip2` 与 `wasm-unknown` 已通过真实 `llgo build -target=...`、wasm magic/symbol closure、无 `GC_*`/undefined 检查，并由 wasmtime 运行返回 0。当前 `wasip2` 产物是 Preview 2 目标的 core module，尚不是 WIT component。 - frame allocator 已有 conservative BDWGC、nogc/WASM malloc 和 tinygogc/baremetal 后端。跨 suspend 的 pointer 目前只在 conservative 或 non-collecting 配置下安全；精确 frame root map、write barrier、STW、weak timer/finalizer 与 cleanup 语义尚未实现，不能据此宣称完整 Go GC 兼容。 -- deterministic single-P runtime 已能管理多个 frame、ready queue、preempt request、park/wake、closed-static spawned G、正常 main-return ready-child cancellation 和 terminal idle/requested/stopping/disabled 状态。尚无动态/closure/method `go` target、等待中 G 的 producer 解注册与取消、真实 tick/alarm request source、channel/select/sync slow path、timer/netpoll、异步 syscall submit/retry、task-local panic/defer/recover/Goexit 或多 P。 -- 完整真实 `entry → allocator → v2 factory → runtime/package init → main → scheduler` linked smoke 仍受上述 runtime/Panic/foreign blockers 限制；现有 runtime adapter 测试和 freestanding wasm CLI fixture 分别证明 scheduler ABI 与目标链接，不能合并表述为完整 Go runtime 已经端到端运行。 +- deterministic single-P runtime 已能管理多个 frame、ready queue、preempt request、park/wake、closed-static spawned G、正常 main-return ready-child cancellation、terminal panic frame destruction和 idle/requested/stopping/disabled 状态。尚无动态/closure/method `go` target、等待中 G 的 producer 解注册与取消、真实 tick/alarm request source、channel/select/sync slow path、timer/netpoll、异步 syscall submit/retry、完整 panic/defer/recover/Goexit 或多 P。 +- native+nogc scheduler-island 已把真实 nested static `go` lowering、V2 entry/factory/control wrapper、production scheduler/spawn/shutdown/coroalloc 最终链接并执行。确定性 fixture 验证 `Before=1, After=0, Leaf=0`，最终符号审计同时要求 production `CommitSpawn`/`BeginCommandShutdown` 且禁止 legacy `Panic/Rethrow/TracePanic/printany`。该测试以四个 bounded init no-op 和 fail-stop nil-check/libc allocation stub 隔离完整标准库 runtime，因此证明的是可运行 scheduler 原型，不是完整 runtime 启动兼容。 +- 完整真实 `entry → allocator → v2 factory → runtime/package init → main → scheduler` linked smoke 仍受上述 runtime/Panic/foreign blockers 限制；scheduler-island、runtime adapter 和 freestanding wasm CLI fixture 各自证明的边界不能合并表述为完整 Go runtime 已经端到端运行。 - 当前 cache digest 只解决同一完整程序计划下的内部 package cache；未知未来 caller 可复用的预编译 archive/标准库仍需 producer summary、canonical boundary Dispatch 和 linker ABI 校验。 -- 后续依赖顺序是：先解除完整 runtime 链的 non-legacy PanicABI/动态 `error.Error` blocker，并为 WaitToken 增加可注销、可静默迟到 completion 的稳定 registration；再接真实 platform request source 与 channel/timer/syscall producer并跑完整 linked smoke；随后补 suspended-frame GC、defer/recover/Goexit、多 P 与各 target event backend。动态/closure/method `go` target只在 canonical descriptor transport 完成后开启。所有阶段保持无栈、单 primary 和未证明即 fail closed。 +- 后续依赖顺序是：先为 terminal ExplicitStatus core 增加 compiler `SuspendPanic`/hook lowering并保持 cleanup/implicit fault fail closed，同时为 WaitToken 增加可注销、可静默迟到 completion 的稳定 registration；再实现 dynamic `error.Error`/`Stringer` descriptor、真实 platform request source 与 channel/timer/syscall producer并跑完整 runtime linked smoke；随后补 suspended-frame GC、defer/recover/Goexit、多 P 与各 target event backend。动态/closure/method `go` target只在 canonical descriptor transport 完成后开启。所有阶段保持无栈、单 primary 和未证明即 fail closed。 ### Phase 1：单 P deterministic scheduler From 7448583cad7ec814f96013bd59f92dc333adb896 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Fri, 17 Jul 2026 00:30:48 +0800 Subject: [PATCH 28/32] runtime(coro): export terminal panic handoff --- runtime/internal/coro/explicit_status.go | 32 +++++-- runtime/internal/coro/explicit_status_test.go | 43 ++++++++++ .../internal/runtime/coro_explicit_status.go | 40 --------- runtime/internal/runtime/coro_program_test.go | 85 +++++++++++++++++++ runtime/internal/runtime/coro_sched.go | 20 +++++ 5 files changed, 175 insertions(+), 45 deletions(-) delete mode 100644 runtime/internal/runtime/coro_explicit_status.go diff --git a/runtime/internal/coro/explicit_status.go b/runtime/internal/coro/explicit_status.go index e5b9f45f09..c7ef69b399 100644 --- a/runtime/internal/coro/explicit_status.go +++ b/runtime/internal/coro/explicit_status.go @@ -78,6 +78,31 @@ func LoadPanicRecord(g *G) (PanicRecordSnapshot, bool) { }, true } +func validPanicAncestor(g *G, frame *Frame) bool { + return frame != nil && frame.owner == g && frame.handle != nil && frame.header != nil && + frame.state == FrameSuspended && frame.header.G == unsafe.Pointer(g) && frame.header.Flags == 0 && + frame.header.SuspendReason == uint16(SuspendCall) && + frame.header.Lifecycle == uint16(FrameSuspended) +} + +// validPanicAncestry proves before publication that every continuation which +// terminal panic unwinding would bypass is a plain suspended await. Version +// zero has no cleanup/recover transport, so any non-zero flags reject the +// entire operation before the active frame or an ancestor can be destroyed. +func validPanicAncestry(g *G, active *Frame) bool { + if g == nil || active == nil || active.header == nil { + return false + } + child := active + for ancestor := active.parent; ancestor != nil; ancestor = ancestor.parent { + if !validPanicAncestor(g, ancestor) || child.header.Parent != ancestor.handle { + return false + } + child = ancestor + } + return child == g.root && child.header.Parent == nil +} + // PrepareExplicitStatus is the independently testable core of the future // compiler hook. Only ExplicitStatusPanic is accepted. The first caller owns // the publication attempt; any malformed winner permanently poisons the record @@ -119,7 +144,7 @@ func PrepareExplicitStatus( if frame == nil || frame != g.active || frame.owner != g || frame.header != header || frame.state != FrameActive || header.G != unsafe.Pointer(g) || header.SuspendReason != uint16(SuspendPanic) || - header.Lifecycle != uint16(FrameFinalSuspended) { + header.Lifecycle != uint16(FrameFinalSuspended) || !validPanicAncestry(g, frame) { return reject() } @@ -142,10 +167,7 @@ func PreparePanic(g *G, handle unsafe.Pointer, header *HeaderV1, typeWord, dataW func preparePanicAncestor(p *P, g *G, frame *Frame) (Action, bool) { if p == nil || g == nil || frame == nil || p.current != g || g.state != GPanicking || !g.panicUnwind || !publishedPanicRecord(&g.panicRecord) || g.destroyTarget != nil || - frame != g.active || frame.owner != g || frame.handle == nil || frame.header == nil || - frame.state != FrameSuspended || frame.header.G != unsafe.Pointer(g) || - frame.header.SuspendReason != uint16(SuspendCall) || - frame.header.Lifecycle != uint16(FrameSuspended) { + frame != g.active || !validPanicAncestor(g, frame) { return Action{}, false } handle := frame.handle diff --git a/runtime/internal/coro/explicit_status_test.go b/runtime/internal/coro/explicit_status_test.go index 30fac729ef..0439d6db5d 100644 --- a/runtime/internal/coro/explicit_status_test.go +++ b/runtime/internal/coro/explicit_status_test.go @@ -319,6 +319,49 @@ func TestExplicitStatusUnsupportedShapesFailClosed(t *testing.T) { } } +func TestExplicitPanicRejectsUnsupportedAncestorBeforeDestroy(t *testing.T) { + fixture := newExplicitPanicFixture(t, 2) + root, leaf := fixture.frames[0], fixture.frames[1] + rootMetadata, leafMetadata := FrameFromStorage(root.storage), FrameFromStorage(leaf.storage) + root.header.Flags = 1 // cleanup/recover metadata is not representable in v0. + leaf.header.SuspendReason = uint16(SuspendPanic) + leaf.header.Lifecycle = uint16(FrameFinalSuspended) + if PreparePanic(fixture.g, leaf.handle, leaf.header, unsafe.Pointer(new(byte)), unsafe.Pointer(new(byte))) { + t.Fatal("panic with unsupported suspended ancestor was published") + } + if fixture.g.pending.kind != pendingNone || fixture.g.panicUnwind || fixture.g.destroyTarget != nil || + leafMetadata.state != FrameActive || rootMetadata.state != FrameSuspended || + leaf.header.Lifecycle != uint16(FrameFinalSuspended) || root.header.Lifecycle != uint16(FrameSuspended) { + t.Fatal("rejected ancestor cleanup mutated frame destruction state") + } + if record, ok := LoadPanicRecord(fixture.g); ok || record != (PanicRecordSnapshot{}) { + t.Fatalf("rejected ancestor cleanup published record (%+v, %t)", record, ok) + } + runtime.KeepAlive(root.memory) + runtime.KeepAlive(leaf.memory) +} + +func TestExplicitPanicRechecksAncestorBeforeDirectDestroy(t *testing.T) { + fixture := newExplicitPanicFixture(t, 2) + root := fixture.frames[0] + rootMetadata := FrameFromStorage(root.storage) + fixture.publish(t, unsafe.Pointer(new(byte)), unsafe.Pointer(new(byte))) + action := fixture.beginPanicDestroy(t) + fixture.release(t, action) + + // Model corrupted or version-skewed metadata after publication. The active + // panic frame may already be gone, but the unsupported ancestor must never + // be directly destroyed or resumed. + root.header.Flags = 1 + next, ok := Destroyed(fixture.p, fixture.g, action) + if ok || next != (Action{}) || fixture.g.destroyTarget != nil || + rootMetadata.state != FrameSuspended || root.header.Lifecycle != uint16(FrameSuspended) { + t.Fatalf("unsupported ancestor entered direct destroy: action=(%+v, %t), state=%d lifecycle=%d", + next, ok, rootMetadata.state, root.header.Lifecycle) + } + runtime.KeepAlive(root.memory) +} + func TestExplicitPanicTerminalScheduleRaceDoesNotRedestroy(t *testing.T) { tests := []struct { name string diff --git a/runtime/internal/runtime/coro_explicit_status.go b/runtime/internal/runtime/coro_explicit_status.go deleted file mode 100644 index 42ee0bd838..0000000000 --- a/runtime/internal/runtime/coro_explicit_status.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package runtime - -import ( - "unsafe" - - "github.com/goplus/llgo/runtime/internal/coro" -) - -// coroPrepareExplicitPanicPrototype is intentionally not exported as a C ABI: -// compiler lowering does not yet publish SuspendPanic or prove the absence of -// cleanup/recover/Goexit/implicit-fault shapes. It demonstrates the future -// no-TLS hook boundary using only the physical G passed by generated code. -func coroPrepareExplicitPanicPrototype( - g *coroG, - handle unsafe.Pointer, - header *coro.HeaderV1, - typeWord, dataWord unsafe.Pointer, -) bool { - return coro.PreparePanic(g, handle, header, typeWord, dataWord) -} - -func coroLoadExplicitPanicPrototype(g *coroG) (coro.PanicRecordSnapshot, bool) { - return coro.LoadPanicRecord(g) -} diff --git a/runtime/internal/runtime/coro_program_test.go b/runtime/internal/runtime/coro_program_test.go index 6a3efa4be8..4df93fad8e 100644 --- a/runtime/internal/runtime/coro_program_test.go +++ b/runtime/internal/runtime/coro_program_test.go @@ -196,6 +196,9 @@ type coroProgramTestDriverV1 struct { completeReady bool released bool requestScheduleOnDestroy bool + panicOnResume bool + panicTypeWord unsafe.Pointer + panicDataWord unsafe.Pointer spawnOnMainReturn bool child *coro.G childFrame *coroProgramTestFrameV1 @@ -270,6 +273,19 @@ func (driver *coroProgramTestDriverV1) resume(handle unsafe.Pointer) { frame := driver.frame frame.header.SuspendReason = uint16(coro.SuspendNone) frame.header.Lifecycle = uint16(coro.FrameActive) + if driver.panicOnResume { + frame.header.SuspendReason = uint16(coro.SuspendPanic) + frame.header.Lifecycle = uint16(coro.FrameFinalSuspended) + __llgo_coro_panic_prepare_v1( + unsafe.Pointer(frame.g), + handle, + unsafe.Pointer(frame.header), + driver.panicTypeWord, + driver.panicDataWord, + ) + driver.completeReady = true + return + } if driver.spawnOnMainReturn { driver.child = new(coro.G) if !coro.BeginSpawn(frame.g, driver.child, unsafe.Pointer(driver.child), coro.TaskStorageSize()) { @@ -434,6 +450,75 @@ func TestCoroProgramTerminalScheduleRetryDoesNotRedestroy(t *testing.T) { runtime.KeepAlive(manifest) } +func requireCoroProgramRuntimeAbort(t *testing.T, want string, call func()) { + t.Helper() + defer func() { + recovered := recover() + if recovered != want { + t.Fatalf("coroutine runtime abort = %#v, want %q", recovered, want) + } + }() + call() + t.Fatal("coroutine runtime ABI violation returned after abort") +} + +func TestCoroProgramExplicitPanicHookAndTerminalDispatcherFailClosed(t *testing.T) { + resetCoroProgramTestStateV1(t) + manifest := newCoroProgramTestManifestV1() + factory := unsafe.Pointer(&manifest.factoryMarker) + + gPointer, ok := coroProgramBeginV1(unsafe.Pointer(&manifest.manifest), factory) + if !ok { + t.Fatal("begin explicit-panic coroutine program") + } + frame := newCoroProgramTestFrameV1(t, &coroProgramGV1State) + typeWord, dataWord := new(byte), new(byte) + driver := &coroProgramTestDriverV1{ + t: t, + frame: frame, + panicOnResume: true, + panicTypeWord: unsafe.Pointer(typeWord), + panicDataWord: unsafe.Pointer(dataWord), + } + activeCoroProgramDriver = driver + if coroProgramRunV1(gPointer, frame.handle) { + t.Fatal("ActionPanicComplete was misclassified as normal program completion") + } + record, published := coro.LoadPanicRecord(&coroProgramGV1State) + if !published || record.Status != coro.ExplicitStatusPanic || + record.TypeWord != unsafe.Pointer(typeWord) || record.DataWord != unsafe.Pointer(dataWord) { + t.Fatalf("terminal adapter panic record = (%+v, %t)", record, published) + } + if coroProgramLifecycleV1State != coroProgramFailedV1 || + driver.doneCalls != 2 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released || + coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) || coro.ReclaimableG(&coroProgramGV1State) { + t.Fatalf("explicit panic adapter = lifecycle:%d done:%d resume:%d destroy:%d released:%t", + coroProgramLifecycleV1State, driver.doneCalls, driver.resumeCalls, driver.destroyCalls, driver.released) + } + + // Publication is once-only at the exported boundary as well: a duplicate + // compiler hook is a non-returning ABI violation, never a normal result. + requireCoroProgramRuntimeAbort(t, "invalid coroutine panic handoff", func() { + __llgo_coro_panic_prepare_v1( + gPointer, + frame.handle, + unsafe.Pointer(frame.header), + unsafe.Pointer(typeWord), + unsafe.Pointer(dataWord), + ) + }) + runtime.KeepAlive(typeWord) + runtime.KeepAlive(dataWord) + runtime.KeepAlive(frame.memory) + runtime.KeepAlive(manifest) +} + +func TestCoroProgramExplicitPanicHookRejectsInvalidPhysicalG(t *testing.T) { + requireCoroProgramRuntimeAbort(t, "invalid coroutine panic handoff", func() { + __llgo_coro_panic_prepare_v1(nil, nil, nil, nil, nil) + }) +} + func TestCoroProgramNormalMainReturnCancelsReadyChild(t *testing.T) { resetCoroProgramTestStateV1(t) manifest := newCoroProgramTestManifestV1() diff --git a/runtime/internal/runtime/coro_sched.go b/runtime/internal/runtime/coro_sched.go index 7ae041f869..556ae8b2b4 100644 --- a/runtime/internal/runtime/coro_sched.go +++ b/runtime/internal/runtime/coro_sched.go @@ -188,3 +188,23 @@ func coroRunActions(p *coroP, g *coroG, action coro.Action) bool { } } } + +// __llgo_coro_panic_prepare_v1 is the compiler-to-runtime terminal panic +// handoff. The physical G is an explicit ABI argument: this boundary must +// never discover scheduler ownership through TLS or a process-global current +// G. A rejected once-only publication is a terminal ABI violation and aborts +// immediately, so malformed cleanup/recover/Goexit/implicit-fault lowering +// cannot resume ordinary execution on a poisoned G. +// +//export __llgo_coro_panic_prepare_v1 +func __llgo_coro_panic_prepare_v1(g, handle, header, typeWord, dataWord unsafe.Pointer) { + if !coro.PreparePanic( + (*coro.G)(g), + handle, + (*coro.HeaderV1)(header), + typeWord, + dataWord, + ) { + coroRuntimeAbort("invalid coroutine panic handoff") + } +} From 1c8829e832e74513b935522f04bc78a360a37320 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Fri, 17 Jul 2026 00:30:59 +0800 Subject: [PATCH 29/32] compiler(coro): lower terminal explicit panic status --- cl/compilation.go | 8 +- cl/compilation_test.go | 6 +- cl/compile.go | 3 + cl/coro_abi.go | 147 +++++++++++++++- cl/coro_entry.go | 15 +- cl/coro_panic.go | 46 +++++ cl/coro_panic_test.go | 373 +++++++++++++++++++++++++++++++++++++++++ ssa/interface.go | 12 ++ 8 files changed, 591 insertions(+), 19 deletions(-) create mode 100644 cl/coro_panic.go create mode 100644 cl/coro_panic_test.go diff --git a/cl/compilation.go b/cl/compilation.go index a692a7c704..2f2a36b2c3 100644 --- a/cl/compilation.go +++ b/cl/compilation.go @@ -51,10 +51,10 @@ type Compilation struct { SchedulerABI string PanicABI string FuncRepABI string - // EnableCoroExplicitStatusPanicABI selects the reserved target-wide - // explicit-status panic identity. This slice does not implement its hidden - // outcome, cleanup, or runtime protocol, so active code generation remains - // fail-closed when the capability is selected. + // EnableCoroExplicitStatusPanicABI selects the target-wide explicit-status + // panic identity. The first lowering slice accepts only exact cleanup-free + // physical coroutine bodies whose explicit panic payload can outlive frame + // destruction; every wider hidden-outcome or unwind shape remains fail-closed. EnableCoroExplicitStatusPanicABI bool // EnableCoroPhysicalABI permits the conservative leaf-only coroutine ABI // lowering implemented by the current experimental slice. It requires entry diff --git a/cl/compilation_test.go b/cl/compilation_test.go index 91dc8163ee..2b0dc0794f 100644 --- a/cl/compilation_test.go +++ b/cl/compilation_test.go @@ -123,7 +123,10 @@ func TestCompilationCoroABIIdentityValidation(t *testing.T) { } newExplicitStatus := func() *Compilation { compilation := newPhysical() + compilation.EnableCoroChildAwait = true compilation.EnableCoroExplicitStatusPanicABI = true + compilation.CoroABI = coro.PhysicalABIV1 + compilation.SchedulerABI = coro.SchedulerChildAwaitABIV0 compilation.PanicABI = coro.PanicExplicitStatusABIV0 return compilation } @@ -144,8 +147,7 @@ func TestCompilationCoroABIIdentityValidation(t *testing.T) { if err := withoutExplicitStatusEntry.preflightCoroPlan(); err == nil || !strings.Contains(err.Error(), "requires coroutine entry resolution") { t.Fatalf("explicit-status panic ABI preflight dependency error = %v", err) } - if err := explicitStatus.preflightCoroPlan(); err == nil || - !strings.Contains(err.Error(), "identity-only") || !strings.Contains(err.Error(), "runtime semantics are not implemented") { + if err := explicitStatus.preflightCoroPlan(); err == nil || !strings.Contains(err.Error(), "requires a compilation CoroPlan") { t.Fatalf("explicit-status panic ABI active preflight error = %v", err) } newChildAwait := func() *Compilation { diff --git a/cl/compile.go b/cl/compile.go index f238c4ef83..934addff65 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -1709,6 +1709,9 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { p.recordPanicLocation(b, v.Pos()) b.RunDefers() case *ssa.Panic: + if p.tryCompileCoroExplicitStatusPanic(b, v) { + return + } arg := p.compileValue(b, v.X) p.recordPanicLocation(b, v.Pos()) b.Panic(arg) diff --git a/cl/coro_abi.go b/cl/coro_abi.go index 2ca2c64ef5..f558355c51 100644 --- a/cl/coro_abi.go +++ b/cl/coro_abi.go @@ -46,6 +46,7 @@ const ( coroPreemptPollHookV1 = "__llgo_coro_preempt_poll_v1" coroYieldPrepareHookV1 = "__llgo_coro_yield_prepare_v1" coroParkPrepareHookV1 = "__llgo_coro_park_prepare_v1" + coroPanicPrepareHookV1 = "__llgo_coro_panic_prepare_v1" coroSpawnBeginHookV1 = "__llgo_coro_spawn_begin_v1" coroSpawnCommitHookV1 = "__llgo_coro_spawn_commit_v1" coroCompletePrepareHookV1 = "__llgo_coro_complete_prepare_v1" @@ -71,6 +72,7 @@ const ( coroSuspendFrameComplete coroSuspendYield coroSuspendPark + coroSuspendPanic ) const ( @@ -99,6 +101,7 @@ type coroPhysicalABI struct { preemptPollHook string yieldPrepareHook string parkPrepareHook string + panicPrepareHook string completePrepareHook string physicalSig *types.Signature resultSlotType types.Type @@ -115,11 +118,14 @@ type coroBodyContext struct { task llssa.Expr resultSlot llssa.Expr completion llssa.BasicBlock + finalSuspend llssa.BasicBlock preemptPoll llssa.Expr yieldPrepare llssa.Expr parkPrepare llssa.Expr + panicPrepare llssa.Expr completePrepare llssa.Expr nextState uint32 + terminalState uint32 needsPreempt bool instructions int } @@ -134,6 +140,7 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type preemptPollHook := "" yieldPrepareHook := "" parkPrepareHook := "" + panicPrepareHook := "" completePrepareHook := "" if p.compilation != nil && p.compilation.EnableCoroChildAwait { version = coroPhysicalABIVersionV1 @@ -147,6 +154,9 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type parkPrepareHook = coroParkPrepareHookV1 completePrepareHook = coroCompletePrepareHookV1 } + if p.compilation != nil && p.compilation.EnableCoroExplicitStatusPanicABI { + panicPrepareHook = coroPanicPrepareHookV1 + } resultFields := make([]*types.Var, sourceSig.Results().Len()) for i := range resultFields { resultFields[i] = types.NewField(token.NoPos, nil, fmt.Sprintf("r%d", i), sourceSig.Results().At(i).Type(), false) @@ -223,6 +233,7 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type preemptPollHook: preemptPollHook, yieldPrepareHook: yieldPrepareHook, parkPrepareHook: parkPrepareHook, + panicPrepareHook: panicPrepareHook, completePrepareHook: completePrepareHook, physicalSig: physicalSig, resultSlotType: resultSlotType, @@ -308,6 +319,9 @@ func (p *context) beginCoroBody(b llssa.Builder, abi coroPhysicalABI) *coroBodyC if abi.parkPrepareHook != "" { body.parkPrepare = p.pkg.NewFunc(abi.parkPrepareHook, coroParkPrepareSignature(), llssa.InC).Expr } + if abi.panicPrepareHook != "" { + body.panicPrepare = p.pkg.NewFunc(abi.panicPrepareHook, coroPanicPrepareSignature(), llssa.InC).Expr + } if abi.preemptPollHook != "" { body.preemptPoll = p.pkg.NewFunc(abi.preemptPollHook, coroPreemptPollSignature(), llssa.InC).Expr } @@ -402,6 +416,18 @@ func coroPreemptPollSignature() *types.Signature { return types.NewSignatureType(nil, nil, nil, params, results, false) } +func coroPanicPrepareSignature() *types.Signature { + pointer := types.Typ[types.UnsafePointer] + params := types.NewTuple( + types.NewParam(token.NoPos, nil, "g", pointer), + types.NewParam(token.NoPos, nil, "handle", pointer), + types.NewParam(token.NoPos, nil, "header", pointer), + types.NewParam(token.NoPos, nil, "typeWord", pointer), + types.NewParam(token.NoPos, nil, "dataWord", pointer), + ) + return types.NewSignatureType(nil, nil, nil, params, nil, false) +} + func (c *coroBodyContext) publishState(b llssa.Builder, reason, lifecycle uint64, stateID uint32) { prog := b.Prog b.Store(b.FieldAddr(c.header, coroHeaderSuspendReason), prog.IntVal(reason, prog.Uint16())) @@ -493,17 +519,43 @@ func (c *coroBodyContext) countInstructionAndMaybeYield(b llssa.Builder) { c.instructions++ } -func (c *coroBodyContext) finish(b llssa.Builder) { +func (c *coroBodyContext) terminalStateID() uint32 { + if c.terminalState == 0 { + c.terminalState = c.nextState + c.nextState++ + } + return c.terminalState +} + +func (c *coroBodyContext) complete(b llssa.Builder) { if c.abi.version < coroPhysicalABIVersionV1 { - c.coro.Finish() + b.Jump(c.finalSuspend) return } - stateID := c.nextState - c.nextState++ - c.publishState(b, coroSuspendFrameComplete, coroLifecycleFinalSuspended, stateID) + c.publishState(b, coroSuspendFrameComplete, coroLifecycleFinalSuspended, c.terminalStateID()) if !c.completePrepare.IsNil() { b.Call(c.completePrepare, c.task, c.coro.Handle(), b.Convert(b.Prog.VoidPtr(), c.header)) } + b.Jump(c.finalSuspend) +} + +func (c *coroBodyContext) panic(b llssa.Builder, typeWord, dataWord llssa.Expr) { + if c.abi.version < coroPhysicalABIVersionV1 || c.panicPrepare.IsNil() || c.finalSuspend == nil { + panic("explicit-status panic requires a PhysicalABIV1 prepare hook and shared final suspend") + } + c.publishState(b, coroSuspendPanic, coroLifecycleFinalSuspended, c.terminalStateID()) + b.Call( + c.panicPrepare, + c.task, + c.coro.Handle(), + b.Convert(b.Prog.VoidPtr(), c.header), + b.Convert(b.Prog.VoidPtr(), typeWord), + b.Convert(b.Prog.VoidPtr(), dataWord), + ) + b.Jump(c.finalSuspend) +} + +func (c *coroBodyContext) finish(b llssa.Builder) { c.coro.Finish() } @@ -544,6 +596,7 @@ func (p *context) compileCoroPhysicalBody(b llssa.Builder, fn *ssa.Function, abi } p.coroSourceBlocks = sourceBlocks physical.completion = p.fn.MakeBlock() + physical.finalSuspend = p.fn.MakeBlock() b.SetBlock(physical.coro.InitialResumeBlock()) physical.activate(b) b.Jump(sourceBlocks[0]) @@ -584,11 +637,13 @@ func (p *context) compileCoroPhysicalBody(b llssa.Builder, fn *ssa.Function, abi } b.SetBlock(physical.completion) + physical.complete(b) + b.SetBlock(physical.finalSuspend) physical.finish(b) } func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, childAwait, programRun bool) error { - return validateCoroPhysicalABIWithUniverseCapabilities(fn, plan, whole, nil, childAwait, programRun, false) + return validateCoroPhysicalABIWithUniverseCapabilities(fn, plan, whole, nil, childAwait, programRun, false, false) } // validateCoroPhysicalABIWithUniverse is the production preflight. The @@ -597,11 +652,14 @@ func validateCoroPhysicalABI(fn *ssa.Function, plan coro.FunctionPlan, whole *co // The wrapper above is retained for narrow structural unit tests; active // Compilation paths always call this form with their frozen universe. func validateCoroPhysicalABIWithUniverse(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, universe *EmissionUniverse, childAwait, programRun bool) error { - return validateCoroPhysicalABIWithUniverseCapabilities(fn, plan, whole, universe, childAwait, programRun, false) + return validateCoroPhysicalABIWithUniverseCapabilities(fn, plan, whole, universe, childAwait, programRun, false, false) } -func validateCoroPhysicalABIWithUniverseCapabilities(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, universe *EmissionUniverse, childAwait, programRun, staticSpawn bool) error { +func validateCoroPhysicalABIWithUniverseCapabilities(fn *ssa.Function, plan coro.FunctionPlan, whole *coro.SSAPlan, universe *EmissionUniverse, childAwait, programRun, staticSpawn, explicitPanic bool) error { if !childAwait { + if explicitPanic { + return fmt.Errorf("coroutine physical ABI: function %q: explicit-status panic requires PhysicalABIV1 child-await lowering", plan.ID) + } return validateCoroLeafPhysicalABI(fn, plan) } @@ -676,6 +734,7 @@ func validateCoroPhysicalABIWithUniverseCapabilities(fn *ssa.Function, plan coro } returns := 0 + panics := 0 awaits := 0 parks := 0 spawns := 0 @@ -699,6 +758,14 @@ func validateCoroPhysicalABIWithUniverseCapabilities(fn *ssa.Function, plan coro case *ssa.DebugRef, *ssa.Jump: case *ssa.Return: returns++ + case *ssa.Panic: + if !explicitPanic { + return coroLeafInstructionError(fn, plan, instr, "explicit panic requires the explicit-status panic ABI") + } + if reason := validateCoroExplicitStatusPanic(pureSSA, instr); reason != "" { + return coroLeafInstructionError(fn, plan, instr, reason) + } + panics++ case *ssa.If: if !coroLeafScalar(instr.Cond.Type()) { return coroLeafInstructionError(fn, plan, instr, "non-scalar branch condition") @@ -740,6 +807,14 @@ func validateCoroPhysicalABIWithUniverseCapabilities(fn *ssa.Function, plan coro awaits++ continue } + if explicitPanic { + if _, targetPlan, plainErr := resolveCoroStaticPlainCall(whole, instr); plainErr == nil { + return coroLeafInstructionError(fn, plan, instr, fmt.Sprintf( + "direct plain target %q (exec=%s) has no certified explicit-status hidden-outcome/unwind contract", + targetPlan.ID, targetPlan.Exec, + )) + } + } if !programRun { return coroLeafInstructionError(fn, plan, instr, "unsupported child await: "+err.Error()) } @@ -769,6 +844,9 @@ func validateCoroPhysicalABIWithUniverseCapabilities(fn *ssa.Function, plan coro if returns == 0 { return fail("requires at least one return instruction") } + if panics != 0 && !plan.Exec.Contains(coro.MayUnwind) { + return fail("explicit panic body lacks may-unwind execution classification: %s", plan.Exec) + } if !plan.Effect.MaySuspend() { return fail("CFG physical body lacks a suspension-capable final effect: %s", plan.Effect) } @@ -796,6 +874,59 @@ func validateCoroPhysicalABIWithUniverseCapabilities(fn *ssa.Function, plan coro return nil } +func validateCoroExplicitStatusPanic(audit *coroPhysicalPureSSAAudit, instruction *ssa.Panic) string { + if instruction == nil || instruction.X == nil { + return "explicit-status panic requires a non-nil operand" + } + boxed, ok := instruction.X.(*ssa.MakeInterface) + if !ok || boxed.X == nil { + return "explicit-status panic requires one concrete MakeInterface operand" + } + if boxed.Parent() != instruction.Parent() { + return "explicit-status panic MakeInterface belongs to a different SSA body" + } + refs := boxed.Referrers() + if refs == nil || len(*refs) != 1 || (*refs)[0] != instruction { + return "explicit-status panic requires its MakeInterface to have the panic site as its sole consumer" + } + target, ok := types.Unalias(boxed.Type()).Underlying().(*types.Interface) + if !ok || !target.Empty() { + return "explicit-status panic requires an empty-interface MakeInterface result" + } + if isUntypedNilConst(boxed.X) { + return "explicit-status panic does not yet support an untyped nil value" + } + source := boxed.X.Type() + if audit != nil { + source = audit.typeOf(source) + } + if source == nil { + return "explicit-status panic MakeInterface has no concrete source type" + } + if _, ok := types.Unalias(source).Underlying().(*types.Pointer); !ok { + return "explicit-status panic currently requires one concrete pointer payload" + } + if audit == nil { + return "explicit-status panic requires a prepared pure-SSA audit" + } + if reason := audit.validateMakeInterface(boxed); reason != "" { + return "explicit-status panic MakeInterface is not pure: " + reason + } + if constant, ok := boxed.X.(*ssa.Const); ok && constant.Value == nil { + // A typed nil pointer still produces a non-nil interface type word and + // carries no frame-owned storage in its data word. + return "" + } + root, reason := audit.stableAddress(boxed.X, make(map[ssa.Value]bool)) + if reason != "" || root != coroPhysicalAddressGlobal { + if reason == "" { + reason = "payload is not rooted in package-global storage" + } + return "explicit-status panic data word may outlive its coroutine frame: " + reason + } + return "" +} + func isCoroProgramManagedEntry(fn *ssa.Function) bool { if fn == nil { return false diff --git a/cl/coro_entry.go b/cl/coro_entry.go index 719c6842ca..95bba34a4f 100644 --- a/cl/coro_entry.go +++ b/cl/coro_entry.go @@ -43,6 +43,7 @@ type plannedFunctionSymbol struct { programRun bool plainDispatch bool staticSpawn bool + explicitPanic bool coroPlan *coro.SSAPlan emission *EmissionUniverse } @@ -91,6 +92,7 @@ func (p *context) resolveFunctionSymbol(fn *ssa.Function) (plannedFunctionSymbol entry.programRun = p.compilation.EnableCoroProgramBootstrapRun entry.plainDispatch = p.compilation.EnableCoroPlainDispatch entry.staticSpawn = p.compilation.EnableCoroClosedStaticSpawn + entry.explicitPanic = p.compilation.EnableCoroExplicitStatusPanicABI entry.coroPlan = p.compilation.CoroPlan entry.emission = p.compilation.EmissionUniverse if p.compilation.CoroPlan.IgnoresBody(fn) { @@ -170,6 +172,9 @@ func (e plannedFunctionSymbol) checkSupported() error { if e.plan.Emission == coro.EmitNone { return fmt.Errorf("coroutine entry resolution: function %q has no emitted entry", e.plan.ID) } + if e.explicitPanic && e.plan.Emission == coro.EmitPlain { + return fmt.Errorf("coroutine explicit-status panic ABI: managed plain function %q has no certified hidden-outcome/unwind contract", e.plan.ID) + } if e.plan.FuncRep == coro.Dispatch { if !e.plainDispatch { return fmt.Errorf("coroutine entry resolution: function %q requires an unimplemented dispatch descriptor", e.plan.ID) @@ -183,7 +188,7 @@ func (e plannedFunctionSymbol) checkSupported() error { if err := validateCoroPhysicalFunctionValueABI(e.plan, e.function.Signature, e.plainDispatch); err != nil { return err } - return validateCoroPhysicalABIWithUniverseCapabilities(e.function, e.plan, e.coroPlan, e.emission, e.childAwait, e.programRun, e.staticSpawn) + return validateCoroPhysicalABIWithUniverseCapabilities(e.function, e.plan, e.coroPlan, e.emission, e.childAwait, e.programRun, e.staticSpawn, e.explicitPanic) } if e.plan.Emission == coro.EmitExternal && e.plan.FuncRep == coro.DirectCoro { return fmt.Errorf("external coroutine emission %q requires coroutine physical ABI lowering", e.plan.ID) @@ -211,6 +216,9 @@ func (c *Compilation) preflightCoroPlan() error { if c.EnableCoroExplicitStatusPanicABI && !c.EnableCoroEntryResolution { return fmt.Errorf("coroutine explicit-status panic ABI requires coroutine entry resolution") } + if c.EnableCoroExplicitStatusPanicABI && !c.EnableCoroChildAwait { + return fmt.Errorf("coroutine explicit-status panic ABI requires PhysicalABIV1 child-await lowering") + } if c.EnableCoroClosedStaticSpawn { if !c.EnableCoroChildAwait { return fmt.Errorf("coroutine closed static spawn requires coroutine child await") @@ -227,10 +235,6 @@ func (c *Compilation) preflightCoroPlan() error { c.coroPreflightErr = err return } - if c.EnableCoroExplicitStatusPanicABI { - c.coroPreflightErr = fmt.Errorf("coroutine explicit-status panic ABI %q is identity-only: lowering and runtime semantics are not implemented", coro.PanicExplicitStatusABIV0) - return - } if c.CoroPlan == nil { c.coroPreflightErr = fmt.Errorf("coroutine entry resolution requires a compilation CoroPlan") return @@ -271,6 +275,7 @@ func (c *Compilation) preflightCoroPlan() error { programRun: c.EnableCoroProgramBootstrapRun, plainDispatch: c.EnableCoroPlainDispatch, staticSpawn: c.EnableCoroClosedStaticSpawn, + explicitPanic: c.EnableCoroExplicitStatusPanicABI, coroPlan: c.CoroPlan, emission: c.EmissionUniverse, } diff --git a/cl/coro_panic.go b/cl/coro_panic.go new file mode 100644 index 0000000000..10d3d7f854 --- /dev/null +++ b/cl/coro_panic.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "fmt" + + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +// tryCompileCoroExplicitStatusPanic owns the terminal source instruction when +// the compilation-wide ExplicitStatus identity is active. Preflight has +// already proved that X is one pure, concrete empty-interface construction; +// reaching this path with any other shape is a compiler-plan violation, never +// permission to fall back to the legacy runtime.Panic call. +func (p *context) tryCompileCoroExplicitStatusPanic(b llssa.Builder, instruction *ssa.Panic) bool { + if p.compilation == nil || !p.compilation.EnableCoroExplicitStatusPanicABI { + return false + } + if instruction == nil || p.currentCoro == nil || b.Func != p.fn { + panic(fmt.Errorf("explicit-status panic escaped its exact physical coroutine body")) + } + if _, ok := instruction.X.(*ssa.MakeInterface); !ok { + panic(fmt.Errorf("explicit-status panic operand escaped its concrete MakeInterface preflight")) + } + value := p.compileValue(b, instruction.X) + typeWord := b.EfaceType(value) + dataWord := b.InterfaceData(value) + p.currentCoro.panic(b, typeWord, dataWord) + return true +} diff --git a/cl/coro_panic_test.go b/cl/coro_panic_test.go new file mode 100644 index 0000000000..16416d0387 --- /dev/null +++ b/cl/coro_panic_test.go @@ -0,0 +1,373 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "bytes" + "regexp" + "strings" + "testing" + + "github.com/goplus/llgo/internal/coro" + "github.com/goplus/llgo/internal/goembed" + llssa "github.com/goplus/llgo/ssa" + "github.com/xgo-dev/llvm" + "golang.org/x/tools/go/ssa" +) + +const coroExplicitStatusPanicFixture = `package foo + +var FirstPayload uint32 +var SecondPayload uint32 + +func Root(mode uint32) uint32 { + if mode == 0 { + return 11 + } + if mode == 1 { + panic(&FirstPayload) + } + if mode == 2 { + return 13 + } + panic(&SecondPayload) +} +` + +func TestCoroExplicitStatusPanicNativeAndWasm32(t *testing.T) { + llssa.Initialize(llssa.InitAll) + for _, test := range []struct { + name string + target *llssa.Target + }{ + {name: "native"}, + {name: "wasm32", target: &llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}}, + } { + t.Run(test.name, func(t *testing.T) { + prog, pkg, plan, root := compileCoroExplicitStatusPanicFixture(t, test.target) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + rootPlan, ok := plan.FunctionPlan(root) + if !ok || rootPlan.Emission != coro.EmitCoroutine || rootPlan.FuncRep != coro.DirectCoro || + rootPlan.Demand != coro.AsyncDemand || !rootPlan.Exec.Contains(coro.MayUnwind) { + t.Fatalf("Root plan = %+v, present=%t; want may-unwind direct coroutine", rootPlan, ok) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify explicit-status panic before CoroSplit: %v\n%s", err, module.String()) + } + body := requireCoroPhysicalFunction(t, module, "foo.Root").String() + assertCoroExplicitStatusPanicBody(t, body, 2) + assertNoLegacyCoroPanicSymbol(t, module.String()) + + runCoroABITestPipeline(t, prog, module) + resume := module.NamedFunction("foo.Root$coro.resume") + if resume.IsNil() { + t.Fatalf("CoroSplit did not create Root resume entry:\n%s", module.String()) + } + if got := strings.Count(resume.String(), "call void @"+coroPanicPrepareHookV1); got != 2 { + t.Fatalf("Root.resume panic prepare calls = %d, want 2:\n%s", got, resume.String()) + } + assertNoLegacyCoroPanicSymbol(t, module.String()) + for _, intrinsic := range []string{"llvm.coro.id", "llvm.coro.begin", "llvm.coro.suspend", "llvm.coro.end"} { + if hasLLVMCall(module.String(), intrinsic) { + t.Fatalf("post-split panic module still calls %s:\n%s", intrinsic, module.String()) + } + } + object, err := prog.TargetMachine().EmitToMemoryBuffer(module, llvm.ObjectFile) + if err != nil { + t.Fatalf("emit post-CoroSplit panic object: %v\n%s", err, module.String()) + } + defer object.Dispose() + if len(object.Bytes()) == 0 || !bytes.Contains(object.Bytes(), []byte(coroPanicPrepareHookV1)) || + !bytes.Contains(object.Bytes(), []byte("foo.Root$coro")) { + t.Fatal("post-CoroSplit object lost the panic hook or physical coroutine symbol") + } + }) + } +} + +func assertCoroExplicitStatusPanicBody(t *testing.T, body string, panicSites int) { + t.Helper() + if got := strings.Count(body, "call void @"+coroPanicPrepareHookV1); got != panicSites { + t.Fatalf("panic prepare calls = %d, want %d:\n%s", got, panicSites, body) + } + if got := strings.Count(body, "call void @"+coroCompletePrepareHookV1); got != 1 { + t.Fatalf("completion prepare calls = %d, want one shared normal completion:\n%s", got, body) + } + if got := strings.Count(body, "call i8 @llvm.coro.suspend"); got != 2 { + t.Fatalf("coro.suspend calls = %d, want initial + one shared final:\n%s", got, body) + } + if got := strings.Count(body, "@llvm.coro.suspend(token none, i1 true)"); got != 1 { + t.Fatalf("final coro.suspend calls = %d, want exactly one shared final suspend:\n%s", got, body) + } + stateAndHook := regexp.MustCompile( + `(?s)store i16 5,.*?store i16 4,.*?store i32 [1-9][0-9]*,.*?call void @` + regexp.QuoteMeta(coroPanicPrepareHookV1) + + `\(ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^)]+\)`, + ) + if got := len(stateAndHook.FindAllStringIndex(body, -1)); got != panicSites { + t.Fatalf("Panic/FinalSuspended/stateID publication followed by the five-pointer hook = %d, want %d:\n%s", got, panicSites, body) + } + hookBranch := regexp.MustCompile( + `call void @`+regexp.QuoteMeta(coroPanicPrepareHookV1)+`\([^\n]+\)\n\s+br label (%[-a-zA-Z$._0-9]+)`, + ).FindAllStringSubmatch(body, -1) + if len(hookBranch) != panicSites { + t.Fatalf("panic hooks followed immediately by an ordinary branch = %d, want %d (no source panic/unreachable path):\n%s", len(hookBranch), panicSites, body) + } + completeBranch := regexp.MustCompile( + `call void @` + regexp.QuoteMeta(coroCompletePrepareHookV1) + `\([^\n]+\)\n\s+br label (%[-a-zA-Z$._0-9]+)`, + ).FindStringSubmatch(body) + if len(completeBranch) != 2 { + t.Fatalf("normal completion does not branch to the shared terminal block:\n%s", body) + } + for _, branch := range hookBranch { + if branch[1] != completeBranch[1] { + t.Fatalf("panic branch target %s differs from normal completion target %s:\n%s", branch[1], completeBranch[1], body) + } + } + finalSuspend := strings.Index(body, "@llvm.coro.suspend(token none, i1 true)") + if finalSuspend < 0 { + t.Fatalf("shared final suspend is absent:\n%s", body) + } + for offset := 0; ; { + relative := strings.Index(body[offset:], "call void @"+coroPanicPrepareHookV1) + if relative < 0 { + break + } + hook := offset + relative + if hook >= finalSuspend { + t.Fatalf("panic hook does not precede the shared final suspend:\n%s", body) + } + offset = hook + len(coroPanicPrepareHookV1) + } +} + +func assertNoLegacyCoroPanicSymbol(t *testing.T, ir string) { + t.Helper() + for _, forbidden := range []string{"runtime.Panic", "runtime.Rethrow"} { + if strings.Contains(ir, forbidden) { + t.Fatalf("explicit-status coroutine retained legacy panic symbol %q:\n%s", forbidden, ir) + } + } +} + +func compileCoroExplicitStatusPanicFixture(t *testing.T, target *llssa.Target) ( + llssa.Program, llssa.Package, *coro.SSAPlan, *ssa.Function, +) { + t.Helper() + ssaPkg, _, files := buildGoSSAPkg(t, coroExplicitStatusPanicFixture) + var prog llssa.Program + if target == nil { + prog = newLLSSAProg(t) + } else { + prog = newLLSSAProgForTarget(t, target) + } + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + root := ssaPkg.Func("Root") + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerChildAwaitABIV0 + functionIDs.ArchiveReady = true + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: root, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == root { + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + }) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroChildAwaitCompilation(compilation) + compilation.EnableCoroExplicitStatusPanicABI = true + compilation.PanicABI = coro.PanicExplicitStatusABIV0 + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + prog.Dispose() + t.Fatal(err) + } + return prog, pkg, plan, root +} + +func TestCoroExplicitStatusPanicPreflightRemainsFailClosed(t *testing.T) { + for _, test := range []struct { + name string + source string + want string + exec coro.ExecFlags + }{ + { + name: "dynamic interface operand", + source: `package foo +func Root(value any, trigger bool) { if trigger { panic(value) } } +`, + want: "concrete MakeInterface operand", + }, + { + name: "untyped nil", + source: `package foo +func Root(trigger bool) { if trigger { panic(nil) } } +`, + want: "explicit-status panic", + }, + { + name: "boxed scalar", + source: `package foo +func Root(trigger bool) { if trigger { panic(uint32(7)) } } +`, + want: "managed backing allocation", + }, + { + name: "frame local pointer", + source: `package foo +func Root(trigger bool) { value := uint32(7); if trigger { panic(&value) }; _ = value } +`, + want: "heap allocation requires managed allocation", + }, + { + name: "parameter pointer", + source: `package foo +func Root(value *uint32, trigger bool) { if trigger { panic(value) } } +`, + want: "may outlive its coroutine frame", + }, + { + name: "implicit fault", + source: `package foo +var Payload uint32 +func Root(values []uint32, index int, trigger bool) uint32 { + value := values[index] + if trigger { panic(&Payload) } + return value +} +`, + want: "index base is not a fixed-array pointer", + }, + { + name: "cleanup frame", + source: `package foo +var Payload uint32 +func cleanup() {} +func Root(trigger bool) { defer cleanup(); if trigger { panic(&Payload) } } +`, + want: "execution flags", + exec: coro.NeedsCleanupFrame, + }, + } { + t.Run(test.name, func(t *testing.T) { + ssaPkg, _, files := buildGoSSAPkg(t, test.source) + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + t.Fatal(err) + } + root := ssaPkg.Func("Root") + plan := coro.FunctionPlan{ + ID: coro.FunctionID("foo.Root"), + External: coro.Defined, + Demand: coro.AsyncDemand, + Emission: coro.EmitCoroutine, + Primary: coro.PrimaryCoroutine, + FuncRep: coro.DirectCoro, + Effect: coro.YieldOnly, + Exec: coro.MayUnwind | test.exec, + } + err = validateCoroPhysicalABIWithUniverseCapabilities(root, plan, nil, universe, true, false, false, true) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("preflight error = %v, want %q", err, test.want) + } + }) + } +} + +func TestCoroExplicitStatusPanicRejectsManagedPlainBody(t *testing.T) { + const source = `package foo +var Payload uint32 +func Plain(value uint32) uint32 { return value + 1 } +func Root(value uint32, trigger bool) uint32 { + result := Plain(value) + if trigger { panic(&Payload) } + return result +} +` + ssaPkg, _, files := buildGoSSAPkg(t, source) + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, err := PrepareEmissionUniverse(prog, nil, []EmissionPackage{{SSA: ssaPkg, Files: files}}) + if err != nil { + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + t.Fatal(err) + } + root := ssaPkg.Func("Root") + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerChildAwaitABIV0 + functionIDs.ArchiveReady = true + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{{Function: root, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + if fn == root { + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + } + return coro.SSAFunctionPolicy{}, nil + }, + }) + if err != nil { + t.Fatal(err) + } + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroChildAwaitCompilation(compilation) + compilation.EnableCoroExplicitStatusPanicABI = true + compilation.PanicABI = coro.PanicExplicitStatusABIV0 + got, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err == nil || !strings.Contains(err.Error(), "managed plain function") || !strings.Contains(err.Error(), "hidden-outcome/unwind contract") { + t.Fatalf("plain-body preflight result = %v, %v; want exact hidden-outcome rejection", got, err) + } + if got != nil { + t.Fatal("plain-body preflight failure returned a partial package") + } +} diff --git a/ssa/interface.go b/ssa/interface.go index 2539c3187e..fc07f71761 100644 --- a/ssa/interface.go +++ b/ssa/interface.go @@ -372,6 +372,18 @@ func (b Builder) InterfaceData(x Expr) Expr { return Expr{b.faceData(x.impl), b.Prog.VoidPtr()} } +// EfaceType returns the dynamic ABI type descriptor stored directly in an +// empty-interface value. It deliberately rejects non-empty interfaces: their +// first word is an itab rather than an ABI type descriptor. +func (b Builder) EfaceType(x Expr) Expr { + raw, ok := types.Unalias(x.raw.Type).Underlying().(*types.Interface) + if !ok || !raw.Empty() { + panic("EfaceType requires an empty-interface value") + } + dbgInstrf("EfaceType %v\n", x.impl) + return Expr{llvm.CreateExtractValue(b.impl, x.impl, 0), b.Prog.AbiTypePtr()} +} + func (b Builder) faceData(x llvm.Value) llvm.Value { return llvm.CreateExtractValue(b.impl, x, 1) } From b6e3641e1af0f6169aeb0b995a61678558882e73 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Fri, 17 Jul 2026 00:31:04 +0800 Subject: [PATCH 30/32] build(coro): retain explicit panic prepare hook --- internal/build/build.go | 7 +++++ internal/build/coro_plan_test.go | 50 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/internal/build/build.go b/internal/build/build.go index 9a539838ba..005c1af1ec 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1916,6 +1916,13 @@ func requiredCoroProgramRuntimePlan(ctx *context) (coro.Roots, map[*ssa.Function "__llgo_coro_frame_free_v1", ) } + if ctx.buildConf.EnableCoroExplicitStatusPanicABI { + // Physical coroutine bodies reference this hook from compiler-generated + // IR, so the source SSA graph has no edge that could retain it. Keep the + // exact runtime body as a synchronous direct-plain root only while the + // target-wide ExplicitStatus panic identity is selected. + names = append(names, "__llgo_coro_panic_prepare_v1") + } if ctx.buildConf.EnableCoroClosedStaticSpawn { names = append(names, "__llgo_coro_spawn_begin_v1", diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index bc3a13f27c..09865c2743 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -417,6 +417,7 @@ func __llgo_coro_yield_prepare_v1() {} func __llgo_coro_park_prepare_v1() {} func __llgo_coro_complete_prepare_v1() {} func __llgo_coro_frame_free_v1() {} +func __llgo_coro_panic_prepare_v1() {} func __llgo_coro_spawn_begin_v1() {} func __llgo_coro_spawn_commit_v1() {} func __llgo_coro_program_main_return_v1() {} @@ -488,6 +489,37 @@ func atomicExchange(*uint32, uint32) uint32 t.Fatalf("required root %d = %+v, want %s/%s", index, root, wantRoots[index], wantDemand) } } + panicHook := ssaPkg.Func("__llgo_coro_panic_prepare_v1") + if panicHook == nil { + t.Fatal("explicit-status panic prepare hook is absent from the runtime fixture") + } + if _, ok := requiredPlain[panicHook]; ok { + t.Fatal("inactive explicit-status panic prepare hook entered the required plain island") + } + panicCtx := &context{ + buildConf: &Config{ + EnableCoroChildAwait: true, + EnableCoroProgramBootstrapRun: true, + EnableCoroExplicitStatusPanicABI: true, + }, + coroEmission: ctx.coroEmission, + coroSSAEmission: ctx.coroSSAEmission, + } + panicRoots, panicPlain, panicDirect, panicClosed, err := requiredCoroProgramRuntimePlan(panicCtx) + if err != nil { + t.Fatal(err) + } + if len(panicRoots) != len(wantRoots)+1 || + panicRoots[len(panicRoots)-1].Function != panicHook || + panicRoots[len(panicRoots)-1].Demand != coro.SyncDemand { + t.Fatalf("explicit-status runtime roots = %+v, want legacy roots plus exact panic prepare/sync", panicRoots) + } + if _, ok := panicPlain[panicHook]; !ok { + t.Fatal("active explicit-status panic prepare hook is absent from the required plain island") + } + if len(panicDirect) != 0 || len(panicClosed) != 0 { + t.Fatalf("explicit-status panic hook produced callback proofs: direct=%d dynamic=%d", len(panicDirect), len(panicClosed)) + } spawnCtx := &context{ buildConf: &Config{ EnableCoroChildAwait: true, @@ -577,6 +609,24 @@ func atomicExchange(*uint32, uint32) uint32 if err != nil { t.Fatal(err) } + panicInput := input + panicInput.requiredRoots = panicRoots + panicInput.requiredPlain = panicPlain + panicInput.requiredDirectPlain = panicDirect + panicInput.requiredClosedDynamic = panicClosed + panicPlan, err := panicInput.Analyze(coro.Roots{{Function: unrelatedLoop, Demand: coro.AsyncDemand}}, coro.SSAConfig{ + MaxPlainInstructions: -1, + FunctionIDs: functionIDs, + }) + if err != nil { + t.Fatal(err) + } + panicHookPlan, ok := panicPlan.FunctionPlan(panicHook) + if !ok || panicHookPlan.Emission != coro.EmitPlain || panicHookPlan.Demand != coro.SyncDemand || + panicHookPlan.FuncRep != coro.DirectPlain || panicHookPlan.Effect.MaySuspend() || + panicHookPlan.Exec.Contains(coro.NeedsPreempt) { + t.Fatalf("explicit-status panic prepare hook plan = %+v, want required sync direct-plain", panicHookPlan) + } closurePlan, ok := plan.FunctionPlan(closureLoop) if !ok || closurePlan.Exec.Contains(coro.NeedsPreempt) || closurePlan.Effect.MaySuspend() || closurePlan.Emission != coro.EmitPlain { t.Fatalf("required closure loop plan = %+v, want one trusted plain body", closurePlan) From 21797b5126899a7b132c7f10535267beb324a1fd Mon Sep 17 00:00:00 2001 From: Li Jie Date: Fri, 17 Jul 2026 00:31:09 +0800 Subject: [PATCH 31/32] test(coro): run terminal panic scheduler island --- internal/build/coro_panic_native_e2e_test.go | 515 +++++++++++++++++++ 1 file changed, 515 insertions(+) create mode 100644 internal/build/coro_panic_native_e2e_test.go diff --git a/internal/build/coro_panic_native_e2e_test.go b/internal/build/coro_panic_native_e2e_test.go new file mode 100644 index 0000000000..988901626f --- /dev/null +++ b/internal/build/coro_panic_native_e2e_test.go @@ -0,0 +1,515 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + stdcontext "context" + goimporter "go/importer" + "go/token" + "go/types" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + "time" + + "github.com/goplus/llgo/cl" + "github.com/goplus/llgo/internal/coro" + "github.com/goplus/llgo/internal/goembed" + "github.com/goplus/llgo/internal/packages" + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +const ( + coroPanicNativeE2EPackage = "example.com/llgo-coro-panic-e2e" + coroPanicNativeE2EEntry = "__llgo_coro_panic_e2e_entry" + coroPanicNativeE2ERunReport = "__llgo_coro_program_run_report_e2e_v1" + coroPanicNativeE2EDestroyObserve = "__llgo_coro_destroy_observe_e2e_v1" + coroPanicNativeE2EDestroyCount = "__llgo_coro_panic_e2e_destroy_count" + coroPanicNativeE2EFirstDestroy = "__llgo_coro_panic_e2e_first_destroy" + coroPanicNativeE2ESecondDestroy = "__llgo_coro_panic_e2e_second_destroy" + coroPanicNativeE2EThirdDestroy = "__llgo_coro_panic_e2e_third_destroy" + coroPanicNativeE2EExplicitStatus = uint64(1) + coroPanicNativeE2EExpectedDestroys = uint64(3) +) + +const coroPanicNativeE2ESource = `package main + +var Before uint32 +var After uint32 +var GlobalPayload byte + +func panicChild(doPanic bool) { + Before = 1 + if doPanic { + panic(&GlobalPayload) + } +} + +func main() { + panicChild(true) + After = 1 +} +` + +// TestCoroExplicitPanicNativeNoStdlibRuntimeE2E is a deliberately closed +// scheduler island. It compiles a real source panic in a physical child frame, +// links the production native-nogc scheduler/core and panic prepare hook, and +// runs without the legacy panic printer/runtime closure. +// +// Production ActionPanicComplete is fail-closed today: coroProgramRunV1 +// returns false and the exported program-run ABI aborts. The entry module is +// therefore retargeted to a test-only report ABI. That ABI still calls the +// production internal runner and accepts only the terminal-panic shape: a +// published record on a dead, non-reclaimable G, the original package-global +// payload word, and exactly one destroy of each distinct handle in the +// child -> main -> bootstrap chain. It does not turn panic into production +// success or provide a replacement printer. +func TestCoroExplicitPanicNativeNoStdlibRuntimeE2E(t *testing.T) { + if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + t.Skip("native coroutine link smoke requires Darwin or Linux") + } + clang, err := exec.LookPath("clang") + if err != nil { + t.Skip("clang is unavailable") + } + ar, err := exec.LookPath("llvm-ar") + if err != nil { + ar, err = exec.LookPath("ar") + if err != nil { + t.Skip("llvm-ar/ar is unavailable") + } + } + + llssa.Initialize(llssa.InitAll) + temp := t.TempDir() + prog := llssa.NewProgram(nil) + prog.SetRuntime(func() *types.Package { + rt, err := goimporter.For("source", nil).Import(llssa.PkgRuntime) + if err != nil { + t.Fatal("load runtime type model:", err) + } + return rt + }) + prog.TypeSizes(types.SizesFor("gc", runtime.GOARCH)) + defer prog.Dispose() + + userObject, anchor := buildCoroPanicNativeE2EUser(t, prog, temp) + entryObject := buildCoroPanicNativeE2EEntry(t, prog, temp, anchor) + driverObject := buildCoroPanicNativeE2EDriver(t, prog, temp) + runtimeObjects := buildCoroSpawnNativeE2ERuntimeIsland(t, temp) + runtimeArchive := filepath.Join(temp, "libllgo-coro-panic-runtime-island.a") + arArgs := append([]string{"rcs", runtimeArchive}, runtimeObjects...) + if output, err := exec.Command(ar, arArgs...).CombinedOutput(); err != nil { + t.Fatalf("archive coroutine panic runtime island: %v\n%s", err, output) + } + + executable := filepath.Join(temp, "coro-panic-e2e") + linkArgs := []string{driverObject, entryObject, userObject, runtimeArchive, "-o", executable} + if runtime.GOOS == "darwin" { + linkArgs = append(linkArgs, "-Wl,-dead_strip") + } else { + linkArgs = append(linkArgs, "-Wl,--gc-sections") + } + if output, err := exec.Command(clang, linkArgs...).CombinedOutput(); err != nil { + t.Fatalf("link native coroutine explicit-panic smoke: %v\n%s", err, output) + } + assertCoroPanicNativeE2ELinkedSymbols(t, executable) + + runCtx, cancel := stdcontext.WithTimeout(stdcontext.Background(), 10*time.Second) + defer cancel() + output, err := exec.CommandContext(runCtx, executable).CombinedOutput() + if runCtx.Err() != nil { + t.Fatalf("native coroutine explicit-panic smoke timed out: %v\n%s", runCtx.Err(), output) + } + if err != nil { + t.Fatalf("native coroutine explicit-panic smoke failed: %v\n%s", err, output) + } +} + +func buildCoroPanicNativeE2EUser(t *testing.T, prog llssa.Program, temp string) (object, anchor string) { + t.Helper() + ssaPkg, files := buildCoroPlanTestPackage(t, coroPanicNativeE2EPackage, coroPanicNativeE2ESource, nil) + universe, err := cl.PrepareEmissionUniverse(prog, nil, []cl.EmissionPackage{{ + SSA: ssaPkg, Files: files, Identity: coroPanicNativeE2EPackage, + }}) + if err != nil { + t.Fatal(err) + } + ssaUniverse, err := coro.NewSSAEmissionUniverse(ssaPkg.Prog, universe.Functions()) + if err != nil { + t.Fatal(err) + } + mainFn, childFn := ssaPkg.Func("main"), ssaPkg.Func("panicChild") + functionIDs := universe.FunctionIDConfig() + functionIDs.CoroABI = coro.PhysicalABIV1 + functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapABIV2 + functionIDs.ArchiveReady = true + plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{ + {Function: mainFn, Demand: coro.AsyncDemand}, + }, coro.SSAConfig{ + EmissionUniverse: ssaUniverse, + FunctionIDs: functionIDs, + MaxPlainInstructions: -1, + ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { + switch fn { + case mainFn, childFn: + return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil + default: + return coro.SSAFunctionPolicy{}, nil + } + }, + }) + if err != nil { + t.Fatal(err) + } + compilation := &cl.Compilation{ + CoroPlan: plan, + EnableCoroEntryResolution: true, + EnableCoroPhysicalABI: true, + EnableCoroChildAwait: true, + EnableCoroExplicitStatusPanicABI: true, + EnableCoroProgramBootstrapRun: true, + CoroABI: coro.PhysicalABIV1, + SchedulerABI: coro.SchedulerProgramBootstrapABIV2, + PanicABI: coro.PanicExplicitStatusABIV0, + FuncRepABI: coro.FuncRepABIV0, + EmissionUniverse: universe, + } + pkg, _, err := cl.NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + cl.PackageOptions{Compilation: compilation}, + ) + if err != nil { + t.Fatal(err) + } + module := pkg.Module() + presplit := module.String() + if strings.Count(presplit, "@__llgo_coro_panic_prepare_v1") != 2 || + !strings.Contains(presplit, "call void @__llgo_coro_panic_prepare_v1") { + t.Fatalf("compiled explicit panic has no unique prepare-hook call:\n%s", presplit) + } + if strings.Contains(presplit, llssa.PkgRuntime+".Panic") { + t.Fatalf("compiled explicit panic retained the legacy runtime.Panic edge:\n%s", presplit) + } + runCoroSpawnNativeE2EPasses(t, prog, module) + ir := module.String() + match := regexp.MustCompile(`@"?(__llgo_coro_root_package_v1\.[0-9a-f]{32})"?\s*=`).FindStringSubmatch(ir) + if len(match) != 2 { + t.Fatalf("compiled panic E2E user module has no root package anchor:\n%s", ir) + } + return emitCoroSpawnNativeE2EObject(t, prog, module, filepath.Join(temp, "panic-user.o")), match[1] +} + +func buildCoroPanicNativeE2EEntry(t *testing.T, prog llssa.Program, temp, anchor string) string { + t.Helper() + conf := &Config{ + BuildMode: BuildModeExe, + Goos: runtime.GOOS, + Goarch: runtime.GOARCH, + EnableCoroEntryResolution: true, + EnableCoroPhysicalABI: true, + EnableCoroChildAwait: true, + EnableCoroProgramBootstrapABI: true, + EnableCoroProgramBootstrapRun: true, + } + ctx := &context{prog: prog, buildConf: conf} + bootstrap := &coroProgramBootstrapV1{ + Version: coroProgramBootstrapVersionV2, + Steps: []coroProgramBootstrapStepV1{ + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleRuntimeInitV2, FunctionID: "panic-e2e-runtime-init", Target: "__llgo_coro_panic_e2e_runtime_init"}, + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleABIInitV2, FunctionID: "panic-e2e-abi-init", Target: "init$abitypes"}, + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRolePublicRuntimeInitV2, FunctionID: coroProgramPublicRuntimeNoopIDV2, Target: coroProgramPublicRuntimeNoopSymbolV2}, + {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRolePackageInitV2, FunctionID: "panic-e2e-package-init", Target: "__llgo_coro_panic_e2e_package_init"}, + { + Kind: coroProgramStepCoroRootV1, Role: coroProgramStepRoleMainV2, + FunctionID: "panic-e2e-main", Target: coroPanicNativeE2EPackage + ".main$coro", + Owner: coroPanicNativeE2EPackage, CatalogTarget: anchor, Aux: 0, + }, + }, + } + var programHash [16]byte + for i := range programHash { + programHash[i] = byte(0x40 + i) + } + entry := genMainModule(ctx, llssa.PkgRuntime, &packages.Package{ + ID: coroPanicNativeE2EPackage, PkgPath: coroPanicNativeE2EPackage, ExportFile: "coro-panic-e2e.a", + }, &genConfig{ + coroRootAnchors: []string{anchor}, + coroManifestHash: programHash, + coroBootstrap: bootstrap, + }) + for _, name := range []string{"__llgo_coro_panic_e2e_runtime_init", "__llgo_coro_panic_e2e_package_init"} { + fn := entry.LPkg.FuncOf(name) + if fn == nil { + t.Fatalf("entry module has no bounded panic-E2E init declaration %q", name) + } + if !fn.HasBody() { + body := fn.MakeBody(1) + body.Return() + } + } + module := entry.LPkg.Module() + entryMain := module.NamedFunction("main") + if entryMain.IsNil() { + t.Fatalf("entry module has no native main:\n%s", entry.LPkg.String()) + } + entryMain.SetName(coroPanicNativeE2EEntry) + run := module.NamedFunction(coroProgramRunSymbolV1) + if run.IsNil() || !run.IsDeclaration() { + t.Fatalf("entry module has no program-run declaration %q:\n%s", coroProgramRunSymbolV1, entry.LPkg.String()) + } + run.SetName(coroPanicNativeE2ERunReport) + + destroy := entry.LPkg.FuncOf("__llgo_coro_destroy_v1") + if destroy == nil || !destroy.HasBody() { + t.Fatalf("entry module has no coroutine destroy wrapper:\n%s", entry.LPkg.String()) + } + observe := entry.LPkg.NewFunc(coroPanicNativeE2EDestroyObserve, newSignature( + []types.Type{types.Typ[types.UnsafePointer]}, nil, + ), llssa.InC) + instrument := destroy.NewBuilder() + instrument.SetBlockEx(destroy.Block(0), llssa.AtStart, true) + instrument.Call(observe.Expr, destroy.Param(0)) + instrument.Dispose() + + if err := lowerCoroControlWrappers(ctx, entry.LPkg); err != nil { + t.Fatal(err) + } + return emitCoroSpawnNativeE2EObject(t, prog, module, filepath.Join(temp, "panic-entry.o")) +} + +func buildCoroPanicNativeE2EDriver(t *testing.T, prog llssa.Program, temp string) string { + t.Helper() + pkg := prog.NewPackage("coro-panic-e2e-driver", "coro-panic-e2e-driver") + defer pkg.Module().Dispose() + pointer := types.Typ[types.UnsafePointer] + uint32Type := types.Typ[types.Uint32] + + abort := pkg.NewFunc("abort", newSignature(nil, nil), llssa.InC) + exit := pkg.NewFunc("exit", newSignature([]types.Type{types.Typ[types.Int32]}, nil), llssa.InC) + require := pkg.NewFunc("__llgo_coro_panic_e2e_require", newSignature( + []types.Type{types.Typ[types.Bool], types.Typ[types.Int32]}, nil, + ), llssa.InC) + requireBody := require.MakeBody(3) + requireFail, requireValid := require.Block(1), require.Block(2) + requireBody.If(require.Param(0), requireValid, requireFail) + requireBody.SetBlock(requireFail).Call(exit.Expr, require.Param(1)) + requireBody.Return() + requireBody.SetBlock(requireValid).Return() + + destroyCount := pkg.NewVar(coroPanicNativeE2EDestroyCount, types.NewPointer(uint32Type), llssa.InC) + destroyCount.InitNil() + firstDestroy := pkg.NewVar(coroPanicNativeE2EFirstDestroy, types.NewPointer(pointer), llssa.InC) + firstDestroy.InitNil() + secondDestroy := pkg.NewVar(coroPanicNativeE2ESecondDestroy, types.NewPointer(pointer), llssa.InC) + secondDestroy.InitNil() + thirdDestroy := pkg.NewVar(coroPanicNativeE2EThirdDestroy, types.NewPointer(pointer), llssa.InC) + thirdDestroy.InitNil() + observe := pkg.NewFunc(coroPanicNativeE2EDestroyObserve, newSignature([]types.Type{pointer}, nil), llssa.InC) + observeBody := observe.MakeBody(5) + firstBlock, laterBlock := observe.Block(1), observe.Block(2) + secondBlock, thirdBlock := observe.Block(3), observe.Block(4) + count := observeBody.Load(destroyCount.Expr) + zero32 := prog.IntVal(0, prog.Uint32()) + one32 := prog.IntVal(1, prog.Uint32()) + observeBody.If(observeBody.BinOp(token.EQL, count, zero32), firstBlock, laterBlock) + firstBody := observeBody.SetBlock(firstBlock) + firstBody.Store(firstDestroy.Expr, observe.Param(0)) + firstBody.Store(destroyCount.Expr, firstBody.BinOp(token.ADD, count, one32)) + firstBody.Return() + laterBody := observeBody.SetBlock(laterBlock) + laterBody.If(laterBody.BinOp(token.EQL, count, one32), secondBlock, thirdBlock) + secondBody := observeBody.SetBlock(secondBlock) + secondBody.Store(secondDestroy.Expr, observe.Param(0)) + secondBody.Store(destroyCount.Expr, secondBody.BinOp(token.ADD, count, one32)) + secondBody.Return() + thirdBody := observeBody.SetBlock(thirdBlock) + thirdBody.Store(thirdDestroy.Expr, observe.Param(0)) + thirdBody.Store(destroyCount.Expr, thirdBody.BinOp(token.ADD, count, one32)) + thirdBody.Return() + + // The production adapter island is compiled from an explicit runtime file + // list, so its private Go symbols belong to command-line-arguments while its + // exported C ABI remains stable. + runtimeRun := pkg.NewFunc("command-line-arguments.coroProgramRunV1", newSignature( + []types.Type{pointer, pointer}, []types.Type{types.Typ[types.Bool]}, + ), llssa.InGo) + panicRecordType := types.NewStruct([]*types.Var{ + types.NewField(token.NoPos, nil, "Status", uint32Type, false), + types.NewField(token.NoPos, nil, "TypeWord", pointer, false), + types.NewField(token.NoPos, nil, "DataWord", pointer, false), + }, nil) + loadPanicRecord := pkg.NewFunc("github.com/goplus/llgo/runtime/internal/coro.LoadPanicRecord", newSignature( + []types.Type{pointer}, []types.Type{panicRecordType, types.Typ[types.Bool]}, + ), llssa.InGo) + deadG := pkg.NewFunc("github.com/goplus/llgo/runtime/internal/coro.DeadG", newSignature( + []types.Type{pointer}, []types.Type{types.Typ[types.Bool]}, + ), llssa.InGo) + reclaimableG := pkg.NewFunc("github.com/goplus/llgo/runtime/internal/coro.ReclaimableG", newSignature( + []types.Type{pointer}, []types.Type{types.Typ[types.Bool]}, + ), llssa.InGo) + payload := pkg.NewVar(coroPanicNativeE2EPackage+".GlobalPayload", types.NewPointer(types.Typ[types.Byte]), llssa.InGo) + before := pkg.NewVar(coroPanicNativeE2EPackage+".Before", types.NewPointer(uint32Type), llssa.InGo) + after := pkg.NewVar(coroPanicNativeE2EPackage+".After", types.NewPointer(uint32Type), llssa.InGo) + + report := pkg.NewFunc(coroPanicNativeE2ERunReport, newSignature([]types.Type{pointer, pointer}, nil), llssa.InC) + reportBody := report.MakeBody(1) + requireCode := uint64(21) + requireCondition := func(condition llssa.Expr) { + reportBody.Call(require.Expr, condition, prog.IntVal(requireCode, prog.Int32())) + requireCode++ + } + normal := reportBody.Call(runtimeRun.Expr, report.Param(0), report.Param(1)) + requireCondition(reportBody.UnOp(token.NOT, normal)) + loaded := reportBody.Call(loadPanicRecord.Expr, report.Param(0)) + record := reportBody.Extract(loaded, 0) + published := reportBody.Extract(loaded, 1) + requireCondition(published) + requireCondition(reportBody.BinOp( + token.EQL, + reportBody.Field(record, 0), + prog.IntVal(coroPanicNativeE2EExplicitStatus, prog.Uint32()), + )) + nilPointer := prog.Nil(prog.VoidPtr()) + typeWord := reportBody.Field(record, 1) + dataWord := reportBody.Field(record, 2) + requireCondition(reportBody.BinOp(token.NEQ, typeWord, nilPointer)) + requireCondition(reportBody.BinOp(token.NEQ, typeWord, dataWord)) + requireCondition(reportBody.BinOp(token.EQL, dataWord, reportBody.Convert(prog.VoidPtr(), payload.Expr))) + requireCondition(reportBody.Call(deadG.Expr, report.Param(0))) + requireCondition(reportBody.UnOp(token.NOT, reportBody.Call(reclaimableG.Expr, report.Param(0)))) + destroyCalls := reportBody.Load(destroyCount.Expr) + requireCondition(reportBody.BinOp( + token.EQL, + destroyCalls, + prog.IntVal(coroPanicNativeE2EExpectedDestroys, prog.Uint32()), + )) + first := reportBody.Load(firstDestroy.Expr) + second := reportBody.Load(secondDestroy.Expr) + third := reportBody.Load(thirdDestroy.Expr) + requireCondition(reportBody.BinOp(token.NEQ, first, nilPointer)) + requireCondition(reportBody.BinOp(token.NEQ, second, nilPointer)) + requireCondition(reportBody.BinOp(token.NEQ, third, nilPointer)) + requireCondition(reportBody.BinOp(token.NEQ, first, second)) + requireCondition(reportBody.BinOp(token.NEQ, first, third)) + requireCondition(reportBody.BinOp(token.NEQ, second, third)) + requireCondition(reportBody.BinOp(token.EQL, reportBody.Load(before.Expr), one32)) + requireCondition(reportBody.BinOp(token.EQL, reportBody.Load(after.Expr), zero32)) + reportBody.Return() + + // The production scheduler core is intentionally compiled without the full + // standard-library runtime package. Keep ordinary pointer checks fail-stop + // and resolve unreachable core allocation edges directly to libc, matching + // the closed-static-spawn island. + assertNil := pkg.NewFunc(llssa.PkgRuntime+".AssertNilDeref", newSignature( + []types.Type{types.Typ[types.Bool]}, nil, + ), llssa.InGo) + assertBody := assertNil.MakeBody(3) + assertFail, assertValid := assertNil.Block(1), assertNil.Block(2) + assertBody.If(assertNil.Param(0), assertFail, assertValid) + assertBody.SetBlock(assertFail).Call(abort.Expr) + assertBody.Return() + assertBody.SetBlock(assertValid).Return() + uintptrType := types.Typ[types.Uintptr] + malloc := pkg.NewFunc("malloc", newSignature([]types.Type{uintptrType}, []types.Type{pointer}), llssa.InC) + calloc := pkg.NewFunc("calloc", newSignature([]types.Type{uintptrType, uintptrType}, []types.Type{pointer}), llssa.InC) + allocU := pkg.NewFunc(llssa.PkgRuntime+".AllocU", newSignature([]types.Type{uintptrType}, []types.Type{pointer}), llssa.InGo) + allocUBody := allocU.MakeBody(1) + allocUBody.Return(allocUBody.Call(malloc.Expr, allocU.Param(0))) + allocZ := pkg.NewFunc(llssa.PkgRuntime+".AllocZ", newSignature([]types.Type{uintptrType}, []types.Type{pointer}), llssa.InGo) + allocZBody := allocZ.MakeBody(1) + allocZBody.Return(allocZBody.Call(calloc.Expr, prog.IntVal(1, prog.Uintptr()), allocZ.Param(0))) + // The concrete *byte panic value materializes pointer and byte type + // descriptors. Their equality callbacks are metadata-only in this fixture; + // provide exact test-island implementations instead of extracting alg.go and + // its unrelated legacy runtime closure. + memequal8 := pkg.NewFunc(llssa.PkgRuntime+".memequal8", newSignature( + []types.Type{pointer, pointer}, []types.Type{types.Typ[types.Bool]}, + ), llssa.InGo) + memequal8Body := memequal8.MakeBody(1) + memequal8Pointer := prog.Pointer(prog.Byte()) + memequal8Body.Return(memequal8Body.BinOp( + token.EQL, + memequal8Body.Load(memequal8Body.Convert(memequal8Pointer, memequal8.Param(0))), + memequal8Body.Load(memequal8Body.Convert(memequal8Pointer, memequal8.Param(1))), + )) + memequalptr := pkg.NewFunc(llssa.PkgRuntime+".memequalptr", newSignature( + []types.Type{pointer, pointer}, []types.Type{types.Typ[types.Bool]}, + ), llssa.InGo) + memequalptrBody := memequalptr.MakeBody(1) + memequalptrPointer := prog.Pointer(prog.Uintptr()) + memequalptrBody.Return(memequalptrBody.BinOp( + token.EQL, + memequalptrBody.Load(memequalptrBody.Convert(memequalptrPointer, memequalptr.Param(0))), + memequalptrBody.Load(memequalptrBody.Convert(memequalptrPointer, memequalptr.Param(1))), + )) + + entry := pkg.NewFunc(coroPanicNativeE2EEntry, newSignature( + []types.Type{types.Typ[types.Int32], pointer}, []types.Type{types.Typ[types.Int32]}, + ), llssa.InC) + main := pkg.NewFunc("main", newSignature( + []types.Type{types.Typ[types.Int32], pointer}, []types.Type{types.Typ[types.Int32]}, + ), llssa.InC) + mainBody := main.MakeBody(1) + mainBody.Call(entry.Expr, main.Param(0), main.Param(1)) + mainBody.Return(prog.IntVal(0, prog.Int32())) + pkg.MaterializePreserveSyms() + return emitCoroSpawnNativeE2EObject(t, prog, pkg.Module(), filepath.Join(temp, "panic-driver.o")) +} + +func assertCoroPanicNativeE2ELinkedSymbols(t *testing.T, executable string) { + t.Helper() + nm, err := exec.LookPath("nm") + if err != nil { + t.Log("nm is unavailable; continuing without the linked coroutine panic symbol audit") + return + } + output, err := exec.Command(nm, executable).CombinedOutput() + if err != nil { + t.Fatalf("inspect linked coroutine panic island: %v\n%s", err, output) + } + symbols := string(output) + for _, required := range []string{ + "__llgo_coro_panic_prepare_v1", + coroPanicNativeE2ERunReport, + coroPanicNativeE2EDestroyObserve, + "github.com/goplus/llgo/runtime/internal/coro.PreparePanic", + "github.com/goplus/llgo/runtime/internal/coro.PanicDestroyed", + "github.com/goplus/llgo/runtime/internal/coro.LoadPanicRecord", + coroPanicNativeE2EPackage + ".panicChild$coro", + } { + if !strings.Contains(symbols, required) { + t.Fatalf("linked coroutine panic island is missing production/test-boundary symbol %q:\n%s", required, symbols) + } + } + for _, forbidden := range []string{ + "github.com/goplus/llgo/runtime/internal/runtime.Panic", + "github.com/goplus/llgo/runtime/internal/runtime.Rethrow", + "github.com/goplus/llgo/runtime/internal/runtime.TracePanic", + "github.com/goplus/llgo/runtime/internal/runtime.printany", + } { + if strings.Contains(symbols, forbidden) { + t.Fatalf("test-only coroutine panic island unexpectedly extracted legacy PanicABI symbol %q", forbidden) + } + } +} From 61b9937fe5a9712000792e5b7e0d5281372f2c4d Mon Sep 17 00:00:00 2001 From: Li Jie Date: Fri, 17 Jul 2026 00:31:16 +0800 Subject: [PATCH 32/32] docs(coro): record terminal panic prototype --- doc/llvm-coro-runtime-design.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/llvm-coro-runtime-design.md b/doc/llvm-coro-runtime-design.md index 66a15bfcda..10319d5ddf 100644 --- a/doc/llvm-coro-runtime-design.md +++ b/doc/llvm-coro-runtime-design.md @@ -1778,7 +1778,7 @@ Pure sync library/archive不需要链接scheduler。Executable一旦选择 `-sch 验收：纯 sync chain 只有 `F`；纯 async chain 只有 `F$coro`；动态 escape 才出现 descriptor/adapter；所有 `go` root和可挂起call都以LLVM-coro frame表示。 -当前落地状态（2026-07-16，实验 physical ABI v0/v1；scheduler ABI 已扩展到 `llgo.coro.scheduler.program-bootstrap.v2.closed-static-spawn.v0`）： +当前落地状态（2026-07-17，实验 physical ABI v0/v1；scheduler ABI 已扩展到 `llgo.coro.scheduler.program-bootstrap.v2.closed-static-spawn.v0`）： - 全程序 SSA 的 Effect、Demand、FuncRep、稳定 FunctionID、精确 emission universe、单 primary symbol 选择和 `CoroPlanDigest` 已落地。明确 plain 或 coro 的函数仍只有一个主体；仅真正动态的 func/`any`/interface consumer 才进入 descriptor/dispatch。缺失、过期或目标布局不匹配的计划与 cache manifest 均 fail closed。 - LLGo 已固定使用 `cpunion/llvm` PR #5 的 LLVM 19–22 绑定。该分支吸收上游 LLVM 22 的完整 switch API 变更，并保留 LLGo 所需的 switched-resume builder/CoroSplit API；19、20、21、22 CI 均通过。LLGo 不再覆盖 LLVM 19 以下版本。 @@ -1787,20 +1787,21 @@ Pure sync library/archive不需要链接scheduler。Executable一旦选择 `-sch - `program-bootstrap.v2` 在 codegen 前冻结五阶段表：`[internal runtime.init, init$abitypes, public runtime.init, selected main-package init, main.main]`。managed Go 阶段根据唯一 primary 选择 `DirectPlain` 或 `CoroRoot`；public runtime init 若存在则必须使用其 exact managed body，不存在时才由 compiler 生成 no-op。Coro 表项只绑定 package anchor/descriptor index，不复制函数体，也不把 catalog 当启动列表。 - planner 已把 internal runtime init、selected package init 和 `main.main` 注入 managed demand。普通同步 Go/标准库调用风格不变，调用者根据精确 effect 自动被染成 coro；scheduler-stack hook closure 则是单独审计的 NoSuspend island，不能通过强改 demand 或放宽 trusted closure 绕过。 - frozen foreign `//llgo:coro noblock` certificate 当前只授予已审计的 `time`、`pthread_self`、`pthread_mutex_init` 和 `pthread_mutex_unlock`。证书只移除未知阻塞，`IRQUnsafe` 仍保留但允许在普通 G 上执行。真实 runtime init 仍被 `pthread_key_create`、`rand`/`srand`、`GC_malloc`、mutex lock、Memcpy/Memset 等未完成边界挡住。 -- legacy PanicABI 仍是完整启动链的正式 blocker。exact proof 可追踪 `runtime.Panic → Rethrow → TracePanic → printany`，并在动态 `error.Error` 调用处停止；不能把该动态调用误标为 plain。新的 `llgo.coro.panic.explicit-status.v0` 已进入 digest、summary、cache、manifest 和 package/root ABI hash，但 active compiler build 仍全局 fail closed，直到下述 runtime core 有对应 compiler lowering。 +- legacy PanicABI 仍是完整启动链的正式 blocker。exact proof 可追踪 `runtime.Panic → Rethrow → TracePanic → printany`，并在动态 `error.Error` 调用处停止；不能把该动态调用误标为 plain。新的 `llgo.coro.panic.explicit-status.v0` 已进入 digest、summary、cache、manifest 和 package/root ABI hash；`cl` 已有下述严格子集的 compiler lowering，但 `internal/build` 仍保留 target-wide 全局 gate，尚不能把它作为完整程序的 production PanicABI 开启。 - 多基本块 CFG、聚合值、PHI 和抢占 lowering 已完成。自然循环、循环入口及每 64 条有效指令的长直线块插入 poll；scheduler 的 P 级原子 request 只有在 slow path 才执行 publish/yield/`llvm.coro.suspend`，fast path 不切换。LLVM 19–22 上均有 native64/wasm32 pre-/post-CoroSplit 与 object 测试。 - 第一条 production `go` 路径已经落地：严格限定为 closed static、top-level、非捕获、非泛型、非变参、零返回的 `go f(args)`。编译器先按 Go 顺序完整求值参数，再以显式 parent G 执行 begin，调用 target 唯一的 `DirectCoro` primary 到 LLVM initial suspend，commit 后在 parent 上 poll/yield；runtime 不接收用户 callback，也不依赖 TLS。owner 与 target 都由精确 `YieldOnly` seed 进入 effect 传播，因此 target 即使当前很短也保留抢占点，普通同步 caller 则透明 await 同一主体。 - Command `main` 的正常 continuation 现在显式通知 runtime。main root 完成后，single-P shutdown 先整体校验 ready/wait/current/action 状态，再封闭调度 gate，按 FIFO 取 ready G、按 active-child 到 root 顺序直接 `llvm.coro.destroy`，最后每个 task storage 只释放一次。该 v1 路径只接收 `YieldOnly|AwaitStructured` target 且拒绝非空 wait set；panic/Goexit 不经过正常 main-return hook。 -- terminal-only ExplicitStatus runtime core 已有 task-local 两字 `PanicRecord` 和原子 once publication。active panic frame 先经过 `coro.done` 验证并 destroy，之后 suspended-await ancestor 不再 resume，而是从深到 root 直接 destroy；最终保留 record 并返回独立 `PanicComplete`。该原型明确拒绝 nil type word、cleanup/recover flags、Goexit、implicit fault 和重复发布；尚未导出 compiler C hook，也未实现 defer/recover 或用户 `Error/String` 报告。 +- terminal-only ExplicitStatus runtime core 已有 task-local 两字 `PanicRecord`、原子 once publication和无 TLS 的 `__llgo_coro_panic_prepare_v1(g, handle, header, typeWord, dataWord)`。compiler 对精确 cleanup-free PhysicalABIV1 body 生成 `SuspendPanic`/`FinalSuspended`，panic 与 normal return branch 到同一个 LLVM final suspend；active panic frame 先经过 `coro.done` 验证并 destroy，之后 suspended-await ancestor 不再 resume，而是从深到 root 直接 destroy，最终保留 record 并返回独立 `PanicComplete`。当前 payload 只接受 typed nil 或从 package global 派生的 concrete pointer，确保 frame destroy 后 data word 仍有效；dynamic interface、scalar/local/parameter payload、cleanup/recover、Goexit、implicit fault、重复发布及 managed plain unwind 均 fail closed。尚未实现用户 `Error/String` 报告、最终进程退出所有权或 defer/recover。 - park/wake handshake 已落地 32-bit 原子 `WaitToken`、generation ticket、early/late completion、唯一 waiter claim、ABA 范围校验及 terminal gate。精确 intrinsic `llgo.coroPark(token, ticket)` 被 Effect 分析识别为 `MayPark`，并在调用者当前 LLVM frame 中生成 park prepare、stateID、`coro.suspend` 和恢复路径；没有隐藏在普通同步 helper 中。channel/timer/syscall 的 submit/retry producer 尚未接入。 - wait/preempt core 要求目标提供可靠的 32-bit atomic load/store/CAS。WASM 可直接满足；带 A 扩展的 RISC-V 可满足；ESP32-C3 RV32IMC 当前会在链接时缺少 `__atomic_*_4`，直到平台用 IRQ critical section 提供单核适配。这里故意不使用非原子 fallback。 - `wasip1`、`wasip2` 和 `wasm-unknown` 明确选择 leaking/nogc frame backend，不依赖 libuv 或 BDWGC。`wasip2` 与 `wasm-unknown` 已通过真实 `llgo build -target=...`、wasm magic/symbol closure、无 `GC_*`/undefined 检查，并由 wasmtime 运行返回 0。当前 `wasip2` 产物是 Preview 2 目标的 core module，尚不是 WIT component。 - frame allocator 已有 conservative BDWGC、nogc/WASM malloc 和 tinygogc/baremetal 后端。跨 suspend 的 pointer 目前只在 conservative 或 non-collecting 配置下安全；精确 frame root map、write barrier、STW、weak timer/finalizer 与 cleanup 语义尚未实现，不能据此宣称完整 Go GC 兼容。 - deterministic single-P runtime 已能管理多个 frame、ready queue、preempt request、park/wake、closed-static spawned G、正常 main-return ready-child cancellation、terminal panic frame destruction和 idle/requested/stopping/disabled 状态。尚无动态/closure/method `go` target、等待中 G 的 producer 解注册与取消、真实 tick/alarm request source、channel/select/sync slow path、timer/netpoll、异步 syscall submit/retry、完整 panic/defer/recover/Goexit 或多 P。 - native+nogc scheduler-island 已把真实 nested static `go` lowering、V2 entry/factory/control wrapper、production scheduler/spawn/shutdown/coroalloc 最终链接并执行。确定性 fixture 验证 `Before=1, After=0, Leaf=0`，最终符号审计同时要求 production `CommitSpawn`/`BeginCommandShutdown` 且禁止 legacy `Panic/Rethrow/TracePanic/printany`。该测试以四个 bounded init no-op 和 fail-stop nil-check/libc allocation stub 隔离完整标准库 runtime，因此证明的是可运行 scheduler 原型，不是完整 runtime 启动兼容。 +- terminal panic 的独立 native+nogc scheduler-island 已真实编译并运行 `panic(&GlobalPayload)`。production runner 必须返回 `PanicComplete` 的失败状态；bootstrap、main、panicChild 三个不同 LLVM handle 各 destroy 一次，两个祖先均不 resume，task-local record 在三层 frame 销毁后仍保持 exact type/data word，且 G 为 Dead/non-Reclaimable。最终二进制要求 production `PreparePanic`/`PanicDestroyed`/`LoadPanicRecord` 并禁止 legacy panic/print 链；测试 report 只观察当前 fail-closed terminal 状态，不代替 production printer/exit owner。 - 完整真实 `entry → allocator → v2 factory → runtime/package init → main → scheduler` linked smoke 仍受上述 runtime/Panic/foreign blockers 限制；scheduler-island、runtime adapter 和 freestanding wasm CLI fixture 各自证明的边界不能合并表述为完整 Go runtime 已经端到端运行。 - 当前 cache digest 只解决同一完整程序计划下的内部 package cache；未知未来 caller 可复用的预编译 archive/标准库仍需 producer summary、canonical boundary Dispatch 和 linker ABI 校验。 -- 后续依赖顺序是：先为 terminal ExplicitStatus core 增加 compiler `SuspendPanic`/hook lowering并保持 cleanup/implicit fault fail closed，同时为 WaitToken 增加可注销、可静默迟到 completion 的稳定 registration；再实现 dynamic `error.Error`/`Stringer` descriptor、真实 platform request source 与 channel/timer/syscall producer并跑完整 runtime linked smoke；随后补 suspended-frame GC、defer/recover/Goexit、多 P 与各 target event backend。动态/closure/method `go` target只在 canonical descriptor transport 完成后开启。所有阶段保持无栈、单 primary 和未证明即 fail closed。 +- 后续依赖顺序是：先为 WaitToken 增加可注销、可静默迟到 completion 的稳定 registration，并为 terminal ExplicitStatus 增加 dynamic `error.Error`/`Stringer` descriptor 与 production printer/exit owner；再接入真实 platform request source、channel/timer/syscall producer并跑完整 runtime linked smoke；随后补 suspended-frame GC、defer/recover/Goexit、多 P 与各 target event backend。动态/closure/method `go` target只在 canonical descriptor transport 完成后开启。所有阶段保持无栈、单 primary 和未证明即 fail closed。 ### Phase 1：单 P deterministic scheduler