diff --git a/cl/coro_abi.go b/cl/coro_abi.go index fabb49fa24..e1c2d20673 100644 --- a/cl/coro_abi.go +++ b/cl/coro_abi.go @@ -1460,8 +1460,7 @@ func validateCoroPhysicalABIForOwner( plan.StaticOutcome && plan.Exec&(coro.BlockForeign|coro.ThreadAffine|coro.NeedsCleanupFrame|coro.OpaqueExec) == 0) && (plan.FuncRep == coro.DirectCoro || outcomePlainTwin && plan.FuncRep == coro.Dispatch) && (plan.Effect == coro.OutcomeStructured || outcomePlainTwin && - plan.Effect.Contains(coro.OutcomeStructured) && - plan.Effect&^(coro.YieldOnly|coro.AwaitStructured|coro.OutcomeStructured) == 0) + plan.Effect&^(coro.AwaitStructured|coro.OutcomeStructured|coro.MayPark) == 0) if plan.Emission != coro.EmitCoroutine && !outcomePlain || plan.FuncRep != coro.DirectCoro && !managedDispatchTarget && !rawMethodDispatchToken { return fail("requires a direct coroutine/outcome or capability-certified Dispatch emission, got emission=%s representation=%s", plan.Emission, plan.FuncRep) @@ -1781,6 +1780,7 @@ func validateCoroPhysicalABIForOwner( panics := 0 awaits := 0 parks := 0 + nativeBlocks := 0 foreignWaits := 0 yields := 0 spawns := 0 @@ -2016,6 +2016,13 @@ func validateCoroPhysicalABIForOwner( "generated C2 foreign suspend has no frozen typed errno operation") } foreignWaits++ + } else if intrinsic && semantics == CoroIntrinsicCallInlineNativeBlock { + if !isLLGoSyscallIntrinsic(frozen.opcode) || + instructionPlan.operation != coroPhysicalOperationNativeSyscall { + return coroLeafInstructionError(fn, plan, instr, + "native blocking intrinsic has no frozen native-syscall recipe") + } + nativeBlocks++ } else if intrinsic && semantics == CoroIntrinsicCallInlineSuspend { if isLLGoSyscallIntrinsic(frozen.opcode) { if instructionPlan.operation != coroPhysicalOperationWorkerSyscall { @@ -2036,7 +2043,9 @@ func validateCoroPhysicalABIForOwner( goexits++ } if intrinsic { - if isLLGoSyscallIntrinsic(frozen.opcode) && semantics != CoroIntrinsicCallInlineSuspend { + if isLLGoSyscallIntrinsic(frozen.opcode) && + semantics != CoroIntrinsicCallInlineSuspend && + semantics != CoroIntrinsicCallInlineNativeBlock { return coroLeafInstructionError(fn, plan, instr, "elided worker llgo.syscall has no frozen function-word capability") } @@ -2166,8 +2175,8 @@ func validateCoroPhysicalABIForOwner( if awaits != 0 && !plan.Effect.Contains(coro.AwaitStructured) { return fail("child-await body lacks await-structured final effect: %s", plan.Effect) } - if parks != 0 && !plan.Effect.Contains(coro.MayPark) { - return fail("structured-park body lacks may-park final effect: %s", plan.Effect) + if parks+nativeBlocks != 0 && !plan.Effect.Contains(coro.MayPark) { + return fail("park/native-block body lacks may-park final effect: %s", plan.Effect) } if foreignWaits != 0 && !plan.Effect.Contains(coro.WaitForeign) { return fail("bounded worker body lacks wait-foreign final effect: %s", plan.Effect) @@ -2183,8 +2192,8 @@ func validateCoroPhysicalABIForOwner( !plan.Effect.Contains(coro.OutcomeStructured)) { return fail("Goexit body lacks outcome-structured owner effect: declared=%s local=%s final=%s", plan.DeclaredEffect, plan.LocalEffect, plan.Effect) } - if plan.DeclaredEffect.Contains(coro.MayPark) && parks == 0 { - return fail("declared may-park effect has no exact structured park intrinsic") + if plan.DeclaredEffect.Contains(coro.MayPark) && parks+nativeBlocks == 0 { + return fail("declared may-park effect has no exact structured park or native blocking intrinsic") } if plan.DeclaredEffect.Contains(coro.WaitForeign) && foreignWaits == 0 { return fail("declared wait-foreign effect has no exact bounded worker operation") diff --git a/cl/coro_defer.go b/cl/coro_defer.go index 20b68275e3..cf7bc9834e 100644 --- a/cl/coro_defer.go +++ b/cl/coro_defer.go @@ -1256,6 +1256,7 @@ func validateCoroStaticCleanupNoUnwind( return fmt.Sprintf("block %d intrinsic: %v", block.Index, err) } if !intrinsic || (semantics != CoroIntrinsicCallInlineNoSuspend && semantics != CoroIntrinsicCallInlineSuspend && + semantics != CoroIntrinsicCallInlineNativeBlock && semantics != CoroIntrinsicCallInlineYield) { return fmt.Sprintf("block %d intrinsic has unproved semantics %d", block.Index, uint8(semantics)) } diff --git a/cl/coro_frame_retention.go b/cl/coro_frame_retention.go index cd8e228656..03645a5243 100644 --- a/cl/coro_frame_retention.go +++ b/cl/coro_frame_retention.go @@ -1572,7 +1572,9 @@ func (b *coroFrameRetentionRootBuilder) boundedCallKind(call *ssa.Call) (coroFra allocation := coroFrameRetentionDirectAllocRoot(call.Common().Args[0], make(map[ssa.Value]bool)) if _, retained := b.proof.allocations[allocation]; retained { semantics, intrinsic, err := coroIntrinsicCallSiteSemantics(b.audit.universe, call) - if err == nil && (!intrinsic || semantics != CoroIntrinsicCallInlineSuspend) { + if err == nil && (!intrinsic || + (semantics != CoroIntrinsicCallInlineSuspend && + semantics != CoroIntrinsicCallInlineNativeBlock)) { return coroFrameRetentionCallParkOwnerV1, true } } @@ -1586,7 +1588,9 @@ func (b *coroFrameRetentionRootBuilder) boundedCallKind(call *ssa.Call) (coroFra return coroFrameRetentionCallWorkerV1, true } } - if err == nil && intrinsic && semantics == CoroIntrinsicCallInlineSuspend { + if err == nil && intrinsic && + (semantics == CoroIntrinsicCallInlineSuspend || + semantics == CoroIntrinsicCallInlineNativeBlock) { workerCertified := false if b.audit.plan == nil { // Report-only physical audits have no lowering authority. They may diff --git a/cl/coro_frame_roots_test.go b/cl/coro_frame_roots_test.go index c683b1dca1..6f15306634 100644 --- a/cl/coro_frame_roots_test.go +++ b/cl/coro_frame_roots_test.go @@ -113,7 +113,9 @@ func TestCoroFrameExactRootsAndUintptrKeepaliveAreFrozen(t *testing.T) { childCall = instruction } semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(instruction) - if err == nil && intrinsic && semantics == CoroIntrinsicCallInlineSuspend { + if err == nil && intrinsic && + (semantics == CoroIntrinsicCallInlineSuspend || + semantics == CoroIntrinsicCallInlineNativeBlock) { workerCall = instruction } case *ssa.IndexAddr: diff --git a/cl/coro_lowering_facts.go b/cl/coro_lowering_facts.go index 31e8815db6..fdfd27737c 100644 --- a/cl/coro_lowering_facts.go +++ b/cl/coro_lowering_facts.go @@ -507,6 +507,8 @@ func coroIntrinsicLoweringRecipe(semantics CoroIntrinsicCallSemantics) (coro.Rec return coro.RecipeID("cl.intrinsic.inline-with-helpers.v0"), coro.NoSuspend case CoroIntrinsicCallInlineSuspend: return coro.RecipeID("cl.intrinsic.inline-suspend.v0"), coro.MayPark + case CoroIntrinsicCallInlineNativeBlock: + return coro.RecipeID("cl.intrinsic.inline-native-block.v1"), coro.MayPark case CoroIntrinsicCallInlineForeignSuspend: return coro.RecipeID("cl.intrinsic.inline-foreign-suspend.v1"), coro.WaitForeign case CoroIntrinsicCallInlineYield: diff --git a/cl/coro_outcome_plain.go b/cl/coro_outcome_plain.go index b96c988767..360611d6b4 100644 --- a/cl/coro_outcome_plain.go +++ b/cl/coro_outcome_plain.go @@ -214,7 +214,9 @@ func validateStaticOutcomeFrozenPlan(plan *coroPhysicalFunctionPlan, logical cor } operationSupported := physical.operation == coroPhysicalOperationNone || physical.operation == coroPhysicalOperationControl && - !physical.operationControl.NativeActivationBound() + !physical.operationControl.NativeActivationBound() || + physical.operation == coroPhysicalOperationNativeSyscall && + physical.semantic.recipe == coro.RecipeID("cl.intrinsic.inline-native-block.v1") if !operationSupported || physical.controlFailureHard || physical.operationFailure != "" || physical.outcomeFailure != "" { return fmt.Errorf( diff --git a/cl/coro_physical_plan.go b/cl/coro_physical_plan.go index b68e255c66..4a88333e0f 100644 --- a/cl/coro_physical_plan.go +++ b/cl/coro_physical_plan.go @@ -193,6 +193,10 @@ const ( coroPhysicalOperationWorkerCgoErrno coroPhysicalOperationHostCall coroPhysicalOperationControl + // NativeSyscall executes the compiler-certified word call synchronously on + // the current M. It owns no park/resume transaction and must not request the + // program worker fleet. + coroPhysicalOperationNativeSyscall ) func (recipe coroPhysicalOperationRecipe) String() string { @@ -225,6 +229,8 @@ func (recipe coroPhysicalOperationRecipe) String() string { return "host-operation" case coroPhysicalOperationControl: return "control-operation" + case coroPhysicalOperationNativeSyscall: + return "native-syscall" default: return fmt.Sprintf("physical-operation-recipe(%d)", uint8(recipe)) } @@ -1407,7 +1413,8 @@ func planCoroPhysicalOperationInstruction( return } if found && frozen.plan.Intrinsic && isLLGoSyscallIntrinsic(frozen.opcode) && - frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineSuspend { + (frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineSuspend || + frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineNativeBlock) { if !capabilities.worker { result.operationFailure = "worker llgo.syscall requires the bounded worker capability" return @@ -1416,7 +1423,11 @@ func planCoroPhysicalOperationInstruction( result.operationFailure = "invalid worker llgo.syscall capability: " + err.Error() return } - result.operation = coroPhysicalOperationWorkerSyscall + if frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineNativeBlock { + result.operation = coroPhysicalOperationNativeSyscall + } else { + result.operation = coroPhysicalOperationWorkerSyscall + } return } } diff --git a/cl/coro_physical_plan_test.go b/cl/coro_physical_plan_test.go index bcbae11662..323fc33e54 100644 --- a/cl/coro_physical_plan_test.go +++ b/cl/coro_physical_plan_test.go @@ -168,6 +168,13 @@ func Worker(enabled bool) { } plan = physical.instructions[reachable] + plan.operation = coroPhysicalOperationNativeSyscall + physical.instructions[reachable] = plan + capabilities, err = commit().programCapabilities() + if err != nil || capabilities.Worker() { + t.Fatalf("reachable native syscall capability = (%v, %v), want no worker", capabilities, err) + } + plan.operation = coroPhysicalOperationWorkerCgo physical.instructions[reachable] = plan capabilities, err = commit().programCapabilities() diff --git a/cl/coro_poll_wait_test.go b/cl/coro_poll_wait_test.go index 906caf6ab4..8fd8eeddcd 100644 --- a/cl/coro_poll_wait_test.go +++ b/cl/coro_poll_wait_test.go @@ -119,7 +119,7 @@ func TestCoroPollWaitCurrentFrameNativeAndWasm32(t *testing.T) { rootPlan, ok := plan.FunctionPlan(root) if !ok || rootPlan.Emission != coro.EmitCoroutine || rootPlan.FuncRep != coro.DirectCoro || !rootPlan.DeclaredEffect.Contains(coro.MayPark) || !rootPlan.LocalEffect.Contains(coro.MayPark) || - !rootPlan.Effect.Contains(coro.MayPark) { + !rootPlan.Effect.Contains(coro.MayPark) || rootPlan.HasStaticOutcome() { t.Fatalf("Root plan = %+v, present=%t; want one local poll-park coroutine", rootPlan, ok) } if !plan.ElidesCall(waitCall) { @@ -268,6 +268,8 @@ func compileCoroPollWaitFixture(t *testing.T, target *llssa.Target) ( EmissionUniverse: ssaUniverse, FunctionIDs: functionIDs, MaxPlainInstructions: -1, + OutcomeMode: coro.OutcomeExplicitStatus, + ClassifyLocalBody: universe.CoroLocalBodyFacts, ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { if fn == root { return coro.SSAFunctionPolicy{Effect: coro.MayPark}, nil diff --git a/cl/coro_program_ir.go b/cl/coro_program_ir.go index 5d34fabf56..ae702c3d8b 100644 --- a/cl/coro_program_ir.go +++ b/cl/coro_program_ir.go @@ -412,14 +412,15 @@ func coroOutcomePlainDAGSemanticRecipe(plan coroSemanticInstructionPlan) bool { // ProgramIR builder step admits only operations whose frozen call recipe is // independently shape-checked and allocation-free. // -// Atomic intrinsics acquire their bounded leaf proof here. A real inline yield -// is also recorded here as an evaluated local effect: this distinguishes an -// explicit scheduler handoff from the synthetic YieldOnly/NeedsPreempt seed -// added later for an otherwise synchronous CFG loop. Static outcome twins may -// omit the latter under the current compute-blocking policy, but must never -// erase the former. Other InlineNoSuspend intrinsics include asm, dynamic -// alloca, control transfer, and target-specific operations; the broad enum is -// therefore not by itself an outcome-plain proof. +// Atomic intrinsics acquire their bounded leaf proof here. Structured waits, +// real yields, and terminal outcomes are also projected into the local semantic +// facts so a static-outcome decision cannot mistake an elided declaration for +// a synchronous operation. The sole dual recipe is InlineNativeBlock: its +// certified native syscall releases the execution domain and returns on the +// same M without suspending the LLVM coroutine, so an exact static caller may +// use the synchronous outcome twin. Other InlineNoSuspend intrinsics include +// asm, dynamic alloca, control transfer, and target-specific operations; the +// broad enum is therefore not by itself an outcome-plain proof. func (ir *coroProgramIR) finalizeOutcomePlainIntrinsicSemantics( prog llssa.Program, functions []*ssa.Function, @@ -447,31 +448,36 @@ func (ir *coroProgramIR) finalizeOutcomePlainIntrinsicSemantics( frozen.plan.Elision != CoroCallElidedIntrinsic { continue } - atomic := frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineNoSuspend && + semantics := frozen.plan.IntrinsicSemantics + atomic := semantics == CoroIntrinsicCallInlineNoSuspend && isCoroAtomicIntrinsic(frozen.opcode) - realYield := frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineYield - if !atomic && !realYield { + structured := semantics == CoroIntrinsicCallInlineSuspend || + semantics == CoroIntrinsicCallInlineNativeBlock || + semantics == CoroIntrinsicCallInlineForeignSuspend || + semantics == CoroIntrinsicCallInlineYield || + semantics == CoroIntrinsicCallInlineOutcome + if !atomic && !structured { continue } for _, owner := range sortedUseOwners(function) { key := emissionFunctionOwnerKey{function: function, owner: owner} if _, sealed := ir.siteOwners[key]; !sealed { - return fmt.Errorf("atomic intrinsic in %q has no frozen semantic owner %q", function.Name(), owner.identity) + return fmt.Errorf("intrinsic refinement in %q has no frozen semantic owner %q", function.Name(), owner.identity) } semantic, present := ir.semanticPlans[key][call] if !present || !semantic.evaluated || semantic.recipe != coro.RecipeID("cl.ssa.call.v1") || semantic.effect != coro.NoSuspend || semantic.exec != 0 { - return fmt.Errorf("atomic intrinsic call %q has an incompatible preliminary semantic recipe", call.String()) + return fmt.Errorf("intrinsic call %q has an incompatible preliminary semantic recipe", call.String()) } if atomic { semantic.recipe = coro.RecipeID("cl.intrinsic.atomic.inline-nosuspend.v1") semantic.outcomePlainLeaf = true semantic.staticOutcome = true } else { - semantic.recipe, semantic.effect = coroIntrinsicLoweringRecipe(CoroIntrinsicCallInlineYield) + semantic.recipe, semantic.effect = coroIntrinsicLoweringRecipe(semantics) semantic.materialized = true semantic.outcomePlainLeaf = false - semantic.staticOutcome = false + semantic.staticOutcome = semantics == CoroIntrinsicCallInlineNativeBlock } ir.semanticPlans[key][call] = semantic } @@ -486,11 +492,11 @@ func (ir *coroProgramIR) finalizeOutcomePlainIntrinsicSemantics( key := emissionFunctionOwnerKey{function: function, owner: owner} preamble, present := ir.functionPreambles[key] if !present { - return fmt.Errorf("finalize atomic intrinsic body %q: owner %q has no function preamble", function.Name(), owner.identity) + return fmt.Errorf("finalize intrinsic body %q: owner %q has no function preamble", function.Name(), owner.identity) } facts, err := deriveCoroLocalBodyFacts(prog, function, ir.semanticPlans[key], preamble.emitsGoBody) if err != nil { - return fmt.Errorf("finalize atomic intrinsic body %q: %w", function.Name(), err) + return fmt.Errorf("finalize intrinsic body %q: %w", function.Name(), err) } if index != 0 && !final.Same(facts) { return fmt.Errorf("function %q acquired owner-dependent finalized local semantic facts", function.Name()) diff --git a/cl/coro_timer_sleep_test.go b/cl/coro_timer_sleep_test.go index 5639e1c626..15232d9cd7 100644 --- a/cl/coro_timer_sleep_test.go +++ b/cl/coro_timer_sleep_test.go @@ -79,7 +79,7 @@ func TestCoroTimerSleepCurrentFrameNativeAndWasm32(t *testing.T) { rootPlan, ok := plan.FunctionPlan(root) if !ok || rootPlan.Emission != coro.EmitCoroutine || rootPlan.FuncRep != coro.DirectCoro || !rootPlan.DeclaredEffect.Contains(coro.MayPark) || !rootPlan.LocalEffect.Contains(coro.MayPark) || - !rootPlan.Effect.Contains(coro.MayPark) { + !rootPlan.Effect.Contains(coro.MayPark) || rootPlan.HasStaticOutcome() { t.Fatalf("Root plan = %+v, present=%t; want one local timer-park coroutine", rootPlan, ok) } if !plan.ElidesCall(sleepCall) { @@ -175,7 +175,7 @@ func TestCoroControlledTimerWaitCurrentFrameNativeAndWasm32(t *testing.T) { rootPlan, ok := plan.FunctionPlan(root) if !ok || rootPlan.Emission != coro.EmitCoroutine || !rootPlan.Effect.Contains(coro.MayPark) || - !plan.ElidesCall(waitCall) { + rootPlan.HasStaticOutcome() || !plan.ElidesCall(waitCall) { t.Fatalf("controlled Timer Root plan = %+v, present=%t, elided=%t", rootPlan, ok, plan.ElidesCall(waitCall)) } if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { @@ -290,6 +290,8 @@ func compileCoroTimerIntrinsicFixture(t *testing.T, target *llssa.Target, source EmissionUniverse: ssaUniverse, FunctionIDs: functionIDs, MaxPlainInstructions: -1, + OutcomeMode: coro.OutcomeExplicitStatus, + ClassifyLocalBody: universe.CoroLocalBodyFacts, ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { if fn == root { return coro.SSAFunctionPolicy{Effect: coro.MayPark}, nil diff --git a/cl/coro_worker.go b/cl/coro_worker.go index 464f8c6b09..559f05e42d 100644 --- a/cl/coro_worker.go +++ b/cl/coro_worker.go @@ -181,6 +181,109 @@ type coroHostWordOperationV1 struct { metadata []llssa.Expr } +// emitCoroSameMWordCall is the shared synchronous status transaction for the +// two runtime boundaries which execute a word call on the current native M. +// The caller owns result storage so a locked-foreign direct branch can share it +// with its worker branch, while a native syscall outcome can use native-stack +// storage without acquiring a coroutine frame. +func (p *context) emitCoroSameMWordCall( + b llssa.Builder, + hookName string, + function llssa.Expr, + traceTarget llssa.Expr, + args []llssa.Expr, + r1, r2, errno llssa.Expr, +) { + if p == nil { + panic("same-M word call requires one active structured body and exact storage") + } + task := p.managedPhysicalTask() + if !p.hasStructuredOutcomePhysicalBody() || b == nil || b.Func != p.fn || + task.IsNil() || function.IsNil() || traceTarget.IsNil() || + r1.IsNil() || r2.IsNil() || errno.IsNil() || len(args) > coroWorkerMaxArgsV1 { + panic("same-M word call requires one active structured body and exact storage") + } + switch hookName { + case coroNativeSyscallCallHookV1: + case coroOSThreadForeignCallHookV1: + if !p.hasCoroPhysicalBody() { + panic("locked foreign word call requires a full coroutine body") + } + default: + panic("same-M word call received an unsupported runtime boundary") + } + word := p.prog.Uintptr() + if !types.Identical(function.RawType(), word.RawType()) || + !types.Identical(traceTarget.RawType(), word.RawType()) { + panic("same-M function or trace target is not uintptr-shaped") + } + for index, argument := range args { + if argument.IsNil() || !types.Identical(argument.RawType(), word.RawType()) { + panic(fmt.Sprintf("same-M word call argument %d is not uintptr-shaped", index)) + } + } + + zero := p.prog.Zero(word) + directArgs := make([]llssa.Expr, 0, 4+coroWorkerMaxArgsV1+3) + directArgs = append(directArgs, + task, + function, + traceTarget, + p.prog.IntVal(uint64(len(args)), p.prog.Uint32()), + ) + for index := 0; index < coroWorkerMaxArgsV1; index++ { + if index < len(args) { + directArgs = append(directArgs, args[index]) + } else { + directArgs = append(directArgs, zero) + } + } + directArgs = append(directArgs, r1, r2, errno) + direct := p.pkg.NewFunc(hookName, coroOSThreadForeignCallSignature(), llssa.InC) + status := b.Call(direct.Expr, directArgs...) + normal := b.Func.MakeBlock() + memoryFault := b.Func.MakeBlock() + divideFault := b.Func.MakeBlock() + invalid := b.Func.MakeBlock() + dispatch := b.Switch(status, invalid) + dispatch.Case(p.prog.IntVal(coroWorkerResumeSuccessV1, p.prog.Uint32()), normal) + dispatch.Case(p.prog.IntVal(coroWorkerResumeFaultMemoryV1, p.prog.Uint32()), memoryFault) + dispatch.Case(p.prog.IntVal(coroWorkerResumeFaultDivideV1, p.prog.Uint32()), divideFault) + dispatch.End(b) + b.SetBlockEx(memoryFault, llssa.AtEnd, false) + p.compileCoroTerminalFault(b, coroFaultNilV1) + b.SetBlockEx(divideFault, llssa.AtEnd, false) + p.compileCoroTerminalFault(b, coroFaultIntegerDivideByZeroV1) + b.SetBlockEx(invalid, llssa.AtEnd, false) + b.Unreachable() + b.SetBlockContinuation(normal) +} + +// compileCoroNativeSyscallWordCall is the synchronous native-M half of a +// certified llgo.syscall operation. The runtime hook releases and reacquires +// the managed execution domain around the blocking call, but it never parks +// this LLVM coroutine. Consequently the same transaction is valid in either a +// full coroutine body or its exact static-outcome twin. +func (p *context) compileCoroNativeSyscallWordCall( + b llssa.Builder, + function llssa.Expr, + traceTarget llssa.Expr, + args []llssa.Expr, + keepalives []llssa.Expr, +) coroWorkerWordResultV1 { + word := p.prog.Uintptr() + r1 := b.Alloc(word, false) + r2 := b.Alloc(word, false) + errno := b.Alloc(word, false) + p.emitCoroSameMWordCall( + b, coroNativeSyscallCallHookV1, function, traceTarget, args, r1, r2, errno, + ) + if len(keepalives) != 0 { + b.KeepAlive(keepalives...) + } + return coroWorkerWordResultV1{r1: b.Load(r1), r2: b.Load(r2), errno: b.Load(errno)} +} + // compileCoroHostOperation lowers one source-style synchronous host request // into the shared scalar external-operation source. Unlike a native worker it // has no current-thread/direct branch: only a later host turn owns the @@ -226,7 +329,6 @@ func (p *context) compileCoroHostOperation( physicalWords, keepaliveSlots, &coroHostWordOperationV1{metadata: metadata}, - false, ) return b.Aggregate( p.type_(results, llssa.InGo), @@ -236,10 +338,12 @@ func (p *context) compileCoroHostOperation( ) } -// compileCoroWorkerWordCall is the one physical ForeignWait transaction used -// by both llgo.syscall and exact ordinary C-call thunks. function always names -// a uniform uintptr (...uintptr) thunk whose arity is len(args); typed foreign -// declarations are never called through this ABI directly. +// compileCoroWorkerWordCall is the one physically suspending ForeignWait +// transaction used by host operations and exact ordinary C-call thunks. +// function always names a uniform uintptr (...uintptr) thunk whose arity is +// len(args); typed foreign declarations are never called through this ABI +// directly. Certified native syscalls use compileCoroNativeSyscallWordCall and +// never enter this park/resume transaction. func (p *context) compileCoroWorkerWordCall( b llssa.Builder, function llssa.Expr, @@ -247,11 +351,9 @@ func (p *context) compileCoroWorkerWordCall( args []llssa.Expr, keepaliveSlots []llssa.Expr, host *coroHostWordOperationV1, - nativeSyscall bool, ) coroWorkerWordResultV1 { body := p.requireCoroWorkerBody(b) if function.IsNil() || host == nil && traceTarget.IsNil() || host != nil && !traceTarget.IsNil() || - host != nil && nativeSyscall || len(args) > coroWorkerMaxArgsV1 || host != nil && len(host.metadata) != 0 && len(host.metadata) != coroHostOperationDeadlineMetadataWordsV1 { @@ -365,53 +467,19 @@ func (p *context) compileCoroWorkerWordCall( } join := b.Func.MakeBlock() - directArgs := make([]llssa.Expr, 0, 4+coroWorkerMaxArgsV1+3) - directArgs = append(directArgs, body.task, function, traceTarget, argcValue) - for index := 0; index < coroWorkerMaxArgsV1; index++ { - if index < len(args) { - directArgs = append(directArgs, args[index]) - } else { - directArgs = append(directArgs, zero) - } - } - directArgs = append(directArgs, r1, r2, errno) - emitDirect := func(hookName string) { - direct := p.pkg.NewFunc( - hookName, coroOSThreadForeignCallSignature(), llssa.InC, - ) - directStatus := b.Call(direct.Expr, directArgs...) - directNormal := b.Func.MakeBlock() - directMemoryFault := b.Func.MakeBlock() - directDivideFault := b.Func.MakeBlock() - directInvalid := b.Func.MakeBlock() - directDispatch := b.Switch(directStatus, directInvalid) - directDispatch.Case(p.prog.IntVal(coroWorkerResumeSuccessV1, p.prog.Uint32()), directNormal) - directDispatch.Case(p.prog.IntVal(coroWorkerResumeFaultMemoryV1, p.prog.Uint32()), directMemoryFault) - directDispatch.Case(p.prog.IntVal(coroWorkerResumeFaultDivideV1, p.prog.Uint32()), directDivideFault) - directDispatch.End(b) - b.SetBlockEx(directMemoryFault, llssa.AtEnd, false) - p.compileCoroTerminalFault(b, coroFaultNilV1) - b.SetBlockEx(directDivideFault, llssa.AtEnd, false) - p.compileCoroTerminalFault(b, coroFaultIntegerDivideByZeroV1) - b.SetBlockEx(directInvalid, llssa.AtEnd, false) - b.Unreachable() - b.SetBlockEx(directNormal, llssa.AtEnd, false) - b.Jump(join) - } - if nativeSyscall { - emitDirect(coroNativeSyscallCallHookV1) - } else { - lockedHook := p.pkg.NewFunc(coroOSThreadLockedHookV1, coroOSThreadLockedSignature(), llssa.InC) - locked := b.Call(lockedHook.Expr, body.task) - directBlock := b.Func.MakeBlock() - workerBlock := b.Func.MakeBlock() - b.If(locked, directBlock, workerBlock) - b.SetBlockEx(directBlock, llssa.AtEnd, false) - emitDirect(coroOSThreadForeignCallHookV1) - b.SetBlockEx(workerBlock, llssa.AtEnd, false) - emitPark(b) - b.Jump(join) - } + lockedHook := p.pkg.NewFunc(coroOSThreadLockedHookV1, coroOSThreadLockedSignature(), llssa.InC) + locked := b.Call(lockedHook.Expr, body.task) + directBlock := b.Func.MakeBlock() + workerBlock := b.Func.MakeBlock() + b.If(locked, directBlock, workerBlock) + b.SetBlockEx(directBlock, llssa.AtEnd, false) + p.emitCoroSameMWordCall( + b, coroOSThreadForeignCallHookV1, function, traceTarget, args, r1, r2, errno, + ) + b.Jump(join) + b.SetBlockEx(workerBlock, llssa.AtEnd, false) + emitPark(b) + b.Jump(join) b.SetBlockContinuation(join) // Keep every independently proved typed owner live until direct return or // worker completion has selected this normal path; llvm.fake.use emits no @@ -427,11 +495,11 @@ func (p *context) compileCoroWorkerWordCall( // in that continuation preserves both valid LLVM SSA and the typed owner until // the physical completion/retirement boundary. func (p *context) coroCallKeepaliveSources(call ssa.CallInstruction) []ssa.Value { - body := p.coroBody() - if body == nil || body.frameRetention == nil || call == nil { + plan := p.coroEmissionPlan() + if plan == nil || plan.frameRetention == nil || call == nil { return nil } - return body.frameRetention.exactCallKeepaliveSources(call) + return plan.frameRetention.exactCallKeepaliveSources(call) } func (p *context) coroCallKeepaliveStorageType(source ssa.Value) llssa.Type { @@ -510,25 +578,42 @@ func (p *context) coroWorkerOrdinaryCall(common *ssa.CallCommon) *ssa.Call { return nil } -// compileCoroWorkerSyscall lowers one source-style synchronous llgo.syscall -// family operation into the common ForeignWait recipe. All conventions share -// one park/resume CFG; only the final errno predicate differs. Argument -// evaluation happens before publication, and the fixed pool receives only -// copied uintptr words. -func (p *context) compileCoroWorkerSyscall( +// compileCoroSyscallOperation lowers one source-style synchronous llgo.syscall +// family operation through its frozen target recipe. A certified native target +// blocks synchronously on the current M after releasing the execution domain; +// a worker-only target uses the full park/resume transaction. All conventions +// share the same result and errno projection. +func (p *context) compileCoroSyscallOperation( b llssa.Builder, call *ssa.CallCommon, args []ssa.Value, results *types.Tuple, convention syscallFailureConvention, + semantics CoroIntrinsicCallSemantics, ) llssa.Expr { direct := p.coroWorkerOrdinaryCall(call) + if direct == nil { + panic("coroutine syscall lowering lost its exact source call") + } compiled := make([]llssa.Expr, len(args)) for index, argument := range args { compiled[index] = p.compileValue(b, argument) } - keepaliveSlots := p.compileCoroCallKeepaliveSlots(b, direct) - result := p.compileCoroWorkerWordCall(b, compiled[0], compiled[0], compiled[1:], keepaliveSlots, nil, true) + var result coroWorkerWordResultV1 + switch semantics { + case CoroIntrinsicCallInlineNativeBlock: + keepalives := p.compileCoroCallKeepaliveValues(b, direct) + result = p.compileCoroNativeSyscallWordCall( + b, compiled[0], compiled[0], compiled[1:], keepalives, + ) + case CoroIntrinsicCallInlineSuspend: + keepaliveSlots := p.compileCoroCallKeepaliveSlots(b, direct) + result = p.compileCoroWorkerWordCall( + b, compiled[0], compiled[0], compiled[1:], keepaliveSlots, nil, + ) + default: + panic(fmt.Sprintf("coroutine syscall lowering received incompatible semantics %d", semantics)) + } errnoValue := p.filterSyscallErrno(b, result.r1, result.errno, convention) return b.Aggregate(p.type_(results, llssa.InGo), result.r1, result.r2, errnoValue) } diff --git a/cl/coro_worker_cgo.go b/cl/coro_worker_cgo.go index c6511b6576..c40f9ad457 100644 --- a/cl/coro_worker_cgo.go +++ b/cl/coro_worker_cgo.go @@ -875,7 +875,6 @@ func (p *context) compileCoroWorkerCgoErrnoCall( []llssa.Expr{b.Convert(p.prog.Uintptr(), record)}, nil, nil, - false, ) b.KeepAlive(record) p.cgoRet = b.LoadKnownNonNil(b.FieldAddr(record, shape.resultField)) @@ -928,7 +927,6 @@ func (p *context) compileCoroWorkerCgoTransaction( []llssa.Expr{b.Convert(p.prog.Uintptr(), record)}, keepaliveSlots, nil, - false, ) b.KeepAlive(record) if shape.result == nil { diff --git a/cl/coro_worker_foreign.go b/cl/coro_worker_foreign.go index c3c57d5c8c..e4af115507 100644 --- a/cl/coro_worker_foreign.go +++ b/cl/coro_worker_foreign.go @@ -1270,7 +1270,6 @@ func (p *context) compileCoroWorkerForeignTransaction( []llssa.Expr{b.Convert(p.prog.Uintptr(), record)}, keepaliveSlots, nil, - false, ) // The native queue carries the record address as an opaque uintptr. This // post-acknowledgement use forces CoroSplit to retain the complete typed diff --git a/cl/coro_worker_syscall_capability_test.go b/cl/coro_worker_syscall_capability_test.go index f5fbfbf7b3..34f50c49af 100644 --- a/cl/coro_worker_syscall_capability_test.go +++ b/cl/coro_worker_syscall_capability_test.go @@ -163,7 +163,7 @@ func TestCoroWorkerSyscallFunctionWordCapabilityIsFailClosed(t *testing.T) { } wantSemantics := CoroIntrinsicCallUnsupported if wantCertified[name] { - wantSemantics = CoroIntrinsicCallInlineSuspend + wantSemantics = CoroIntrinsicCallInlineNativeBlock if certificate.ID == "" || certificate.WorkerABISignature == "" || certificate.PhysicalTargetSetID == "" || certificate.CallableShadowSetID == "" || certificate.StaticTargetCount != 1 { diff --git a/cl/coro_worker_test.go b/cl/coro_worker_test.go index bf13734d8f..ab362f436c 100644 --- a/cl/coro_worker_test.go +++ b/cl/coro_worker_test.go @@ -83,6 +83,42 @@ func Root(fn, a0 uintptr, f0 float64) (uintptr, uintptr, uintptr) { } ` +const coroWorkerStaticOutcomeChainTestSource = `package chain + +import _ "unsafe" + +//go:linkname raw llgo.syscall +func raw(fn, a0 uintptr) (uintptr, uintptr, uintptr) + +//go:linkname funcPCABI0 llgo.funcPCABI0 +func funcPCABI0(fn any) uintptr + +//llgo:coro workeraddr 1 +func libc_worker1_v1_trampoline() + +func Leaf(a0 uintptr) uintptr { + r1, _, _ := raw(funcPCABI0(libc_worker1_v1_trampoline), a0) + return r1 +} + +func Middle(a0 uintptr) uintptr { + return Leaf(a0) + 1 +} + +func Top(a0 uintptr) uintptr { + return Middle(a0) + 2 +} + +// Plain is deliberately outside the managed root closure. Its call still has +// a globally frozen native-block SitePlan, but ordinary emission must not use +// the hidden managed-task ABI. +func Plain(a0 uintptr) uintptr { + r1, _, _ := raw(funcPCABI0(libc_worker1_v1_trampoline), a0) + return r1 +} + +` + func TestCoroTypedSyncSyscallDoesNotForgeWordWorkerABI(t *testing.T) { ssaPkg, _, _ := buildGoSSAPkg(t, coroWorkerTypedSyncSyscallTestSource) root := ssaPkg.Func("Root") @@ -120,8 +156,9 @@ func TestCoroWorkerSyscallCurrentFrame(t *testing.T) { rootPlan, ok := plan.FunctionPlan(root) if !ok || rootPlan.Emission != coro.EmitCoroutine || rootPlan.FuncRep != coro.DirectCoro || rootPlan.Demand != coro.AsyncDemand || !rootPlan.Effect.Contains(coro.MayPark) || - !rootPlan.LocalEffect.Contains(coro.MayPark) { - t.Fatalf("Root plan = %+v, present=%t; want one local may-park coroutine", rootPlan, ok) + !rootPlan.LocalEffect.Contains(coro.MayPark) || !rootPlan.StaticOutcome || + !rootPlan.HasStaticOutcome() { + t.Fatalf("Root plan = %+v, present=%t; want one local may-park coroutine plus static outcome twin", rootPlan, ok) } if !plan.ElidesCall(rawCall) { t.Fatal("llgo.syscall declaration call is not frozen as a frontend-elided worker site") @@ -170,12 +207,37 @@ func TestCoroWorkerSyscallCurrentFrame(t *testing.T) { t.Fatalf("Root native syscall switch lacks status %d:\n%s", status, dispatch[0]) } } + outcome := module.NamedFunction("foo.Root$outcome") + if outcome.IsNil() { + t.Fatalf("Root static outcome twin is missing:\n%s", module.String()) + } + outcomeBody := outcome.String() + if got := strings.Count(outcomeBody, "@"+coroNativeSyscallCallHookV1); got != 1 { + t.Fatalf("Root outcome native syscall calls = %d, want 1:\n%s", got, outcomeBody) + } + for _, forbidden := range []string{ + "llvm.coro.", + coroFrameAllocHookV1, + coroWorkerParkHookV1, + coroWorkerResumeHookV1, + coroAwaitPrepareInlineHookV4, + coroAwaitConsumeHookV1, + } { + if strings.Contains(outcomeBody, forbidden) { + t.Fatalf("Root outcome retained coroutine/worker operation %q:\n%s", forbidden, outcomeBody) + } + } runCoroABITestPipeline(t, prog, module) resumeBody := module.NamedFunction("foo.Root$coro.resume") if resumeBody.IsNil() || !strings.Contains(resumeBody.String(), "call i32 @"+coroNativeSyscallCallHookV1) { t.Fatalf("CoroSplit lost native syscall dispatch:\n%s", module.String()) } + postSplitOutcome := module.NamedFunction("foo.Root$outcome") + if postSplitOutcome.IsNil() || strings.Contains(postSplitOutcome.String(), "llvm.coro.") || + !strings.Contains(postSplitOutcome.String(), "call i32 @"+coroNativeSyscallCallHookV1) { + t.Fatalf("CoroSplit changed the synchronous native syscall outcome:\n%s", module.String()) + } object, err := prog.TargetMachine().EmitToMemoryBuffer(module, llvm.ObjectFile) if err != nil { t.Fatalf("emit post-CoroSplit worker object: %v\n%s", err, module.String()) @@ -253,6 +315,62 @@ func TestCoroWorkerSyscallFailureConventionsShareLowering(t *testing.T) { } } +func TestCoroNativeSyscallStaticOutcomePropagatesThroughExactCallChain(t *testing.T) { + llssa.Initialize(llssa.InitAll) + compiled := compileCoroWorkerSourceFixture( + t, coroWorkerStaticOutcomeChainTestSource, []string{"Top"}, []string{"Plain"}, + ) + defer compiled.prog.Dispose() + module := compiled.pkg.Module() + defer module.Dispose() + + topPlan, found := compiled.plan.FunctionPlan(compiled.roots["Top"]) + if !found || topPlan.Emission != coro.EmitCoroutine || !topPlan.StaticOutcome || + !topPlan.HasStaticOutcome() || !topPlan.Effect.Contains(coro.MayPark) { + t.Fatalf("Top plan = %+v, present=%t; want propagated native-block static outcome", topPlan, found) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify native syscall outcome chain before CoroSplit: %v\n%s", err, module.String()) + } + for _, edge := range []struct { + caller string + callee string + }{ + {caller: "chain.Top", callee: "chain.Middle$outcome"}, + {caller: "chain.Middle", callee: "chain.Leaf$outcome"}, + } { + coroutineBody := requireCoroPhysicalFunction(t, module, edge.caller).String() + if !strings.Contains(coroutineBody, edge.callee) || + strings.Contains(coroutineBody, coroAwaitPrepareInlineHookV4) || + strings.Contains(coroutineBody, coroAwaitConsumeHookV1) { + t.Fatalf("%s coroutine did not flatten exact call through %s:\n%s", edge.caller, edge.callee, coroutineBody) + } + outcomeBody := module.NamedFunction(edge.caller + "$outcome") + if outcomeBody.IsNil() || !strings.Contains(outcomeBody.String(), edge.callee) || + strings.Contains(outcomeBody.String(), "llvm.coro.") { + t.Fatalf("%s outcome did not flatten exact call through %s:\n%s", edge.caller, edge.callee, module.String()) + } + } + leafOutcome := module.NamedFunction("chain.Leaf$outcome") + if leafOutcome.IsNil() || strings.Count(leafOutcome.String(), "@"+coroNativeSyscallCallHookV1) != 1 || + strings.Contains(leafOutcome.String(), "llvm.coro.") { + t.Fatalf("Leaf has no synchronous native syscall outcome:\n%s", module.String()) + } + plain := module.NamedFunction("chain.Plain") + if plain.IsNil() || strings.Contains(plain.String(), coroNativeSyscallCallHookV1) || + strings.Contains(plain.String(), "llvm.coro.") { + t.Fatalf("unmanaged Plain selected the hidden managed syscall ABI:\n%s", module.String()) + } + + runCoroABITestPipeline(t, compiled.prog, module) + for _, name := range []string{"chain.Top$outcome", "chain.Middle$outcome", "chain.Leaf$outcome"} { + outcome := module.NamedFunction(name) + if outcome.IsNil() || strings.Contains(outcome.String(), "llvm.coro.") { + t.Fatalf("post-CoroSplit static chain lost %s:\n%s", name, module.String()) + } + } +} + func TestCoroWorkerSyscallFailureConventionIdentityIsFrozen(t *testing.T) { ssaPkg, _, files := buildGoSSAPkg(t, coroWorkerTestSource) prog := newLLSSAProg(t) @@ -420,8 +538,12 @@ func compileCoroWorkerFixture(t *testing.T) ( ) { t.Helper() compiled := compileCoroWorkerSourceFixture( - t, coroWorkerTestSource, []string{"Root", "RootInt32", "RootPointer"}, + t, coroWorkerTestSource, []string{"Root", "RootInt32", "RootPointer"}, nil, ) + if compiled.calls["Root"] == nil { + compiled.prog.Dispose() + t.Fatal("worker fixture Root has no direct llgo.syscall call") + } return compiled.prog, compiled.pkg, compiled.plan, compiled.roots["Root"], compiled.calls["Root"] } @@ -429,6 +551,7 @@ func compileCoroWorkerSourceFixture( t *testing.T, source string, rootNames []string, + rawRootNames []string, ) compiledCoroWorkerFixture { t.Helper() ssaPkg, _, files := buildGoSSAPkg(t, source) @@ -466,19 +589,34 @@ func compileCoroWorkerSourceFixture( prog.Dispose() t.Fatal(err) } - rootsByName := make(map[string]*ssa.Function, len(rootNames)) - callsByName := make(map[string]*ssa.Call, len(rootNames)) - analysisRoots := make(coro.Roots, 0, len(rootNames)) - rootSet := make(map[*ssa.Function]bool, len(rootNames)) + type rootSpec struct { + name string + root coro.Root + } + rootSpecs := make([]rootSpec, 0, len(rootNames)+len(rawRootNames)) for _, name := range rootNames { + rootSpecs = append(rootSpecs, rootSpec{name: name, root: coro.Root{Demand: coro.AsyncDemand}}) + } + for _, name := range rawRootNames { + rootSpecs = append(rootSpecs, rootSpec{name: name, root: coro.Root{RawPlainDemand: true}}) + } + rootsByName := make(map[string]*ssa.Function, len(rootSpecs)) + callsByName := make(map[string]*ssa.Call, len(rootSpecs)) + rawRootFunctions := make(map[*ssa.Function]bool, len(rawRootNames)) + analysisRoots := make(coro.Roots, 0, len(rootSpecs)) + for _, spec := range rootSpecs { + name := spec.name root := ssaPkg.Func(name) if root == nil { prog.Dispose() t.Fatalf("worker fixture lacks root %q", name) } rootsByName[name] = root - rootSet[root] = true - analysisRoots = append(analysisRoots, coro.Root{Function: root, Demand: coro.AsyncDemand}) + if spec.root.RawPlainDemand { + rawRootFunctions[root] = true + } + spec.root.Function = root + analysisRoots = append(analysisRoots, spec.root) for _, block := range root.Blocks { for _, instruction := range block.Instrs { call, ok := instruction.(*ssa.Call) @@ -499,9 +637,24 @@ func compileCoroWorkerSourceFixture( } } } - if callsByName[name] == nil { - prog.Dispose() - t.Fatalf("worker fixture root %q has no direct llgo.syscall call", name) + } + intrinsicEffects := make(map[*ssa.Function]coro.Effect) + for _, function := range universe.Functions() { + for _, block := range function.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(*ssa.Call) + if !ok { + continue + } + semantics, intrinsic, semanticsErr := universe.CoroIntrinsicCallSiteSemantics(call) + if semanticsErr != nil { + prog.Dispose() + t.Fatalf("classify %s intrinsic effect: %v", function.String(), semanticsErr) + } + if intrinsic && semantics.SuspendsCurrentFrame() { + intrinsicEffects[function] = intrinsicEffects[function].Join(semantics.CurrentFrameEffect()) + } + } } } functionIDs := universe.FunctionIDConfig() @@ -512,14 +665,16 @@ func compileCoroWorkerSourceFixture( EmissionUniverse: ssaUniverse, FunctionIDs: functionIDs, MaxPlainInstructions: -1, + OutcomeMode: coro.OutcomeExplicitStatus, + ClassifyLocalBody: universe.CoroLocalBodyFacts, ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { - if rootSet[fn] { - // Production builds seed this fact through - // CoroPlanInput.intrinsicCallSemantics. This isolated cl fixture - // has no build-driver wrapper, so freeze the same owner effect. - return coro.SSAFunctionPolicy{Effect: coro.MayPark}, nil - } - return coro.SSAFunctionPolicy{}, nil + // Production builds seed owner-local intrinsic effects from the same + // frozen call SitePlans. Reproduce that bridge without assuming every + // externally demanded root contains the syscall itself. + return coro.SSAFunctionPolicy{ + Effect: intrinsicEffects[fn], + RawPlainEntry: rawRootFunctions[fn], + }, nil }, ClassifyElidedCall: func(_ *ssa.Function, call ssa.CallInstruction) (bool, error) { callee := call.Common().StaticCallee() diff --git a/cl/emission_universe.go b/cl/emission_universe.go index 2f3900d97f..7fe3fcdb96 100644 --- a/cl/emission_universe.go +++ b/cl/emission_universe.go @@ -418,6 +418,14 @@ const ( // The build analyzer seeds the owner with MayPark; there is no callable sync // helper and no managed callee edge. CoroIntrinsicCallInlineSuspend + // CoroIntrinsicCallInlineNativeBlock means cl erases one exact certified + // llgo.syscall declaration call and executes it synchronously on the current + // native M after releasing the managed execution domain. The operation may + // block the M, so its owner retains the conservative MayPark managed effect, + // but it never parks or resumes the current LLVM coroutine. This target- + // specific dual recipe can therefore also execute in a synchronous outcome + // twin while timer, poll, host, and generic worker waits cannot. + CoroIntrinsicCallInlineNativeBlock // CoroIntrinsicCallInlineForeignSuspend is the same current-frame erasure // shape, but the structured suspension waits for a bounded foreign-worker // transaction rather than a scheduler-local event. Keeping it distinct @@ -440,17 +448,21 @@ const ( // through the owner's exact frozen lowered-call set. func (s CoroIntrinsicCallSemantics) ElidesManagedCall() bool { return s == CoroIntrinsicCallInlineNoSuspend || s == CoroIntrinsicCallInlineWithLoweredCalls || - s == CoroIntrinsicCallInlineSuspend || s == CoroIntrinsicCallInlineForeignSuspend || + s == CoroIntrinsicCallInlineSuspend || s == CoroIntrinsicCallInlineNativeBlock || + s == CoroIntrinsicCallInlineForeignSuspend || s == CoroIntrinsicCallInlineYield || s == CoroIntrinsicCallInlineOutcome } -// SuspendsCurrentFrame reports whether an intrinsic requires its owner to have -// a coroutine primary even though the declaration call itself is erased by -// frontend lowering. Terminal structured outcomes share this physical -// requirement even though they do not have a resumable continuation. +// SuspendsCurrentFrame is the legacy name for the owner-local structured-effect +// predicate. It reports whether an erased intrinsic requires a managed +// structured body rather than an ordinary plain body. A native blocking call +// and terminal outcome do not literally suspend a resumable LLVM frame, but +// they still require the hidden task/completion ABI and therefore participate +// in the same analyzer seed. func (s CoroIntrinsicCallSemantics) SuspendsCurrentFrame() bool { - return s == CoroIntrinsicCallInlineSuspend || s == CoroIntrinsicCallInlineForeignSuspend || + return s == CoroIntrinsicCallInlineSuspend || s == CoroIntrinsicCallInlineNativeBlock || + s == CoroIntrinsicCallInlineForeignSuspend || s == CoroIntrinsicCallInlineYield || s == CoroIntrinsicCallInlineOutcome @@ -471,7 +483,7 @@ const ( // coroutine effect. func (s CoroIntrinsicCallSemantics) CurrentFrameEffect() coro.Effect { switch s { - case CoroIntrinsicCallInlineSuspend: + case CoroIntrinsicCallInlineSuspend, CoroIntrinsicCallInlineNativeBlock: return coro.MayPark case CoroIntrinsicCallInlineForeignSuspend: return coro.WaitForeign @@ -1910,6 +1922,9 @@ func (u *EmissionUniverse) classifyCoroIntrinsicCallSite( // physical preflight instead of submitting an arbitrary uintptr. return CoroIntrinsicCallUnsupported, true, nil } + if u.coroCapabilities.NativeFleet() { + return CoroIntrinsicCallInlineNativeBlock, true, nil + } return CoroIntrinsicCallInlineSuspend, true, nil } semantics = coroIntrinsicCallSemantics(opcode) diff --git a/cl/instr.go b/cl/instr.go index 96c7e65574..9a2f781c40 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -2725,20 +2725,52 @@ func (p *context) callEx( if !ok { panic("unknown coroutine llgo.syscall failure convention") } + semantics := CoroIntrinsicCallUnsupported + semanticsPlanned := false + if sourceCall != nil && p.emissionUniverse != nil { + var semanticsErr error + semantics, semanticsPlanned, semanticsErr = coroIntrinsicCallSiteSemantics( + p.emissionUniverse, sourceCall, + ) + if semanticsErr != nil { + panic(fmt.Errorf("coroutine llgo.syscall semantics: %w", semanticsErr)) + } + } + expectedOperation := coroPhysicalOperationWorkerSyscall + if semantics == CoroIntrinsicCallInlineNativeBlock { + expectedOperation = coroPhysicalOperationNativeSyscall + } _, managedWorker, operationPlanned := p.selectCoroPhysicalOperation( - sourceCall, coroPhysicalOperationWorkerSyscall, + sourceCall, expectedOperation, ) if !managedWorker && !operationPlanned { - if semantics, planned := p.plannedCoroIntrinsicCall(ftype); planned { - managedWorker = semantics == CoroIntrinsicCallInlineSuspend + // A frozen call-site classification describes what this call + // means if its owner is emitted through the managed physical ABI; + // it is not by itself authority to use the hidden task ABI. A + // simultaneously emitted ordinary/raw twin has the same global + // SitePlan but no managed task. Only the active source-site + // observer can select the managed fallback when no physical + // operation record is installed. + if activeSemantics, active := p.plannedCoroIntrinsicCall(ftype); active { + semantics = activeSemantics + semanticsPlanned = true + managedWorker = semantics == CoroIntrinsicCallInlineSuspend || + semantics == CoroIntrinsicCallInlineNativeBlock } } if managedWorker { - if act != llssa.Call || ds != nil { + if !semanticsPlanned || + (semantics != CoroIntrinsicCallInlineSuspend && + semantics != CoroIntrinsicCallInlineNativeBlock) { + panic("coroutine llgo.syscall operation has incompatible frozen semantics") + } + if act != llssa.Call || ds != nil || sourceCall == nil { panic("coroutine llgo.syscall requires an exact direct call") } - ret = p.compileCoroWorkerSyscall(b, call, args, call.Signature().Results(), convention) - p.completeCoroIntrinsicCallEmission(ftype, CoroIntrinsicCallInlineSuspend) + ret = p.compileCoroSyscallOperation( + b, call, args, call.Signature().Results(), convention, semantics, + ) + p.completeCoroIntrinsicCallEmission(ftype, semantics) } else { ret = p.syscallIntrinsic(b, args, call.Signature().Results(), convention) p.completeCoroIntrinsicCallEmission(ftype, CoroIntrinsicCallUnsupported) diff --git a/doc/coro-performance-baseline.md b/doc/coro-performance-baseline.md index 6ee94bb7ff..526725b79c 100644 --- a/doc/coro-performance-baseline.md +++ b/doc/coro-performance-baseline.md @@ -2021,3 +2021,94 @@ remaining standard-file gap primarily in synchronous child coroutine allocation, frame publication, inline-await completion, and destroy/consume transactions through `syscall`, `internal/poll`, `os.File`, and `io`, rather than in replacement-slot or full driver-route recovery. + +### Certified native static-outcome chain checkpoint + +The 2026-08-15 follow-up starts at merge `db6a6eb2e`; the measured +implementation is `3fa2cfc44`. It completes the compiler-owned static outcome +chain for a certified native syscall. A native call whose contract proves a +bounded synchronous result now releases the managed execution domain, performs +the call on the current native M, and reacquires the domain without allocating +or parking an LLVM coroutine. The physical M may block in the kernel, but a +durable runnable/event demand can still start a replacement M, so the managed +scheduler is not blocked. Network sockets retain their nonblocking poll/event +path. + +ProgramIR records this as `cl.intrinsic.inline-native-block.v1`. Exact static +Go callers consume the generated `$outcome` entry point recursively; dynamic +function values and interface calls retain the ordinary `$coro` entry point. +Library-effect summaries carry the no-unwind static-outcome proof across package +boundaries. Timer, poll, host, foreign/generic worker waits, yield-only effects, +retained-memory calls, and unresolved contracts fail closed. The production +Darwin gate verifies the real `syscall.syscall`, `syscall.{Read,Write,Seek}` and +the private write wrapper rather than a synthetic fixture. + +The parent and candidate use Go 1.26.5, LLVM 22.1.8, full LTO, stripped Darwin +arm64 outputs, independent build caches, and process-start `GOMAXPROCS=1`. +Fifteen AB/BA-interleaved process runs gave the following wall-time medians; +brackets are the first and third quartiles: + +| Workload | Go median [Q1, Q3] | LLGo median [Q1, Q3] | LLGo / Go | previous LLGo / Go | +| --- | ---: | ---: | ---: | ---: | +| direct `syscall` file round trip, 5,000 operations | 7.763 ms [7.726, 7.929] | 11.591 ms [11.511, 11.698] | 1.49x | 2.07x | +| cache-hot 4 KiB standard file round trip, 5,000 operations | 7.909 ms [7.818, 8.117] | 30.736 ms [30.410, 30.938] | 3.89x | 4.94x | +| loopback TCP echo, 500 operations | 8.714 ms [8.409, 9.263] | 17.722 ms [17.210, 18.038] | 2.03x | 2.04x | + +Relative to the preceding equal-count checkpoint, direct syscalls improve +26.8% and the standard-file path improves 19.9%. TCP remains effectively +unchanged: it does not use the newly flattened blocking-native path, and the +shared-host distributions do not support a network speedup claim. The +standard-file gap is now above the native leaf, in the remaining +`internal/poll`, `os.File`, and `io` coroutine/transaction layers. + +Eleven AB/BA-interleaved core runs on the same binaries show that the static +compiler path is already competitive, while goroutine creation is the largest +core latency gap: + +| Core workload | Go median | LLGo median | LLGo / Go | +| --- | ---: | ---: | ---: | +| scalar compute, 5,000,000 iterations | 8.243 ms | 7.752 ms | 0.94x | +| buffered channel, 1,000,000 operations | 16.108 ms | 20.154 ms | 1.25x | +| ready `select`, 500,000 operations | 26.295 ms | 21.268 ms | 0.81x | +| spawn 100 x 100 goroutines | 0.964 ms | 3.007 ms | 3.12x | +| unbuffered handoff, 5,000 operations | 0.798 ms | 0.660 ms | 0.83x | +| timers, 100 x 10 expirations | 12.908 ms | 14.761 ms | 1.14x | + +A four-worker compute fixture scales from one to four Ps by 3.83x in Go and +3.85x in LLGo, but LLGo remains 2.22x slower in absolute time (14.545 versus +6.543 ms at P=1; 3.780 versus 1.707 ms at P=4). This isolates the remaining +cost in goroutine entry/completion and channel coordination rather than a +failure of parallel execution. + +Nine `/usr/bin/time -l` AB/BA runs validate the stackless memory advantage at +high concurrency: + +| Parked goroutines | Go median max RSS | LLGo median max RSS | +| ---: | ---: | ---: | +| 0 | 3,522,560 B | 6,176,768 B | +| 1,000 | 6,471,680 B | 8,372,224 B | +| 5,000 | 17,580,032 B | 16,056,320 B | +| 10,000 | 31,522,816 B | 25,657,344 B | + +From 0 to 10,000 parked goroutines, Go grows by about 2,800 B per goroutine and +LLGo by about 1,948 B, a 30.4% lower LLGo incremental slope. LLGo has a roughly +2.65 MiB higher fixed process cost, but is 1.52 MiB smaller at 5,000 parked +goroutines and 5.87 MiB smaller at 10,000. + +Code size remains a major unresolved cost. The core executable is 4,093,696 B +for LLGo versus 1,488,034 B for Go (2.75x), and its Mach-O `__text` is +2,046,804 B versus 582,500 B (3.51x). The I/O executable is 7,085,200 B versus +1,797,874 B (3.94x), with `__text` at 3,524,144 B versus 696,708 B (5.06x). +Within LLGo, however, this change removes 86,444 text bytes (-2.39%) and +111,760 file bytes (-1.55%); `coro_resume` symbols fall from 1,859 to 1,792 +while `$outcome` symbols rise from 503 to 551. + +Semantic gates pass for effect/ProgramIR propagation, raw ABI, exact frame +retention, timer/poll/worker negative cases, the runtime coroutine module, and +actual standard-file, direct-syscall, sole-M blocking-pipe-with-timer, and TCP +execution. This is not yet the final compatibility checkpoint: at measurement +time `cpunion/llvm-coro` is 101 `xgo-dev/main` commits behind (merge base +`c9515d8c`, upstream tip `8b76e388`). The next integration gate is to merge +this focused change, synchronize the branch with current upstream main, run all +repository and selected GOROOT tests, and repeat the performance measurements +on the synchronized tree. diff --git a/internal/build/build.go b/internal/build/build.go index 0ee099b983..bc842339e4 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -2151,7 +2151,7 @@ func (in CoroPlanInput) liveCoroRawABIPlainClosure( if callSite.ElidesCall() || callSite.ElisionCertificate != "" { return nil, fmt.Errorf("raw-plain synchronous call %q in %q has a malformed retained managed recipe", direct.String(), fn.Name()) } - case cl.CoroIntrinsicCallInlineSuspend: + case cl.CoroIntrinsicCallInlineSuspend, cl.CoroIntrinsicCallInlineNativeBlock: if callSite.Elision != cl.CoroCallElidedIntrinsic || callSite.ElisionCertificate == "" { return nil, fmt.Errorf("raw-plain synchronous call %q in %q has no exact worker elision certificate", direct.String(), fn.Name()) } @@ -2510,7 +2510,7 @@ func validateLiveCoroRawABIPlainClosure(plan *coro.SSAPlan, raw *coroRawABIPlain if _, planned := plan.CallPlan(call); !planned { return fmt.Errorf("live raw ABI plain closure function %q (%s) lost its retained managed fail-closed syscall edge", functionPlan.ID, fn.String()) } - case cl.CoroIntrinsicCallInlineSuspend: + case cl.CoroIntrinsicCallInlineSuspend, cl.CoroIntrinsicCallInlineNativeBlock: plannedCertificate, planned := plan.ElidedCallCertificate(call) if !planned || !plan.ElidesCall(call) || plannedCertificate != site.ElisionCertificate { return fmt.Errorf("live raw ABI plain closure function %q (%s) lost its exact worker syscall certificate", functionPlan.ID, fn.String()) diff --git a/internal/build/coro_architecture_gate_test.go b/internal/build/coro_architecture_gate_test.go index 28e76d2043..2a7563fa76 100644 --- a/internal/build/coro_architecture_gate_test.go +++ b/internal/build/coro_architecture_gate_test.go @@ -147,7 +147,7 @@ var currentCoroArchitectureDebtBudget = coroArchitectureDebtBudget{ legacyPhysicalSelector: 0, legacySplitEmissionState: 0, emissionSessionAccess: 20, - bodyCapabilityAccess: 33, + bodyCapabilityAccess: 32, // The outcome-plain cohort replaced the coroutine-only begin/bind/complete // entry points with one exclusive managed-body transaction. Keep the legacy // names at zero so a second physical-emission lifecycle cannot grow back. diff --git a/internal/build/coro_panic_native_e2e_test.go b/internal/build/coro_panic_native_e2e_test.go index fd362d17bc..186da79984 100644 --- a/internal/build/coro_panic_native_e2e_test.go +++ b/internal/build/coro_panic_native_e2e_test.go @@ -200,7 +200,7 @@ func buildCoroPanicNativeE2EUser(t *testing.T, prog llssa.Program, temp string) ClassifyLoweredCalls: universe.CoroLoweredCalls, ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { switch fn { - case mainFn, childFn: + case mainFn: return coro.SSAFunctionPolicy{Effect: coro.YieldOnly}, nil default: return coro.SSAFunctionPolicy{}, nil @@ -222,6 +222,10 @@ func buildCoroPanicNativeE2EUser(t *testing.T, prog llssa.Program, temp string) middlePlan.AtomicCostProof != coro.AtomicCostDAG || middlePlan.AtomicCost <= leafPlan.AtomicCost { t.Fatalf("panic middle plan = %+v, present=%t; want outcome-plain DAG above leaf cost %d", middlePlan, found, leafPlan.AtomicCost) } + childPlan, found := plan.FunctionPlan(childFn) + if !found || !childPlan.HasStaticOutcome() || childPlan.Effect.Contains(coro.YieldOnly) { + t.Fatalf("panic child plan = %+v, present=%t; want synchronous outcome without a forged yield effect", childPlan, found) + } compilation := &cl.Compilation{ CoroPlan: plan, diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index 1f1f9e2839..0e368553be 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -3289,7 +3289,7 @@ func TestCoroUnsupportedEntryResolutionReturnsErrorBeforeCodegen(t *testing.T) { t.Fatalf("CoroPlanBuilder did not successfully return a plan: %v", err) } if err == nil || - !strings.Contains(err.Error(), "declared may-park effect has no exact structured park intrinsic") { + !strings.Contains(err.Error(), "declared may-park effect has no exact structured park or native blocking intrinsic") { t.Fatalf("Do error = %v, want exact coroutine physical-ABI rejection before codegen", err) } if len(pkgs) != 0 { diff --git a/internal/build/coro_static_outcome_stdlib_test.go b/internal/build/coro_static_outcome_stdlib_test.go index ab7496dfc2..3438f2491a 100644 --- a/internal/build/coro_static_outcome_stdlib_test.go +++ b/internal/build/coro_static_outcome_stdlib_test.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" "testing" @@ -361,3 +362,106 @@ func main() { t.Logf("%s: emission=%s static=%t declared=%s local=%s effect=%s declared-exec=%s local-exec=%s exec=%s proof=%s cost=%d facts=%+v trace=%s", name, plan.Emission, plan.StaticOutcome, plan.DeclaredEffect, plan.LocalEffect, plan.Effect, plan.DeclaredExec, plan.LocalExec, plan.Exec, plan.AtomicCostProof, plan.AtomicCost, frozenFacts[name], traces[name]) } } + +func TestCoroDarwinSyscallWrappersPublishStaticOutcomeChain(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("Darwin fixed-target syscall patch coverage") + } + t.Setenv(llgoBuildCache, "off") + source := filepath.Join(t.TempDir(), "main.go") + if err := os.WriteFile(source, []byte(`package main + +import "syscall" + +var buffer [1]byte + +func main() { + _, _ = syscall.Seek(0, 0, 0) + _, _ = syscall.Write(1, buffer[:]) + _, _ = syscall.Read(0, buffer[:]) +} +`), 0o644); err != nil { + t.Fatal(err) + } + + wanted := map[string]bool{ + "syscall": true, + "write": true, + "Write": true, + "Read": true, + "Seek": true, + } + plans := make(map[string]coro.FunctionPlan, len(wanted)) + facts := make(map[string]coro.SSAFunctionBodyFacts, len(wanted)) + definitions := make(map[string]bool, len(wanted)) + + conf := NewDefaultConf(ModeGen) + conf.ForceRebuild = true + conf.CoroPlanBuilder = func(input CoroPlanInput) (*coro.SSAPlan, error) { + plan, err := defaultCoroPlanBuilder(input) + if err != nil { + return nil, err + } + for _, fn := range input.EmissionUniverse.Functions() { + if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil || !wanted[fn.Name()] { + continue + } + path := fn.Pkg.Pkg.Path() + if path != "syscall" && !(fn.Name() == "syscall" && + strings.HasSuffix(path, "/runtime/internal/lib/syscall")) { + continue + } + functionPlan, found := plan.FunctionPlan(fn) + if !found { + return nil, fmt.Errorf("syscall.%s has no whole-program plan", fn.Name()) + } + bodyFacts, err := input.localBodyFacts(fn) + if err != nil { + return nil, fmt.Errorf("syscall.%s local body facts: %w", fn.Name(), err) + } + if previous, duplicate := plans[fn.Name()]; duplicate && previous.ID != functionPlan.ID { + return nil, fmt.Errorf("syscall.%s has multiple canonical plans", fn.Name()) + } + plans[fn.Name()] = functionPlan + facts[fn.Name()] = bodyFacts + } + return plan, nil + } + conf.ModuleHook = func(pkg Package) { + if pkg.PkgPath != "syscall" { + return + } + for name := range wanted { + function := pkg.LPkg.FuncOf("syscall." + name + "$outcome") + definitions[name] = function != nil && function.HasBody() + } + } + if _, err := Do([]string{source}, conf); err != nil { + t.Fatalf("compile Darwin syscall static-outcome fixture: %v", err) + } + + for name := range wanted { + plan, found := plans[name] + if !found { + t.Errorf("syscall.%s is absent from the whole-program plan", name) + continue + } + if plan.Emission != coro.EmitCoroutine || plan.ManagedEntry != coro.ManagedEntryCoroutine || + !plan.Effect.Contains(coro.MayPark) || !plan.StaticOutcome || !plan.HasStaticOutcome() { + t.Errorf("syscall.%s plan = %+v, want coroutine primary plus static outcome twin", name, plan) + } + if !facts[name].StaticOutcomeLocal { + t.Errorf("syscall.%s local facts = %+v, want closed static-outcome vocabulary", name, facts[name]) + } + if !definitions[name] { + t.Errorf("syscall.%s$outcome has no emitted definition", name) + } + } + bottom, found := plans["syscall"] + if !found { + return + } + if bottom.Exec.Contains(coro.MayUnwind) || bottom.Effect.Contains(coro.OutcomeStructured) { + t.Fatalf("syscall.syscall plan = %+v, fixture no longer covers the no-unwind native-block case", bottom) + } +} diff --git a/internal/coro/graph.go b/internal/coro/graph.go index af053ee8d5..ae628835fc 100644 --- a/internal/coro/graph.go +++ b/internal/coro/graph.go @@ -241,8 +241,7 @@ func (g *Graph) AddFunction(spec FunctionSpec) error { if spec.StaticOutcome { if spec.External != ExternalKnown || spec.ManagedEntry != ManagedEntryCoroutine || spec.AtomicCostProof != AtomicCostUnproven || spec.AtomicCost != 0 || spec.AtomicCostCertificate != "" || - spec.Seed&^(YieldOnly|AwaitStructured|OutcomeStructured) != 0 || - !spec.Seed.Contains(OutcomeStructured) || + spec.Seed&^(AwaitStructured|OutcomeStructured|MayPark) != 0 || spec.Exec&(BlockForeign|ThreadAffine|NeedsCleanupFrame|OpaqueExec) != 0 { return fmt.Errorf("coro: function %q: invalid imported unbounded static outcome capability", spec.ID) } diff --git a/internal/coro/library_effect_summary.go b/internal/coro/library_effect_summary.go index 802403f25c..ada9739b33 100644 --- a/internal/coro/library_effect_summary.go +++ b/internal/coro/library_effect_summary.go @@ -368,8 +368,7 @@ func (function LibraryEffectFunction) validate() error { if function.StaticOutcome { if function.AtomicCostProof.ProvesOutcomePlain() || function.ManagedEntry != ManagedEntryCoroutine || function.Primary != PrimaryCoroutine || - function.Effect&^(YieldOnly|AwaitStructured|OutcomeStructured) != 0 || - !function.Effect.Contains(OutcomeStructured) || + function.Effect&^(AwaitStructured|OutcomeStructured|MayPark) != 0 || function.Exec&(BlockForeign|ThreadAffine|NeedsCleanupFrame|OpaqueExec) != 0 { return fmt.Errorf("coro: library function %q has an invalid unbounded static outcome capability", function.ID) } diff --git a/internal/coro/library_effect_summary_analysis_test.go b/internal/coro/library_effect_summary_analysis_test.go index 75c46ddbd7..e74f0070e7 100644 --- a/internal/coro/library_effect_summary_analysis_test.go +++ b/internal/coro/library_effect_summary_analysis_test.go @@ -83,3 +83,71 @@ func caller() { imported() } t.Fatalf("caller was not automatically colored from library summary: %+v", callerPlan) } } + +func TestLibraryEffectSummaryPropagatesNoUnwindStaticOutcome(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "library_static_outcome.go", `package coroid +func imported() +func caller() { imported() } +`) + imported := packageFunction(t, pkg, "imported") + caller := packageFunction(t, pkg, "caller") + functionIDs := FunctionIDConfig{} + importedID, err := StableFunctionID(imported, functionIDs) + if err != nil { + t.Fatal(err) + } + summary := testLibraryEffectSummary(t, "example/static", false) + summary.Functions = []LibraryEffectFunction{{ + ID: importedID, + ABIHash: strings.Repeat("4", 64), + Effect: MayPark, + FuncRep: DirectCoro, + Primary: PrimaryCoroutine, + ManagedEntry: ManagedEntryCoroutine, + StaticOutcome: true, + PrimarySymbol: "example/static.imported$coro", + OutcomePlainSymbol: "example/static.imported$outcome", + }} + summary.ForeignCallables = nil + summary.ExportBindings = nil + index, err := NewLibraryEffectIndex([]LibraryEffectSummary{summary}, testLibraryEffectMetadata()) + if err != nil { + t.Fatal(err) + } + plan, err := AnalyzeSSA(prog, Roots{{Function: caller, ManagedDemand: AsyncDemand}}, SSAConfig{ + FunctionIDs: functionIDs, + OutcomeMode: OutcomeExplicitStatus, + MaxPlainInstructions: -1, + ClassifyFunction: func(function *ssa.Function) (SSAFunctionPolicy, error) { + id, idErr := StableFunctionID(function, functionIDs) + if idErr != nil { + return SSAFunctionPolicy{}, idErr + } + fact, ok := index.Lookup(id) + if !ok { + return SSAFunctionPolicy{}, nil + } + return fact.ImportedPolicy() + }, + ClassifyLocalBody: func(function *ssa.Function) (SSAFunctionBodyFacts, error) { + facts := scanSSAFunctionBody(function) + if function == caller { + facts.StaticOutcomeLocal = true + } + return facts, nil + }, + }) + if err != nil { + t.Fatal(err) + } + importedPlan := functionPlanFor(t, plan, imported) + if importedPlan.External != ExternalKnown || importedPlan.Effect != MayPark || + !importedPlan.StaticOutcome || !importedPlan.HasStaticOutcome() { + t.Fatalf("imported no-unwind static outcome plan = %+v", importedPlan) + } + callerPlan := functionPlanFor(t, plan, caller) + if callerPlan.Emission != EmitCoroutine || !callerPlan.Effect.Contains(MayPark|AwaitStructured) || + !callerPlan.StaticOutcome || !callerPlan.HasStaticOutcome() { + t.Fatalf("caller did not inherit imported static outcome = %+v", callerPlan) + } +} diff --git a/internal/coro/library_effect_summary_test.go b/internal/coro/library_effect_summary_test.go index a027fe77a4..10c6f76952 100644 --- a/internal/coro/library_effect_summary_test.go +++ b/internal/coro/library_effect_summary_test.go @@ -238,6 +238,54 @@ func TestLibraryEffectSummaryCanonicalRecordAndImportPolicy(t *testing.T) { } } +func TestLibraryEffectSummaryPublishesNoUnwindStaticOutcome(t *testing.T) { + summary := testLibraryEffectSummary(t, "example/native", false) + function := LibraryEffectFunction{ + ID: "llgo.function.v0:native-block", + ABIHash: strings.Repeat("4", 64), + Effect: MayPark, + FuncRep: DirectCoro, + Primary: PrimaryCoroutine, + ManagedEntry: ManagedEntryCoroutine, + StaticOutcome: true, + PrimarySymbol: "example/native.Block$coro", + OutcomePlainSymbol: "example/native.Block$outcome", + } + summary.Functions = append(summary.Functions, function) + data, err := summary.MarshalStable() + if err != nil { + t.Fatal(err) + } + parsed, err := ParseLibraryEffectSummary(data) + if err != nil { + t.Fatal(err) + } + var imported LibraryEffectFunction + for _, candidate := range parsed.Functions { + if candidate.ID == function.ID { + imported = candidate + break + } + } + if !imported.StaticOutcome || imported.Effect != MayPark || imported.Exec != 0 { + t.Fatalf("no-unwind static outcome metadata = %+v", imported) + } + policy, err := imported.ImportedPolicy() + if err != nil { + t.Fatal(err) + } + if !policy.StaticOutcome || policy.Effect != MayPark || policy.Exec != 0 { + t.Fatalf("no-unwind static outcome import policy = %+v", policy) + } + invalid := summary + invalid.Functions = append([]LibraryEffectFunction(nil), summary.Functions...) + invalid.Functions[len(invalid.Functions)-1].Effect |= YieldOnly + if _, err := invalid.MarshalStable(); err == nil || + !strings.Contains(err.Error(), "invalid unbounded static outcome") { + t.Fatalf("yielding static outcome error = %v", err) + } +} + func TestLibraryEffectSummaryCarriesOutcomePlainCapability(t *testing.T) { summary := testLibraryEffectSummary(t, "example/outcome", false) summary.Functions = []LibraryEffectFunction{{ diff --git a/internal/coro/plan.go b/internal/coro/plan.go index e1f85c9543..4444b33e87 100644 --- a/internal/coro/plan.go +++ b/internal/coro/plan.go @@ -373,9 +373,12 @@ func validateManagedEntryPlan(plan FunctionPlan) error { if plan.AtomicCostProof.ProvesOutcomePlain() || plan.AtomicCost != 0 || plan.AtomicCostCertificate != "" { return fmt.Errorf("coro: function %q mixes bounded and unbounded static outcome capabilities", plan.ID) } + // The outcome twin is also useful for a no-unwind native block. Such a + // function always returns the success status, but exact static callers can + // still erase its coroutine frame. OutcomeStructured is therefore optional; + // functions which can unwind acquire it through the ordinary effect solver. if plan.Recursive || plan.Exec&(BlockForeign|ThreadAffine|NeedsCleanupFrame|OpaqueExec) != 0 || - plan.Effect&^(YieldOnly|AwaitStructured|OutcomeStructured) != 0 || - !plan.Effect.Contains(OutcomeStructured) { + plan.Effect&^(AwaitStructured|OutcomeStructured|MayPark) != 0 { return fmt.Errorf( "coro: function %q has an invalid unbounded static outcome capability (effect=%s exec=%s recursive=%t)", plan.ID, plan.Effect, plan.Exec, plan.Recursive, diff --git a/internal/coro/ssa_outcome_plain_test.go b/internal/coro/ssa_outcome_plain_test.go index 69a4e2691c..e145327d25 100644 --- a/internal/coro/ssa_outcome_plain_test.go +++ b/internal/coro/ssa_outcome_plain_test.go @@ -525,3 +525,46 @@ func caller(value any, fail bool) int { return leaf(value, fail) } }) } } + +func TestAnalyzeSSAStaticOutcomeAdmitsNoUnwindNativeBlock(t *testing.T) { + prog, pkg := buildCoroTestSSA(t, "static_outcome_native_block.go", `package coroid + +func nativeBlock() int { return 7 } +`) + nativeBlock := packageFunction(t, pkg, "nativeBlock") + config := planDigestSSAConfig() + config.OutcomeMode = OutcomeExplicitStatus + config.MaxPlainInstructions = -1 + config.ClassifyFunction = func(fn *ssa.Function) (SSAFunctionPolicy, error) { + if fn == nativeBlock { + return SSAFunctionPolicy{Effect: MayPark, TrustedNoUnwind: true}, nil + } + return SSAFunctionPolicy{}, nil + } + config.ClassifyLocalBody = func(fn *ssa.Function) (SSAFunctionBodyFacts, error) { + facts := scanSSAFunctionBody(fn) + if fn == nativeBlock { + // This is the analyzer projection of ProgramIR's certified native- + // block recipe. The physical planner independently verifies that the + // corresponding instruction is a direct native syscall operation. + facts.Effect = MayPark + facts.StaticOutcomeLocal = true + facts.OutcomePlainLeaf = false + facts.OutcomePlainDAG = false + } + return facts, nil + } + plan, err := AnalyzeSSA(prog, Roots{{Function: nativeBlock, ManagedDemand: AsyncDemand}}, config) + if err != nil { + t.Fatal(err) + } + got := functionPlanFor(t, plan, nativeBlock) + if got.Emission != EmitCoroutine || got.ManagedEntry != ManagedEntryCoroutine || + got.Effect != MayPark || got.Exec.Contains(MayUnwind) || !got.StaticOutcome || + !got.HasStaticOutcome() { + t.Fatalf("no-unwind native-block plan = %+v, want may-park coroutine plus static outcome twin", got) + } + if _, err := plan.CoroPlanDigest(validPlanDigestMetadata()); err != nil { + t.Fatalf("digest no-unwind native-block plan: %v", err) + } +} diff --git a/internal/coro/ssa_plan.go b/internal/coro/ssa_plan.go index 3f3c6fc2ae..a75f295789 100644 --- a/internal/coro/ssa_plan.go +++ b/internal/coro/ssa_plan.go @@ -2470,14 +2470,25 @@ func applySSAStaticOutcomePlans( for _, plan := range base.functions { function := byID[plan.ID] facts, classified := localBodyFacts[function] + // MayPark is admitted only when ProgramIR kept StaticOutcomeLocal true + // while projecting that exact local effect. Today the sole such recipe is + // a target-certified native syscall which releases the execution domain + // but returns synchronously on the same M. A timer, poll, channel, host, or + // generic worker wait clears StaticOutcomeLocal before analysis reaches + // this closure, even though it contributes the same aggregate effect bit. + // A proved no-unwind syscall has no OutcomeStructured effect; its outcome + // twin is still valid and simply returns success on every normal return. + localStaticEffects := facts.Effect & MayPark if function == nil || !classified || !facts.StaticOutcomeLocal || facts.Effect.Contains(YieldOnly) || + facts.Effect&^MayPark != NoSuspend || len(function.FreeVars) != 0 || invalidCall[plan.ID] || plan.HasStaticOutcome() || plan.External != Defined || plan.Emission != EmitCoroutine || plan.ManagedEntry != ManagedEntryCoroutine || plan.Primary != PrimaryCoroutine || plan.ManagedDemand == NoDemand || plan.RawPlainOnly || - plan.Recursive || plan.Effect&^(YieldOnly|AwaitStructured|OutcomeStructured) != 0 || - !plan.Effect.Contains(OutcomeStructured) || + plan.Recursive || plan.DeclaredEffect&^(OutcomeStructured|localStaticEffects) != 0 || + plan.LocalEffect&^(AwaitStructured|OutcomeStructured|localStaticEffects) != 0 || + plan.Effect&^(AwaitStructured|OutcomeStructured|MayPark) != 0 || plan.Exec&(BlockForeign|ThreadAffine|NeedsCleanupFrame|OpaqueExec) != 0 { continue }