Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions cl/coro_abi.go
Original file line number Diff line number Diff line change
Expand Up @@ -1460,8 +1460,7 @@ func validateCoroPhysicalABIForOwner(
plan.StaticOutcome && plan.Exec&(coro.BlockForeign|coro.ThreadAffine|coro.NeedsCleanupFrame|coro.OpaqueExec) == 0) &&
(plan.FuncRep == coro.DirectCoro || outcomePlainTwin && plan.FuncRep == coro.Dispatch) &&
(plan.Effect == coro.OutcomeStructured || outcomePlainTwin &&
plan.Effect.Contains(coro.OutcomeStructured) &&
plan.Effect&^(coro.YieldOnly|coro.AwaitStructured|coro.OutcomeStructured) == 0)
plan.Effect&^(coro.AwaitStructured|coro.OutcomeStructured|coro.MayPark) == 0)
if plan.Emission != coro.EmitCoroutine && !outcomePlain ||
plan.FuncRep != coro.DirectCoro && !managedDispatchTarget && !rawMethodDispatchToken {
return fail("requires a direct coroutine/outcome or capability-certified Dispatch emission, got emission=%s representation=%s", plan.Emission, plan.FuncRep)
Expand Down Expand Up @@ -1781,6 +1780,7 @@ func validateCoroPhysicalABIForOwner(
panics := 0
awaits := 0
parks := 0
nativeBlocks := 0
foreignWaits := 0
yields := 0
spawns := 0
Expand Down Expand Up @@ -2016,6 +2016,13 @@ func validateCoroPhysicalABIForOwner(
"generated C2 foreign suspend has no frozen typed errno operation")
}
foreignWaits++
} else if intrinsic && semantics == CoroIntrinsicCallInlineNativeBlock {
if !isLLGoSyscallIntrinsic(frozen.opcode) ||
instructionPlan.operation != coroPhysicalOperationNativeSyscall {
return coroLeafInstructionError(fn, plan, instr,
"native blocking intrinsic has no frozen native-syscall recipe")
}
nativeBlocks++
} else if intrinsic && semantics == CoroIntrinsicCallInlineSuspend {
if isLLGoSyscallIntrinsic(frozen.opcode) {
if instructionPlan.operation != coroPhysicalOperationWorkerSyscall {
Expand All @@ -2036,7 +2043,9 @@ func validateCoroPhysicalABIForOwner(
goexits++
}
if intrinsic {
if isLLGoSyscallIntrinsic(frozen.opcode) && semantics != CoroIntrinsicCallInlineSuspend {
if isLLGoSyscallIntrinsic(frozen.opcode) &&
semantics != CoroIntrinsicCallInlineSuspend &&
semantics != CoroIntrinsicCallInlineNativeBlock {
return coroLeafInstructionError(fn, plan, instr,
"elided worker llgo.syscall has no frozen function-word capability")
}
Expand Down Expand Up @@ -2166,8 +2175,8 @@ func validateCoroPhysicalABIForOwner(
if awaits != 0 && !plan.Effect.Contains(coro.AwaitStructured) {
return fail("child-await body lacks await-structured final effect: %s", plan.Effect)
}
if parks != 0 && !plan.Effect.Contains(coro.MayPark) {
return fail("structured-park body lacks may-park final effect: %s", plan.Effect)
if parks+nativeBlocks != 0 && !plan.Effect.Contains(coro.MayPark) {
return fail("park/native-block body lacks may-park final effect: %s", plan.Effect)
}
if foreignWaits != 0 && !plan.Effect.Contains(coro.WaitForeign) {
return fail("bounded worker body lacks wait-foreign final effect: %s", plan.Effect)
Expand All @@ -2183,8 +2192,8 @@ func validateCoroPhysicalABIForOwner(
!plan.Effect.Contains(coro.OutcomeStructured)) {
return fail("Goexit body lacks outcome-structured owner effect: declared=%s local=%s final=%s", plan.DeclaredEffect, plan.LocalEffect, plan.Effect)
}
if plan.DeclaredEffect.Contains(coro.MayPark) && parks == 0 {
return fail("declared may-park effect has no exact structured park intrinsic")
if plan.DeclaredEffect.Contains(coro.MayPark) && parks+nativeBlocks == 0 {
return fail("declared may-park effect has no exact structured park or native blocking intrinsic")
}
if plan.DeclaredEffect.Contains(coro.WaitForeign) && foreignWaits == 0 {
return fail("declared wait-foreign effect has no exact bounded worker operation")
Expand Down
1 change: 1 addition & 0 deletions cl/coro_defer.go
Original file line number Diff line number Diff line change
Expand Up @@ -1256,6 +1256,7 @@ func validateCoroStaticCleanupNoUnwind(
return fmt.Sprintf("block %d intrinsic: %v", block.Index, err)
}
if !intrinsic || (semantics != CoroIntrinsicCallInlineNoSuspend && semantics != CoroIntrinsicCallInlineSuspend &&
semantics != CoroIntrinsicCallInlineNativeBlock &&
semantics != CoroIntrinsicCallInlineYield) {
return fmt.Sprintf("block %d intrinsic has unproved semantics %d", block.Index, uint8(semantics))
}
Expand Down
8 changes: 6 additions & 2 deletions cl/coro_frame_retention.go
Original file line number Diff line number Diff line change
Expand Up @@ -1572,7 +1572,9 @@ func (b *coroFrameRetentionRootBuilder) boundedCallKind(call *ssa.Call) (coroFra
allocation := coroFrameRetentionDirectAllocRoot(call.Common().Args[0], make(map[ssa.Value]bool))
if _, retained := b.proof.allocations[allocation]; retained {
semantics, intrinsic, err := coroIntrinsicCallSiteSemantics(b.audit.universe, call)
if err == nil && (!intrinsic || semantics != CoroIntrinsicCallInlineSuspend) {
if err == nil && (!intrinsic ||
(semantics != CoroIntrinsicCallInlineSuspend &&
semantics != CoroIntrinsicCallInlineNativeBlock)) {
return coroFrameRetentionCallParkOwnerV1, true
}
}
Expand All @@ -1586,7 +1588,9 @@ func (b *coroFrameRetentionRootBuilder) boundedCallKind(call *ssa.Call) (coroFra
return coroFrameRetentionCallWorkerV1, true
}
}
if err == nil && intrinsic && semantics == CoroIntrinsicCallInlineSuspend {
if err == nil && intrinsic &&
(semantics == CoroIntrinsicCallInlineSuspend ||
semantics == CoroIntrinsicCallInlineNativeBlock) {
workerCertified := false
if b.audit.plan == nil {
// Report-only physical audits have no lowering authority. They may
Expand Down
4 changes: 3 additions & 1 deletion cl/coro_frame_roots_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,9 @@ func TestCoroFrameExactRootsAndUintptrKeepaliveAreFrozen(t *testing.T) {
childCall = instruction
}
semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(instruction)
if err == nil && intrinsic && semantics == CoroIntrinsicCallInlineSuspend {
if err == nil && intrinsic &&
(semantics == CoroIntrinsicCallInlineSuspend ||
semantics == CoroIntrinsicCallInlineNativeBlock) {
workerCall = instruction
}
case *ssa.IndexAddr:
Expand Down
2 changes: 2 additions & 0 deletions cl/coro_lowering_facts.go
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,8 @@ func coroIntrinsicLoweringRecipe(semantics CoroIntrinsicCallSemantics) (coro.Rec
return coro.RecipeID("cl.intrinsic.inline-with-helpers.v0"), coro.NoSuspend
case CoroIntrinsicCallInlineSuspend:
return coro.RecipeID("cl.intrinsic.inline-suspend.v0"), coro.MayPark
case CoroIntrinsicCallInlineNativeBlock:
return coro.RecipeID("cl.intrinsic.inline-native-block.v1"), coro.MayPark
case CoroIntrinsicCallInlineForeignSuspend:
return coro.RecipeID("cl.intrinsic.inline-foreign-suspend.v1"), coro.WaitForeign
case CoroIntrinsicCallInlineYield:
Expand Down
4 changes: 3 additions & 1 deletion cl/coro_outcome_plain.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,9 @@ func validateStaticOutcomeFrozenPlan(plan *coroPhysicalFunctionPlan, logical cor
}
operationSupported := physical.operation == coroPhysicalOperationNone ||
physical.operation == coroPhysicalOperationControl &&
!physical.operationControl.NativeActivationBound()
!physical.operationControl.NativeActivationBound() ||
physical.operation == coroPhysicalOperationNativeSyscall &&
physical.semantic.recipe == coro.RecipeID("cl.intrinsic.inline-native-block.v1")
if !operationSupported || physical.controlFailureHard ||
physical.operationFailure != "" || physical.outcomeFailure != "" {
return fmt.Errorf(
Expand Down
15 changes: 13 additions & 2 deletions cl/coro_physical_plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,10 @@ const (
coroPhysicalOperationWorkerCgoErrno
coroPhysicalOperationHostCall
coroPhysicalOperationControl
// NativeSyscall executes the compiler-certified word call synchronously on
// the current M. It owns no park/resume transaction and must not request the
// program worker fleet.
coroPhysicalOperationNativeSyscall
)

func (recipe coroPhysicalOperationRecipe) String() string {
Expand Down Expand Up @@ -225,6 +229,8 @@ func (recipe coroPhysicalOperationRecipe) String() string {
return "host-operation"
case coroPhysicalOperationControl:
return "control-operation"
case coroPhysicalOperationNativeSyscall:
return "native-syscall"
default:
return fmt.Sprintf("physical-operation-recipe(%d)", uint8(recipe))
}
Expand Down Expand Up @@ -1407,7 +1413,8 @@ func planCoroPhysicalOperationInstruction(
return
}
if found && frozen.plan.Intrinsic && isLLGoSyscallIntrinsic(frozen.opcode) &&
frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineSuspend {
(frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineSuspend ||
frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineNativeBlock) {
if !capabilities.worker {
result.operationFailure = "worker llgo.syscall requires the bounded worker capability"
return
Expand All @@ -1416,7 +1423,11 @@ func planCoroPhysicalOperationInstruction(
result.operationFailure = "invalid worker llgo.syscall capability: " + err.Error()
return
}
result.operation = coroPhysicalOperationWorkerSyscall
if frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineNativeBlock {
result.operation = coroPhysicalOperationNativeSyscall
} else {
result.operation = coroPhysicalOperationWorkerSyscall
}
return
}
}
Expand Down
7 changes: 7 additions & 0 deletions cl/coro_physical_plan_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,13 @@ func Worker(enabled bool) {
}

plan = physical.instructions[reachable]
plan.operation = coroPhysicalOperationNativeSyscall
physical.instructions[reachable] = plan
capabilities, err = commit().programCapabilities()
if err != nil || capabilities.Worker() {
t.Fatalf("reachable native syscall capability = (%v, %v), want no worker", capabilities, err)
}

plan.operation = coroPhysicalOperationWorkerCgo
physical.instructions[reachable] = plan
capabilities, err = commit().programCapabilities()
Expand Down
4 changes: 3 additions & 1 deletion cl/coro_poll_wait_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ func TestCoroPollWaitCurrentFrameNativeAndWasm32(t *testing.T) {
rootPlan, ok := plan.FunctionPlan(root)
if !ok || rootPlan.Emission != coro.EmitCoroutine || rootPlan.FuncRep != coro.DirectCoro ||
!rootPlan.DeclaredEffect.Contains(coro.MayPark) || !rootPlan.LocalEffect.Contains(coro.MayPark) ||
!rootPlan.Effect.Contains(coro.MayPark) {
!rootPlan.Effect.Contains(coro.MayPark) || rootPlan.HasStaticOutcome() {
t.Fatalf("Root plan = %+v, present=%t; want one local poll-park coroutine", rootPlan, ok)
}
if !plan.ElidesCall(waitCall) {
Expand Down Expand Up @@ -268,6 +268,8 @@ func compileCoroPollWaitFixture(t *testing.T, target *llssa.Target) (
EmissionUniverse: ssaUniverse,
FunctionIDs: functionIDs,
MaxPlainInstructions: -1,
OutcomeMode: coro.OutcomeExplicitStatus,
ClassifyLocalBody: universe.CoroLocalBodyFacts,
ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) {
if fn == root {
return coro.SSAFunctionPolicy{Effect: coro.MayPark}, nil
Expand Down
40 changes: 23 additions & 17 deletions cl/coro_program_ir.go
Original file line number Diff line number Diff line change
Expand Up @@ -412,14 +412,15 @@ func coroOutcomePlainDAGSemanticRecipe(plan coroSemanticInstructionPlan) bool {
// ProgramIR builder step admits only operations whose frozen call recipe is
// independently shape-checked and allocation-free.
//
// Atomic intrinsics acquire their bounded leaf proof here. A real inline yield
// is also recorded here as an evaluated local effect: this distinguishes an
// explicit scheduler handoff from the synthetic YieldOnly/NeedsPreempt seed
// added later for an otherwise synchronous CFG loop. Static outcome twins may
// omit the latter under the current compute-blocking policy, but must never
// erase the former. Other InlineNoSuspend intrinsics include asm, dynamic
// alloca, control transfer, and target-specific operations; the broad enum is
// therefore not by itself an outcome-plain proof.
// Atomic intrinsics acquire their bounded leaf proof here. Structured waits,
// real yields, and terminal outcomes are also projected into the local semantic
// facts so a static-outcome decision cannot mistake an elided declaration for
// a synchronous operation. The sole dual recipe is InlineNativeBlock: its
// certified native syscall releases the execution domain and returns on the
// same M without suspending the LLVM coroutine, so an exact static caller may
// use the synchronous outcome twin. Other InlineNoSuspend intrinsics include
// asm, dynamic alloca, control transfer, and target-specific operations; the
// broad enum is therefore not by itself an outcome-plain proof.
func (ir *coroProgramIR) finalizeOutcomePlainIntrinsicSemantics(
prog llssa.Program,
functions []*ssa.Function,
Expand Down Expand Up @@ -447,31 +448,36 @@ func (ir *coroProgramIR) finalizeOutcomePlainIntrinsicSemantics(
frozen.plan.Elision != CoroCallElidedIntrinsic {
continue
}
atomic := frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineNoSuspend &&
semantics := frozen.plan.IntrinsicSemantics
atomic := semantics == CoroIntrinsicCallInlineNoSuspend &&
isCoroAtomicIntrinsic(frozen.opcode)
realYield := frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineYield
if !atomic && !realYield {
structured := semantics == CoroIntrinsicCallInlineSuspend ||
semantics == CoroIntrinsicCallInlineNativeBlock ||
semantics == CoroIntrinsicCallInlineForeignSuspend ||
semantics == CoroIntrinsicCallInlineYield ||
semantics == CoroIntrinsicCallInlineOutcome
if !atomic && !structured {
continue
}
for _, owner := range sortedUseOwners(function) {
key := emissionFunctionOwnerKey{function: function, owner: owner}
if _, sealed := ir.siteOwners[key]; !sealed {
return fmt.Errorf("atomic intrinsic in %q has no frozen semantic owner %q", function.Name(), owner.identity)
return fmt.Errorf("intrinsic refinement in %q has no frozen semantic owner %q", function.Name(), owner.identity)
}
semantic, present := ir.semanticPlans[key][call]
if !present || !semantic.evaluated || semantic.recipe != coro.RecipeID("cl.ssa.call.v1") ||
semantic.effect != coro.NoSuspend || semantic.exec != 0 {
return fmt.Errorf("atomic intrinsic call %q has an incompatible preliminary semantic recipe", call.String())
return fmt.Errorf("intrinsic call %q has an incompatible preliminary semantic recipe", call.String())
}
if atomic {
semantic.recipe = coro.RecipeID("cl.intrinsic.atomic.inline-nosuspend.v1")
semantic.outcomePlainLeaf = true
semantic.staticOutcome = true
} else {
semantic.recipe, semantic.effect = coroIntrinsicLoweringRecipe(CoroIntrinsicCallInlineYield)
semantic.recipe, semantic.effect = coroIntrinsicLoweringRecipe(semantics)
semantic.materialized = true
semantic.outcomePlainLeaf = false
semantic.staticOutcome = false
semantic.staticOutcome = semantics == CoroIntrinsicCallInlineNativeBlock
}
ir.semanticPlans[key][call] = semantic
}
Expand All @@ -486,11 +492,11 @@ func (ir *coroProgramIR) finalizeOutcomePlainIntrinsicSemantics(
key := emissionFunctionOwnerKey{function: function, owner: owner}
preamble, present := ir.functionPreambles[key]
if !present {
return fmt.Errorf("finalize atomic intrinsic body %q: owner %q has no function preamble", function.Name(), owner.identity)
return fmt.Errorf("finalize intrinsic body %q: owner %q has no function preamble", function.Name(), owner.identity)
}
facts, err := deriveCoroLocalBodyFacts(prog, function, ir.semanticPlans[key], preamble.emitsGoBody)
if err != nil {
return fmt.Errorf("finalize atomic intrinsic body %q: %w", function.Name(), err)
return fmt.Errorf("finalize intrinsic body %q: %w", function.Name(), err)
}
if index != 0 && !final.Same(facts) {
return fmt.Errorf("function %q acquired owner-dependent finalized local semantic facts", function.Name())
Expand Down
6 changes: 4 additions & 2 deletions cl/coro_timer_sleep_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func TestCoroTimerSleepCurrentFrameNativeAndWasm32(t *testing.T) {
rootPlan, ok := plan.FunctionPlan(root)
if !ok || rootPlan.Emission != coro.EmitCoroutine || rootPlan.FuncRep != coro.DirectCoro ||
!rootPlan.DeclaredEffect.Contains(coro.MayPark) || !rootPlan.LocalEffect.Contains(coro.MayPark) ||
!rootPlan.Effect.Contains(coro.MayPark) {
!rootPlan.Effect.Contains(coro.MayPark) || rootPlan.HasStaticOutcome() {
t.Fatalf("Root plan = %+v, present=%t; want one local timer-park coroutine", rootPlan, ok)
}
if !plan.ElidesCall(sleepCall) {
Expand Down Expand Up @@ -175,7 +175,7 @@ func TestCoroControlledTimerWaitCurrentFrameNativeAndWasm32(t *testing.T) {

rootPlan, ok := plan.FunctionPlan(root)
if !ok || rootPlan.Emission != coro.EmitCoroutine || !rootPlan.Effect.Contains(coro.MayPark) ||
!plan.ElidesCall(waitCall) {
rootPlan.HasStaticOutcome() || !plan.ElidesCall(waitCall) {
t.Fatalf("controlled Timer Root plan = %+v, present=%t, elided=%t", rootPlan, ok, plan.ElidesCall(waitCall))
}
if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil {
Expand Down Expand Up @@ -290,6 +290,8 @@ func compileCoroTimerIntrinsicFixture(t *testing.T, target *llssa.Target, source
EmissionUniverse: ssaUniverse,
FunctionIDs: functionIDs,
MaxPlainInstructions: -1,
OutcomeMode: coro.OutcomeExplicitStatus,
ClassifyLocalBody: universe.CoroLocalBodyFacts,
ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) {
if fn == root {
return coro.SSAFunctionPolicy{Effect: coro.MayPark}, nil
Expand Down
Loading